Jeffrey Ward
fac3adbaf6
Text files sorted by stage 3 now flow onto a new work queue and worker pool that enrich them with natural language (Languages.jl LanguageDetector: name, ISO 639-3 code, confidence) and programming language (github-linguist), writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file pipeline. github-linguist reads the git blob of a path inside a repo, so untracked data/ files are copied to /tmp (outside any repo, name preserved for extension heuristics) before detection. Programming-language lookup is best-effort (startup warning if missing, degraded/null on failure); natural-language failure yields a degraded sidecar, not a quarantine. Factored exiftool's timeout-kill into shared run_with_timeout and the durable sidecar-first commit into commit_enriched!, both reused by stage 4. Recovery re-drives data/text/; graceful drain closes the text queue after its stage-3 producers finish.
36 lines
927 B
TOML
36 lines
927 B
TOML
name = "FileServer"
|
|
uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
|
|
version = "0.1.0"
|
|
authors = ["wardjm@gmail.com"]
|
|
|
|
[deps]
|
|
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
|
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
|
|
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
|
|
Languages = "8ef0a80b-9436-5d2c-a485-80b904378c43"
|
|
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
|
|
Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
|
|
Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
|
|
Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
|
|
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
|
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
|
|
|
|
[compat]
|
|
HTTP = "1.11.0"
|
|
JLD2 = "0.6.4"
|
|
JSON3 = "1.14.3"
|
|
Languages = "0.4.7"
|
|
Logging = "1.11.0"
|
|
Lux = "1.31.4"
|
|
Optimisers = "0.4.7"
|
|
Oxygen = "1.10.2"
|
|
UUIDs = "1.11.0"
|
|
Zygote = "0.7.11"
|
|
|
|
[extras]
|
|
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
|
|
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
|
|
|
|
[targets]
|
|
test = ["Test", "JSON3"]
|