Add stage-4 language enrichment for text files

Text files sorted by stage 3 now flow onto a new work queue and worker
pool that enrich them with natural language (Languages.jl LanguageDetector:
name, ISO 639-3 code, confidence) and programming language (github-linguist),
writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file
pipeline.

github-linguist reads the git blob of a path inside a repo, so untracked
data/ files are copied to /tmp (outside any repo, name preserved for
extension heuristics) before detection. Programming-language lookup is
best-effort (startup warning if missing, degraded/null on failure);
natural-language failure yields a degraded sidecar, not a quarantine.

Factored exiftool's timeout-kill into shared run_with_timeout and the
durable sidecar-first commit into commit_enriched!, both reused by stage 4.
Recovery re-drives data/text/; graceful drain closes the text queue after
its stage-3 producers finish.
This commit is contained in:
2026-07-03 11:38:50 -04:00
parent 9fd1bf385b
commit fac3adbaf6
9 changed files with 522 additions and 81 deletions

View File

@@ -7,6 +7,7 @@ authors = ["wardjm@gmail.com"]
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
Languages = "8ef0a80b-9436-5d2c-a485-80b904378c43"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
@@ -14,20 +15,21 @@ Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
[extras]
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
[targets]
test = ["Test", "JSON3"]
[compat]
HTTP = "1.11.0"
JLD2 = "0.6.4"
JSON3 = "1.14.3"
Languages = "0.4.7"
Logging = "1.11.0"
Lux = "1.31.4"
Optimisers = "0.4.7"
Oxygen = "1.10.2"
UUIDs = "1.11.0"
Zygote = "0.7.11"
[extras]
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
[targets]
test = ["Test", "JSON3"]