Add stage-4 language enrichment for text files

Text files sorted by stage 3 now flow onto a new work queue and worker
pool that enrich them with natural language (Languages.jl LanguageDetector:
name, ISO 639-3 code, confidence) and programming language (github-linguist),
writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file
pipeline.

github-linguist reads the git blob of a path inside a repo, so untracked
data/ files are copied to /tmp (outside any repo, name preserved for
extension heuristics) before detection. Programming-language lookup is
best-effort (startup warning if missing, degraded/null on failure);
natural-language failure yields a degraded sidecar, not a quarantine.

Factored exiftool's timeout-kill into shared run_with_timeout and the
durable sidecar-first commit into commit_enriched!, both reused by stage 4.
Recovery re-drives data/text/; graceful drain closes the text queue after
its stage-3 producers finish.
This commit is contained in:
2026-07-03 11:38:50 -04:00
parent 9fd1bf385b
commit fac3adbaf6
9 changed files with 522 additions and 81 deletions

View File

@@ -2,7 +2,7 @@
julia_version = "1.12.6"
manifest_format = "2.0"
project_hash = "1d7ce552eaac13c97732dccac45c97d40527ec58"
project_hash = "a623ff56053e3a56c1799a1cb2080ec48d933b73"
[[deps.ADTypes]]
git-tree-sha1 = "d9aaef7c63466eee4de23b4d9dad03629df54bea"
@@ -309,7 +309,7 @@ weakdeps = ["HTTP"]
HTTPExt = "HTTP"
[[deps.FileServer]]
deps = ["HTTP", "JSON3", "Logging", "Oxygen", "UUIDs"]
deps = ["HTTP", "JLD2", "JSON3", "Logging", "Lux", "Optimisers", "Oxygen", "UUIDs", "Zygote"]
path = "."
uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
version = "0.1.0"
@@ -468,6 +468,12 @@ weakdeps = ["Serialization"]
[deps.LRUCache.extensions]
SerializationExt = ["Serialization"]
[[deps.Languages]]
deps = ["InteractiveUtils", "JSON", "RelocatableFolders"]
git-tree-sha1 = "023ac3b12f82da68ed2556c71a134a03e1a11343"
uuid = "8ef0a80b-9436-5d2c-a485-80b904378c43"
version = "0.4.7"
[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"