Add stage-4 language enrichment for text files
Text files sorted by stage 3 now flow onto a new work queue and worker pool that enrich them with natural language (Languages.jl LanguageDetector: name, ISO 639-3 code, confidence) and programming language (github-linguist), writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file pipeline. github-linguist reads the git blob of a path inside a repo, so untracked data/ files are copied to /tmp (outside any repo, name preserved for extension heuristics) before detection. Programming-language lookup is best-effort (startup warning if missing, degraded/null on failure); natural-language failure yields a degraded sidecar, not a quarantine. Factored exiftool's timeout-kill into shared run_with_timeout and the durable sidecar-first commit into commit_enriched!, both reused by stage 4. Recovery re-drives data/text/; graceful drain closes the text queue after its stage-3 producers finish.
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
|
||||
julia_version = "1.12.6"
|
||||
manifest_format = "2.0"
|
||||
project_hash = "1d7ce552eaac13c97732dccac45c97d40527ec58"
|
||||
project_hash = "a623ff56053e3a56c1799a1cb2080ec48d933b73"
|
||||
|
||||
[[deps.ADTypes]]
|
||||
git-tree-sha1 = "d9aaef7c63466eee4de23b4d9dad03629df54bea"
|
||||
@@ -309,7 +309,7 @@ weakdeps = ["HTTP"]
|
||||
HTTPExt = "HTTP"
|
||||
|
||||
[[deps.FileServer]]
|
||||
deps = ["HTTP", "JSON3", "Logging", "Oxygen", "UUIDs"]
|
||||
deps = ["HTTP", "JLD2", "JSON3", "Logging", "Lux", "Optimisers", "Oxygen", "UUIDs", "Zygote"]
|
||||
path = "."
|
||||
uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
|
||||
version = "0.1.0"
|
||||
@@ -468,6 +468,12 @@ weakdeps = ["Serialization"]
|
||||
[deps.LRUCache.extensions]
|
||||
SerializationExt = ["Serialization"]
|
||||
|
||||
[[deps.Languages]]
|
||||
deps = ["InteractiveUtils", "JSON", "RelocatableFolders"]
|
||||
git-tree-sha1 = "023ac3b12f82da68ed2556c71a134a03e1a11343"
|
||||
uuid = "8ef0a80b-9436-5d2c-a485-80b904378c43"
|
||||
version = "0.4.7"
|
||||
|
||||
[[deps.LibCURL]]
|
||||
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
|
||||
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
|
||||
|
||||
Reference in New Issue
Block a user