Add stage-4 language enrichment for text files

Text files sorted by stage 3 now flow onto a new work queue and worker
pool that enrich them with natural language (Languages.jl LanguageDetector:
name, ISO 639-3 code, confidence) and programming language (github-linguist),
writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file
pipeline.

github-linguist reads the git blob of a path inside a repo, so untracked
data/ files are copied to /tmp (outside any repo, name preserved for
extension heuristics) before detection. Programming-language lookup is
best-effort (startup warning if missing, degraded/null on failure);
natural-language failure yields a degraded sidecar, not a quarantine.

Factored exiftool's timeout-kill into shared run_with_timeout and the
durable sidecar-first commit into commit_enriched!, both reused by stage 4.
Recovery re-drives data/text/; graceful drain closes the text queue after
its stage-3 producers finish.
This commit is contained in:
2026-07-03 11:38:50 -04:00
parent 9fd1bf385b
commit fac3adbaf6
9 changed files with 522 additions and 81 deletions

155
src/language.jl Normal file
View File

@@ -0,0 +1,155 @@
# Stage-4 language enrichment for text files.
#
# A file that stage-3 sorted into `text/` is human-readable, but we don't yet
# know *what* it is. This stage answers two questions and records them in a
# `.meta.json` sidecar, exactly like the stage-2 known-file enrichment:
#
# * natural language — via Languages.jl's `LanguageDetector` (a Julia port of
# the `whatlang` n-gram model): English vs. French vs. Japanese, plus a
# confidence score. Pure Julia, no subprocess.
# * programming language — via the `github-linguist` CLI, which recognizes
# source and markup by extension + content heuristics. There is no
# comparable native Julia library, so we shell out (mirroring stage-2's
# exiftool dependency).
#
# Neither detector failing quarantines the file: a text file is wanted whether
# or not we can name its language, so a failure yields a *degraded* sidecar
# (what we know plus an `error` note), just like stage 2.
#
# The github-linguist quirk that shapes this code: run against a path *inside* a
# git repository, linguist reads the file's committed git blob, not the bytes on
# disk — and an untracked file (which every file under `data/` is) has no blob,
# so it crashes. We sidestep this by copying the file to a fresh temp dir outside
# any repo (preserving its name so linguist's extension heuristics still fire)
# and pointing linguist there.
# How much of a text file to feed the natural-language detector. The whatlang
# model saturates quickly, so a bounded prefix keeps memory flat on huge logs
# while still giving the detector plenty of signal.
const LANG_SAMPLE_BYTES = 65_536
"Return true if the `github-linguist` binary is on PATH."
function linguist_available()
try
Base.run(pipeline(`github-linguist --version`; stdout=devnull, stderr=devnull))
return true
catch
return false
end
end
"""
read_text_sample(path) -> String
Read up to `LANG_SAMPLE_BYTES` of `path` as UTF-8 text, trimming a multi-byte
character the window may have cut in half (reusing stage-3's `trim_truncated_utf8`)
so the tail isn't misread as garbage.
"""
function read_text_sample(path::AbstractString)::String
open(path, "r") do io
chunk = read(io, LANG_SAMPLE_BYTES)
return String(copy(trim_truncated_utf8(chunk)))
end
end
"""
detect_natural_language(detector, text) -> (name, code, confidence)
Run the `LanguageDetector` on `text`, returning the language's English name
(e.g. `"English"`), its ISO 639-3 code (e.g. `"eng"`), and the model's
confidence in `[0,1]`. Returns `(nothing, nothing, nothing)` when there is no
usable text (empty/whitespace) or the detector errors — the caller records that
as a degraded result rather than failing the file.
"""
function detect_natural_language(detector, text::AbstractString)
isempty(strip(text)) && return (nothing, nothing, nothing)
try
lang, _script, confidence = detector(text)
return (Languages.english_name(lang), Languages.isocode(lang), confidence)
catch
return (nothing, nothing, nothing)
end
end
"""
run_linguist(path, timeout) -> Union{String,Nothing}
Ask `github-linguist --json` for the programming/markup language of the file at
`path`, returning the language name (e.g. `"Python"`, `"Markdown"`) or `nothing`
when linguist can't identify one. Plain prose reports as `"Text"` and
unrecognized content as JSON `null`; both collapse to `nothing` here, since only
a real programming/markup language is worth recording.
`path` MUST be outside any git repository — see the module header for why.
"""
function run_linguist(path::AbstractString, timeout::Integer)
bytes = run_with_timeout(`github-linguist --json $path`, timeout)
bytes === nothing && return nothing
parsed = try
JSON3.read(String(bytes))
catch
return nothing
end
# linguist --json emits a single object keyed by the file path; pull the one
# entry rather than depend on the exact key spelling.
isempty(parsed) && return nothing
entry = first(values(parsed))
lang = get(entry, :language, nothing)
(lang === nothing || lang == "Text") && return nothing
return String(lang)
end
"""
detect_programming_language(job, cfg) -> Union{String,Nothing}
Programming/markup language of a text file, or `nothing`. Copies the file to a
throwaway temp dir *outside* the git repo — under its sanitized original name so
linguist's extension heuristics still apply — runs linguist there, and cleans up.
"""
function detect_programming_language(job::Job, cfg::Config)
lang = nothing
mktempdir() do dir # tempdir() → /tmp, outside the repo
safe = sanitize_filename(job.original_name)
tmp = joinpath(dir, safe)
cp(job.path, tmp; force=true)
lang = run_linguist(tmp, cfg.linguist_timeout)
end
return lang
end
"""
build_text_metadata(detector, job, cfg) -> NamedTuple
Build the stage-4 sidecar payload for a text file: its natural language (name +
ISO code + confidence) and programming/markup language, plus the Job's
authoritative id/name/size. `error` is set only when natural-language detection
produced nothing usable (the file is still enriched and committed); programming
language is best-effort and its absence is normal, not an error.
"""
function build_text_metadata(detector, job::Job, cfg::Config)
text = read_text_sample(job.path)
name, code, confidence = detect_natural_language(detector, text)
programming_language = detect_programming_language(job, cfg)
return (
id = job.id,
original_name = job.original_name,
file_size = job.size, # authoritative, from intake
content_type = "text",
language = name,
language_code = code,
language_confidence = confidence,
programming_language = programming_language,
error = name === nothing ? "language detection produced no result" : nothing,
)
end
"""
finalize_text!(cfg, job, meta) -> (file_dest, sidecar_dest)
Commit an enriched text file (stage 4) to `text_done/` via the shared
sidecar-first `commit_enriched!`, giving text files the same crash-safe
"file implies sidecar" guarantee as stage-2 known files.
"""
finalize_text!(cfg::Config, job::Job, meta) = commit_enriched!(cfg.text_done_dir, job, meta)