Add stage-4 language enrichment for text files

Text files sorted by stage 3 now flow onto a new work queue and worker
pool that enrich them with natural language (Languages.jl LanguageDetector:
name, ISO 639-3 code, confidence) and programming language (github-linguist),
writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file
pipeline.

github-linguist reads the git blob of a path inside a repo, so untracked
data/ files are copied to /tmp (outside any repo, name preserved for
extension heuristics) before detection. Programming-language lookup is
best-effort (startup warning if missing, degraded/null on failure);
natural-language failure yields a degraded sidecar, not a quarantine.

Factored exiftool's timeout-kill into shared run_with_timeout and the
durable sidecar-first commit into commit_enriched!, both reused by stage 4.
Recovery re-drives data/text/; graceful drain closes the text queue after
its stage-3 producers finish.
This commit is contained in:
2026-07-03 11:38:50 -04:00
parent 9fd1bf385b
commit fac3adbaf6
9 changed files with 522 additions and 81 deletions

View File

@@ -1,11 +1,12 @@
# Worker tasks: pull jobs off a queue and process them. The loop scaffolding
# (dequeue-until-drained, try/catch, quarantine-on-throw) is identical for every
# stage, so `worker_loop` is parametrized with a `handler` and reused. Today
# there are three stages:
# there are four stages:
#
# stage 1 handle_classify_job spool/ → classify → known/ (+known queue) | unknown/ (+unknown queue)
# stage 2 handle_known_job known/ → exiftool enrich → done/ (+ .meta.json)
# stage 3 handle_unknown_job unknown/ → binary-vs-text sniff → binary/ | text/
# stage 3 handle_unknown_job unknown/ → binary-vs-text sniff → binary/ | text/ (+text queue)
# stage 4 handle_text_job text/ → language enrich → text_done/ (+ .meta.json)
#
# Adding a stage later is just another queue + pool + handler; the loop below
# doesn't change.
@@ -70,17 +71,42 @@ function handle_known_job(job::Job, cfg::Config, worker_id::Int)
end
"""
handle_unknown_job(job, cfg, worker_id)
handle_unknown_job(job, cfg, worker_id, text_queue)
Stage 3. Sort an unrecognized file into a coarse content bucket by sniffing its
first bytes: `binary/` if it looks like binary data, `text/` otherwise. Terminal
— there is no further stage. Simple by design for now; richer handling can hang
off either bucket later.
first bytes: `binary/` (terminal — no further stage) if it looks like binary
data, `text/` otherwise. A text file is then routed onward to the stage-4
language-enrichment queue, retrying on a full queue rather than dropping the file
(the same blocking backpressure stage 1 uses for its downstream queues).
"""
function handle_unknown_job(job::Job, cfg::Config, worker_id::Int)
binary = is_binary(job.path)
dest = move_to(binary ? cfg.binary_dir : cfg.text_dir, job)
@info "sorted unknown" worker=worker_id id=job.id name=job.original_name kind=(binary ? :binary : :text) dest=dest
function handle_unknown_job(job::Job, cfg::Config, worker_id::Int, text_queue::JobQueue)
if is_binary(job.path)
dest = move_to(cfg.binary_dir, job)
@info "sorted unknown" worker=worker_id id=job.id name=job.original_name kind=:binary dest=dest
else
dest = move_to(cfg.text_dir, job)
routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
while !enqueue!(text_queue, routed)
sleep(ROUTE_ENQUEUE_RETRY_SECONDS) # text queue full → back off, don't drop
end
@info "routed to language enrichment" worker=worker_id id=job.id dest=dest
end
return nothing
end
"""
handle_text_job(job, cfg, worker_id, detector)
Stage 4. Enrich a text file with its natural language (via `detector`) and
programming language (via github-linguist): build the sidecar and commit both to
`text_done/` sidecar-first. Detection failure yields a *degraded* sidecar (the
file is still wanted text), so the only way to land in `failed/` is a genuine I/O
error committing — handled by `worker_loop`'s quarantine.
"""
function handle_text_job(job::Job, cfg::Config, worker_id::Int, detector)
meta = build_text_metadata(detector, job, cfg)
file_dest, sidecar = finalize_text!(cfg, job, meta)
@info "enriched text" worker=worker_id id=job.id dest=file_dest sidecar=basename(sidecar) language=meta.language confidence=meta.language_confidence programming_language=meta.programming_language degraded=(meta.error !== nothing)
return nothing
end