jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add stage-4 language enrichment for text files

Browse Source 
Text files sorted by stage 3 now flow onto a new work queue and worker
pool that enrich them with natural language (Languages.jl LanguageDetector:
name, ISO 639-3 code, confidence) and programming language (github-linguist),
writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file
pipeline.

github-linguist reads the git blob of a path inside a repo, so untracked
data/ files are copied to /tmp (outside any repo, name preserved for
extension heuristics) before detection. Programming-language lookup is
best-effort (startup warning if missing, degraded/null on failure);
natural-language failure yields a degraded sidecar, not a quarantine.

Factored exiftool's timeout-kill into shared run_with_timeout and the
durable sidecar-first commit into commit_enriched!, both reused by stage 4.
Recovery re-drives data/text/; graceful drain closes the text queue after
its stage-3 producers finish.
This commit is contained in:
Jeffrey Ward
2026-07-03 11:38:50 -04:00
parent 9fd1bf385b
commit fac3adbaf6
9 changed files with 522 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/FileServer.jl
View File

@@ -7,6 +7,7 @@ using JSON3
using Oxygen
using Lux
using JLD2
using Languages
include("config.jl")
include("job.jl")
@@ -16,6 +17,7 @@ include("model.jl") # build_model() + read_features(); shared with bin/train
include("classify.jl") # Classifier + load_classifier/classify (needs model.jl)
include("metadata.jl") # exiftool extraction + sidecar enrichment (stage 2)
include("content.jl") # binary-vs-text triage for unknown files (stage 3)
include("language.jl") # natural + programming language enrichment for text (stage 4)
include("worker.jl")
# Globals the HTTP handlers read at request time. Set once in `run`, before the
@@ -25,7 +27,9 @@ const CONFIG = Ref{Config}()
const QUEUE = Ref{ChannelQueue}() # stage-1 (classification) queue; HTTP intake enqueues here
const KNOWN_QUEUE = Ref{ChannelQueue}() # stage-2 (enrichment) queue; stage-1 workers enqueue here
const UNKNOWN_QUEUE = Ref{ChannelQueue}() # stage-3 (content triage) queue; stage-1 workers enqueue here
const TEXT_QUEUE = Ref{ChannelQueue}() # stage-4 (language enrichment) queue; stage-3 workers enqueue here
const CLASSIFIER = Ref{Classifier}() # loaded once at startup, shared read-only across workers
const DETECTOR = Ref{LanguageDetector}() # natural-language detector; built once at startup, shared read-only
include("server.jl") # registers routes (references CONFIG/QUEUE at call time)
@@ -74,43 +78,60 @@ function run(; overrides...)
# All pools draw from the same OS threads. Warn on the *combined* size (still
# allowed): oversubscription just means tasks share threads, not a failure.
total_workers = cfg.worker_count + cfg.known_worker_count + cfg.unknown_worker_count
total_workers = cfg.worker_count + cfg.known_worker_count + cfg.unknown_worker_count + cfg.text_worker_count
if total_workers > Threads.nthreads()
@warn "combined worker count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" classify_workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count total=total_workers nthreads=Threads.nthreads()
@warn "combined worker count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" classify_workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count total=total_workers nthreads=Threads.nthreads()
end
# exiftool is a hard prerequisite for stage-2 enrichment. Fail fast at
# startup rather than discover it missing on the first known file.
assert_exiftool()
# github-linguist powers stage-4 *programming*-language detection, but it's
# best-effort (natural-language enrichment stands on its own), so a missing
# binary is a warning, not a fatal error — per-file lookups degrade to none.
linguist_available() || @warn "github-linguist not found on PATH; stage-4 text files will have no programming language (install it to enable)"
queue = ChannelQueue(cfg.queue_capacity)
known_queue = ChannelQueue(cfg.known_queue_capacity)
unknown_queue = ChannelQueue(cfg.unknown_queue_capacity)
text_queue = ChannelQueue(cfg.text_queue_capacity)
CONFIG[] = cfg
QUEUE[] = queue
KNOWN_QUEUE[] = known_queue
UNKNOWN_QUEUE[] = unknown_queue
TEXT_QUEUE[] = text_queue
# Load the classifier before serving. Fail fast: a server that silently
# doesn't classify is a worse surprise than a clear startup error.
CLASSIFIER[] = load_classifier(cfg.model_path)
@info "loaded classifier" path=cfg.model_path
# Build the natural-language detector once (it loads the whatlang n-gram
# model) and share it read-only across the stage-4 pool, like the classifier.
DETECTOR[] = LanguageDetector()
@info "loaded language detector"
# Stage-aware recovery: re-drive each stage's leftovers onto its own queue so
# files resume where they were, not from scratch. spool/ → stage-1,
# known/ → stage-2, unknown/ → stage-3.
recovered = recover_dir!(cfg.spool_dir, queue)
recovered_known = recover_dir!(cfg.known_dir, known_queue)
recovered_unknown = recover_dir!(cfg.unknown_dir, unknown_queue)
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity recovered=recovered recovered_known=recovered_known recovered_unknown=recovered_unknown
recovered_text = recover_dir!(cfg.text_dir, text_queue)
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity recovered=recovered recovered_known=recovered_known recovered_unknown=recovered_unknown recovered_text=recovered_text
workers = [Threads.@spawn worker_loop(i, cfg, queue,
(job, c, wid) -> handle_classify_job(job, c, wid, known_queue, unknown_queue))
for i in 1:cfg.worker_count]
known_workers = [Threads.@spawn worker_loop(i, cfg, known_queue, handle_known_job)
for i in 1:cfg.known_worker_count]
unknown_workers = [Threads.@spawn worker_loop(i, cfg, unknown_queue, handle_unknown_job)
unknown_workers = [Threads.@spawn worker_loop(i, cfg, unknown_queue,
(job, c, wid) -> handle_unknown_job(job, c, wid, text_queue))
for i in 1:cfg.unknown_worker_count]
text_workers = [Threads.@spawn worker_loop(i, cfg, text_queue,
(job, c, wid) -> handle_text_job(job, c, wid, DETECTOR[]))
for i in 1:cfg.text_worker_count]
register_routes()
serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false)
@@ -130,10 +151,12 @@ function run(; overrides...)
close!(queue) # 2. no new classify jobs; stage-1 drains buffered
foreach(wait, workers) # 3. wait out stage-1 — the ONLY producer of BOTH the
# known and unknown queues — so nothing else enqueues
close!(known_queue) # 4. now safe to close the downstream queues
close!(known_queue) # 4. now safe to close the queues stage-1 fed
close!(unknown_queue)
foreach(wait, known_workers) # 5. wait out stage-2 and stage-3
foreach(wait, unknown_workers)
foreach(wait, known_workers) # 5. wait out stage-2 (terminal) and stage-3 — stage-3 is
foreach(wait, unknown_workers) # the ONLY producer of the text queue
close!(text_queue) # 6. now safe to close the queue stage-3 fed
foreach(wait, text_workers) # 7. wait out stage-4
@info "shutdown complete"
end
atexit(drain)