Add stage-3 content triage: sort unknown files into binary/ and text/
Unknown files are no longer terminal. Stage 1 now routes :unknown onto a dedicated queue (with the same blocking backpressure as the known queue), and a third worker pool sorts each file into data/binary/ or data/text/ using a NUL-byte sniff of the first 8000 bytes. - content.jl: is_binary content sniff (stage 3) - worker.jl: handle_unknown_job; stage-1 routes unknown with backpressure; KNOWN_ENQUEUE_RETRY_SECONDS -> ROUTE_ENQUEUE_RETRY_SECONDS (serves both) - config.jl: unknown_worker_count/queue_capacity, binary_dir, text_dir + env - FileServer.jl: unknown queue, pool, stage-aware recovery, drain ordering - tests for is_binary and handle_unknown_job; tmp_config isolates new dirs - README: three-stage pipeline
This commit is contained in:
@@ -15,6 +15,7 @@ include("spool.jl")
|
||||
include("model.jl") # build_model() + read_features(); shared with bin/train.jl
|
||||
include("classify.jl") # Classifier + load_classifier/classify (needs model.jl)
|
||||
include("metadata.jl") # exiftool extraction + sidecar enrichment (stage 2)
|
||||
include("content.jl") # binary-vs-text triage for unknown files (stage 3)
|
||||
include("worker.jl")
|
||||
|
||||
# Globals the HTTP handlers read at request time. Set once in `run`, before the
|
||||
@@ -23,6 +24,7 @@ include("worker.jl")
|
||||
const CONFIG = Ref{Config}()
|
||||
const QUEUE = Ref{ChannelQueue}() # stage-1 (classification) queue; HTTP intake enqueues here
|
||||
const KNOWN_QUEUE = Ref{ChannelQueue}() # stage-2 (enrichment) queue; stage-1 workers enqueue here
|
||||
const UNKNOWN_QUEUE = Ref{ChannelQueue}() # stage-3 (content triage) queue; stage-1 workers enqueue here
|
||||
const CLASSIFIER = Ref{Classifier}() # loaded once at startup, shared read-only across workers
|
||||
|
||||
include("server.jl") # registers routes (references CONFIG/QUEUE at call time)
|
||||
@@ -70,22 +72,24 @@ function run(; overrides...)
|
||||
cfg = config_from_env(; overrides...)
|
||||
ensure_dirs(cfg)
|
||||
|
||||
# Both pools draw from the same OS threads. Warn on the *combined* size (still
|
||||
# All pools draw from the same OS threads. Warn on the *combined* size (still
|
||||
# allowed): oversubscription just means tasks share threads, not a failure.
|
||||
total_workers = cfg.worker_count + cfg.known_worker_count
|
||||
total_workers = cfg.worker_count + cfg.known_worker_count + cfg.unknown_worker_count
|
||||
if total_workers > Threads.nthreads()
|
||||
@warn "combined worker count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" classify_workers=cfg.worker_count known_workers=cfg.known_worker_count total=total_workers nthreads=Threads.nthreads()
|
||||
@warn "combined worker count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" classify_workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count total=total_workers nthreads=Threads.nthreads()
|
||||
end
|
||||
|
||||
# exiftool is a hard prerequisite for stage-2 enrichment. Fail fast at
|
||||
# startup rather than discover it missing on the first known file.
|
||||
assert_exiftool()
|
||||
|
||||
queue = ChannelQueue(cfg.queue_capacity)
|
||||
known_queue = ChannelQueue(cfg.known_queue_capacity)
|
||||
CONFIG[] = cfg
|
||||
QUEUE[] = queue
|
||||
KNOWN_QUEUE[] = known_queue
|
||||
queue = ChannelQueue(cfg.queue_capacity)
|
||||
known_queue = ChannelQueue(cfg.known_queue_capacity)
|
||||
unknown_queue = ChannelQueue(cfg.unknown_queue_capacity)
|
||||
CONFIG[] = cfg
|
||||
QUEUE[] = queue
|
||||
KNOWN_QUEUE[] = known_queue
|
||||
UNKNOWN_QUEUE[] = unknown_queue
|
||||
|
||||
# Load the classifier before serving. Fail fast: a server that silently
|
||||
# doesn't classify is a worse surprise than a clear startup error.
|
||||
@@ -93,17 +97,20 @@ function run(; overrides...)
|
||||
@info "loaded classifier" path=cfg.model_path
|
||||
|
||||
# Stage-aware recovery: re-drive each stage's leftovers onto its own queue so
|
||||
# files resume where they were, not from scratch. (unknown/ has no consumer
|
||||
# yet, so it isn't recovered — it just accumulates for a future pool.)
|
||||
recovered = recover_dir!(cfg.spool_dir, queue)
|
||||
recovered_known = recover_dir!(cfg.known_dir, known_queue)
|
||||
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity recovered=recovered recovered_known=recovered_known
|
||||
# files resume where they were, not from scratch. spool/ → stage-1,
|
||||
# known/ → stage-2, unknown/ → stage-3.
|
||||
recovered = recover_dir!(cfg.spool_dir, queue)
|
||||
recovered_known = recover_dir!(cfg.known_dir, known_queue)
|
||||
recovered_unknown = recover_dir!(cfg.unknown_dir, unknown_queue)
|
||||
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity recovered=recovered recovered_known=recovered_known recovered_unknown=recovered_unknown
|
||||
|
||||
workers = [Threads.@spawn worker_loop(i, cfg, queue,
|
||||
(job, c, wid) -> handle_classify_job(job, c, wid, known_queue))
|
||||
(job, c, wid) -> handle_classify_job(job, c, wid, known_queue, unknown_queue))
|
||||
for i in 1:cfg.worker_count]
|
||||
known_workers = [Threads.@spawn worker_loop(i, cfg, known_queue, handle_known_job)
|
||||
for i in 1:cfg.known_worker_count]
|
||||
unknown_workers = [Threads.@spawn worker_loop(i, cfg, unknown_queue, handle_unknown_job)
|
||||
for i in 1:cfg.unknown_worker_count]
|
||||
|
||||
register_routes()
|
||||
serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false)
|
||||
@@ -121,10 +128,12 @@ function run(; overrides...)
|
||||
@info "draining queues and stopping workers"
|
||||
terminate() # 1. stop accepting new HTTP requests
|
||||
close!(queue) # 2. no new classify jobs; stage-1 drains buffered
|
||||
foreach(wait, workers) # 3. wait out stage-1 — the ONLY producer of the
|
||||
# known queue — so nothing else will be enqueued
|
||||
close!(known_queue) # 4. now safe to close stage-2's queue
|
||||
foreach(wait, known_workers) # 5. wait out stage-2
|
||||
foreach(wait, workers) # 3. wait out stage-1 — the ONLY producer of BOTH the
|
||||
# known and unknown queues — so nothing else enqueues
|
||||
close!(known_queue) # 4. now safe to close the downstream queues
|
||||
close!(unknown_queue)
|
||||
foreach(wait, known_workers) # 5. wait out stage-2 and stage-3
|
||||
foreach(wait, unknown_workers)
|
||||
@info "shutdown complete"
|
||||
end
|
||||
atexit(drain)
|
||||
|
||||
Reference in New Issue
Block a user