jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add stage-3 content triage: sort unknown files into binary/ and text/

Browse Source 
Unknown files are no longer terminal. Stage 1 now routes :unknown onto a
dedicated queue (with the same blocking backpressure as the known queue),
and a third worker pool sorts each file into data/binary/ or data/text/
using a NUL-byte sniff of the first 8000 bytes.

- content.jl: is_binary content sniff (stage 3)
- worker.jl: handle_unknown_job; stage-1 routes unknown with backpressure;
  KNOWN_ENQUEUE_RETRY_SECONDS -> ROUTE_ENQUEUE_RETRY_SECONDS (serves both)
- config.jl: unknown_worker_count/queue_capacity, binary_dir, text_dir + env
- FileServer.jl: unknown queue, pool, stage-aware recovery, drain ordering
- tests for is_binary and handle_unknown_job; tmp_config isolates new dirs
- README: three-stage pipeline
This commit is contained in:
Jeffrey Ward
2026-07-02 16:54:17 -04:00
parent 2a46f5021a
commit e42e8ef8af
6 changed files with 227 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/config.jl
View File

@@ -12,9 +12,16 @@ Base.@kwdef struct Config
# the two pools are tuned independently.
known_worker_count::Int = Threads.nthreads()
known_queue_capacity::Int = 1000
# Stage 3 (content triage) also gets its own pool + queue: sorting an
# unrecognized file into binary/ vs text/ is cheap I/O, tuned independently
# of the classify and enrich pools.
unknown_worker_count::Int = Threads.nthreads()
unknown_queue_capacity::Int = 1000
spool_dir::String = "data/spool" # files land here on intake (pending classification)
known_dir::String = "data/known" # classified-known, awaiting enrichment (stage 2)
unknown_dir::String = "data/unknown" # classified-unknown, parked for a future pipeline
unknown_dir::String = "data/unknown" # classified-unknown, awaiting content triage (stage 3)
binary_dir::String = "data/binary" # stage-3 sink: unknown files that look like binary data
text_dir::String = "data/text" # stage-3 sink: unknown files that look like text
done_dir::String = "data/done" # fully enriched known files (+ .meta.json sidecars)
failed_dir::String = "data/failed" # files move here if a worker throws
model_path::String = "model/classifier.jld2" # committed classifier artifact, loaded at startup
@@ -31,34 +38,42 @@ and for `FileServer.run(; port=...)`).
Recognised variables:
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
FS_KNOWN_WORKERS, FS_KNOWN_QUEUE_CAPACITY,
FS_SPOOL_DIR, FS_KNOWN_DIR, FS_UNKNOWN_DIR, FS_DONE_DIR, FS_FAILED_DIR,
FS_MODEL_PATH, FS_EXIFTOOL_TIMEOUT
FS_UNKNOWN_WORKERS, FS_UNKNOWN_QUEUE_CAPACITY,
FS_SPOOL_DIR, FS_KNOWN_DIR, FS_UNKNOWN_DIR, FS_BINARY_DIR, FS_TEXT_DIR,
FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH, FS_EXIFTOOL_TIMEOUT
"""
function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
queue_capacity=nothing, known_worker_count=nothing,
known_queue_capacity=nothing, spool_dir=nothing,
known_dir=nothing, unknown_dir=nothing, done_dir=nothing,
known_queue_capacity=nothing, unknown_worker_count=nothing,
unknown_queue_capacity=nothing, spool_dir=nothing,
known_dir=nothing, unknown_dir=nothing, binary_dir=nothing,
text_dir=nothing, done_dir=nothing,
failed_dir=nothing, model_path=nothing, exiftool_timeout=nothing)
Config(
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
worker_count = something(worker_count, parse(Int, get(ENV, "FS_WORKERS", string(Threads.nthreads())))),
queue_capacity = something(queue_capacity, parse(Int, get(ENV, "FS_QUEUE_CAPACITY", "1000"))),
known_worker_count = something(known_worker_count, parse(Int, get(ENV, "FS_KNOWN_WORKERS", string(Threads.nthreads())))),
known_queue_capacity = something(known_queue_capacity, parse(Int, get(ENV, "FS_KNOWN_QUEUE_CAPACITY", "1000"))),
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
known_dir = something(known_dir, get(ENV, "FS_KNOWN_DIR", "data/known")),
unknown_dir = something(unknown_dir, get(ENV, "FS_UNKNOWN_DIR", "data/unknown")),
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
exiftool_timeout = something(exiftool_timeout, parse(Int, get(ENV, "FS_EXIFTOOL_TIMEOUT", "30"))),
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
worker_count = something(worker_count, parse(Int, get(ENV, "FS_WORKERS", string(Threads.nthreads())))),
queue_capacity = something(queue_capacity, parse(Int, get(ENV, "FS_QUEUE_CAPACITY", "1000"))),
known_worker_count = something(known_worker_count, parse(Int, get(ENV, "FS_KNOWN_WORKERS", string(Threads.nthreads())))),
known_queue_capacity = something(known_queue_capacity, parse(Int, get(ENV, "FS_KNOWN_QUEUE_CAPACITY", "1000"))),
unknown_worker_count = something(unknown_worker_count, parse(Int, get(ENV, "FS_UNKNOWN_WORKERS", string(Threads.nthreads())))),
unknown_queue_capacity = something(unknown_queue_capacity, parse(Int, get(ENV, "FS_UNKNOWN_QUEUE_CAPACITY", "1000"))),
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
known_dir = something(known_dir, get(ENV, "FS_KNOWN_DIR", "data/known")),
unknown_dir = something(unknown_dir, get(ENV, "FS_UNKNOWN_DIR", "data/unknown")),
binary_dir = something(binary_dir, get(ENV, "FS_BINARY_DIR", "data/binary")),
text_dir = something(text_dir, get(ENV, "FS_TEXT_DIR", "data/text")),
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
exiftool_timeout = something(exiftool_timeout, parse(Int, get(ENV, "FS_EXIFTOOL_TIMEOUT", "30"))),
)
end
"Create all the pipeline-stage directories if they don't already exist."
function ensure_dirs(cfg::Config)
for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.done_dir, cfg.failed_dir)
for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.binary_dir,
cfg.text_dir, cfg.done_dir, cfg.failed_dir)
mkpath(d)
end
return nothing