Jeffrey Ward
e42e8ef8af
Unknown files are no longer terminal. Stage 1 now routes :unknown onto a dedicated queue (with the same blocking backpressure as the known queue), and a third worker pool sorts each file into data/binary/ or data/text/ using a NUL-byte sniff of the first 8000 bytes. - content.jl: is_binary content sniff (stage 3) - worker.jl: handle_unknown_job; stage-1 routes unknown with backpressure; KNOWN_ENQUEUE_RETRY_SECONDS -> ROUTE_ENQUEUE_RETRY_SECONDS (serves both) - config.jl: unknown_worker_count/queue_capacity, binary_dir, text_dir + env - FileServer.jl: unknown queue, pool, stage-aware recovery, drain ordering - tests for is_binary and handle_unknown_job; tmp_config isolates new dirs - README: three-stage pipeline
81 lines
5.1 KiB
Julia
81 lines
5.1 KiB
Julia
# Runtime configuration. Built once at startup, then treated as immutable.
|
|
# Every knob has a default so the service runs with zero configuration, and
|
|
# every knob can be overridden by an environment variable (see `config_from_env`).
|
|
|
|
Base.@kwdef struct Config
|
|
host::String = "127.0.0.1"
|
|
port::Int = 8080
|
|
worker_count::Int = Threads.nthreads()
|
|
queue_capacity::Int = 1000
|
|
# Stage 2 (enrichment) has its own pool + queue: exiftool work is process-spawn
|
|
# and I/O bound, a different cost profile than the CPU-bound Lux classify, so
|
|
# the two pools are tuned independently.
|
|
known_worker_count::Int = Threads.nthreads()
|
|
known_queue_capacity::Int = 1000
|
|
# Stage 3 (content triage) also gets its own pool + queue: sorting an
|
|
# unrecognized file into binary/ vs text/ is cheap I/O, tuned independently
|
|
# of the classify and enrich pools.
|
|
unknown_worker_count::Int = Threads.nthreads()
|
|
unknown_queue_capacity::Int = 1000
|
|
spool_dir::String = "data/spool" # files land here on intake (pending classification)
|
|
known_dir::String = "data/known" # classified-known, awaiting enrichment (stage 2)
|
|
unknown_dir::String = "data/unknown" # classified-unknown, awaiting content triage (stage 3)
|
|
binary_dir::String = "data/binary" # stage-3 sink: unknown files that look like binary data
|
|
text_dir::String = "data/text" # stage-3 sink: unknown files that look like text
|
|
done_dir::String = "data/done" # fully enriched known files (+ .meta.json sidecars)
|
|
failed_dir::String = "data/failed" # files move here if a worker throws
|
|
model_path::String = "model/classifier.jld2" # committed classifier artifact, loaded at startup
|
|
exiftool_timeout::Int = 30 # seconds before a stuck exiftool is killed → degraded sidecar
|
|
end
|
|
|
|
"""
|
|
config_from_env(; overrides...)
|
|
|
|
Build a `Config` from environment variables, falling back to the struct
|
|
defaults. Any keyword `overrides` win over the environment (useful for tests
|
|
and for `FileServer.run(; port=...)`).
|
|
|
|
Recognised variables:
|
|
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
|
|
FS_KNOWN_WORKERS, FS_KNOWN_QUEUE_CAPACITY,
|
|
FS_UNKNOWN_WORKERS, FS_UNKNOWN_QUEUE_CAPACITY,
|
|
FS_SPOOL_DIR, FS_KNOWN_DIR, FS_UNKNOWN_DIR, FS_BINARY_DIR, FS_TEXT_DIR,
|
|
FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH, FS_EXIFTOOL_TIMEOUT
|
|
"""
|
|
function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
|
|
queue_capacity=nothing, known_worker_count=nothing,
|
|
known_queue_capacity=nothing, unknown_worker_count=nothing,
|
|
unknown_queue_capacity=nothing, spool_dir=nothing,
|
|
known_dir=nothing, unknown_dir=nothing, binary_dir=nothing,
|
|
text_dir=nothing, done_dir=nothing,
|
|
failed_dir=nothing, model_path=nothing, exiftool_timeout=nothing)
|
|
Config(
|
|
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
|
|
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
|
|
worker_count = something(worker_count, parse(Int, get(ENV, "FS_WORKERS", string(Threads.nthreads())))),
|
|
queue_capacity = something(queue_capacity, parse(Int, get(ENV, "FS_QUEUE_CAPACITY", "1000"))),
|
|
known_worker_count = something(known_worker_count, parse(Int, get(ENV, "FS_KNOWN_WORKERS", string(Threads.nthreads())))),
|
|
known_queue_capacity = something(known_queue_capacity, parse(Int, get(ENV, "FS_KNOWN_QUEUE_CAPACITY", "1000"))),
|
|
unknown_worker_count = something(unknown_worker_count, parse(Int, get(ENV, "FS_UNKNOWN_WORKERS", string(Threads.nthreads())))),
|
|
unknown_queue_capacity = something(unknown_queue_capacity, parse(Int, get(ENV, "FS_UNKNOWN_QUEUE_CAPACITY", "1000"))),
|
|
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
|
|
known_dir = something(known_dir, get(ENV, "FS_KNOWN_DIR", "data/known")),
|
|
unknown_dir = something(unknown_dir, get(ENV, "FS_UNKNOWN_DIR", "data/unknown")),
|
|
binary_dir = something(binary_dir, get(ENV, "FS_BINARY_DIR", "data/binary")),
|
|
text_dir = something(text_dir, get(ENV, "FS_TEXT_DIR", "data/text")),
|
|
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
|
|
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
|
|
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
|
|
exiftool_timeout = something(exiftool_timeout, parse(Int, get(ENV, "FS_EXIFTOOL_TIMEOUT", "30"))),
|
|
)
|
|
end
|
|
|
|
"Create all the pipeline-stage directories if they don't already exist."
|
|
function ensure_dirs(cfg::Config)
|
|
for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.binary_dir,
|
|
cfg.text_dir, cfg.done_dir, cfg.failed_dir)
|
|
mkpath(d)
|
|
end
|
|
return nothing
|
|
end
|