Add Lux.jl file classifier (known/unknown) with offline trainer

Each uploaded file is scored by a fixed-structure neural net that labels it
known (resembling the training set) or unknown — novelty detection over the
first 16 + last 16 bytes (scaled to [0,1]), Dense(32->64->16->2), argmax.

- src/model.jl: shared architecture + byte->feature mapping (trainer + server)
- src/classify.jl: load committed artifact, classify a file at inference
- bin/train.jl: offline trainer, 1:1 blended negatives (random + grab-bag),
  seeded 80/20 split, writes model/classifier.jld2
- worker: classify (annotate-only) and log classification=known|unknown
- config: FS_MODEL_PATH; server fails fast if the artifact is missing
- deps: Lux, JLD2, Optimisers, Zygote
This commit is contained in:
2026-07-02 14:13:57 -04:00
parent 6d685cfcbb
commit e55129e3a4
10 changed files with 1145 additions and 12 deletions

View File

@@ -10,6 +10,7 @@ Base.@kwdef struct Config
spool_dir::String = "data/spool" # files land here on intake (pending)
done_dir::String = "data/done" # files move here after successful processing
failed_dir::String = "data/failed" # files move here if a worker throws
model_path::String = "model/classifier.jld2" # committed classifier artifact, loaded at startup
end
"""
@@ -21,11 +22,11 @@ and for `FileServer.run(; port=...)`).
Recognised variables:
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR
FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH
"""
function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
queue_capacity=nothing, spool_dir=nothing,
done_dir=nothing, failed_dir=nothing)
done_dir=nothing, failed_dir=nothing, model_path=nothing)
Config(
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
@@ -34,6 +35,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
)
end