Add Lux.jl file classifier (known/unknown) with offline trainer
Each uploaded file is scored by a fixed-structure neural net that labels it known (resembling the training set) or unknown — novelty detection over the first 16 + last 16 bytes (scaled to [0,1]), Dense(32->64->16->2), argmax. - src/model.jl: shared architecture + byte->feature mapping (trainer + server) - src/classify.jl: load committed artifact, classify a file at inference - bin/train.jl: offline trainer, 1:1 blended negatives (random + grab-bag), seeded 80/20 split, writes model/classifier.jld2 - worker: classify (annotate-only) and log classification=known|unknown - config: FS_MODEL_PATH; server fails fast if the artifact is missing - deps: Lux, JLD2, Optimisers, Zygote
This commit is contained in:
@@ -10,6 +10,7 @@ Base.@kwdef struct Config
|
||||
spool_dir::String = "data/spool" # files land here on intake (pending)
|
||||
done_dir::String = "data/done" # files move here after successful processing
|
||||
failed_dir::String = "data/failed" # files move here if a worker throws
|
||||
model_path::String = "model/classifier.jld2" # committed classifier artifact, loaded at startup
|
||||
end
|
||||
|
||||
"""
|
||||
@@ -21,11 +22,11 @@ and for `FileServer.run(; port=...)`).
|
||||
|
||||
Recognised variables:
|
||||
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
|
||||
FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR
|
||||
FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH
|
||||
"""
|
||||
function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
|
||||
queue_capacity=nothing, spool_dir=nothing,
|
||||
done_dir=nothing, failed_dir=nothing)
|
||||
done_dir=nothing, failed_dir=nothing, model_path=nothing)
|
||||
Config(
|
||||
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
|
||||
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
|
||||
@@ -34,6 +35,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
|
||||
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
|
||||
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
|
||||
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
|
||||
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user