Add Lux.jl file classifier (known/unknown) with offline trainer

Each uploaded file is scored by a fixed-structure neural net that labels it
known (resembling the training set) or unknown — novelty detection over the
first 16 + last 16 bytes (scaled to [0,1]), Dense(32->64->16->2), argmax.

- src/model.jl: shared architecture + byte->feature mapping (trainer + server)
- src/classify.jl: load committed artifact, classify a file at inference
- bin/train.jl: offline trainer, 1:1 blended negatives (random + grab-bag),
  seeded 80/20 split, writes model/classifier.jld2
- worker: classify (annotate-only) and log classification=known|unknown
- config: FS_MODEL_PATH; server fails fast if the artifact is missing
- deps: Lux, JLD2, Optimisers, Zygote

src/FileServer.jl

@@ -5,11 +5,15 @@ using UUIDs
 using HTTP
 using JSON3
 using Oxygen
+using Lux
+using JLD2
 
 include("config.jl")
 include("job.jl")
 include("queue.jl")
 include("spool.jl")
+include("model.jl")     # build_model() + read_features(); shared with bin/train.jl
+include("classify.jl")  # Classifier + load_classifier/classify (needs model.jl)
 include("worker.jl")
 
 # Globals the HTTP handlers read at request time. Set once in `run`, before the
@@ -17,6 +21,7 @@ include("worker.jl")
 # `Config`/`ChannelQueue` types exist.
 const CONFIG = Ref{Config}()
 const QUEUE  = Ref{ChannelQueue}()
+const CLASSIFIER = Ref{Classifier}()  # loaded once at startup, shared read-only across workers
 
 include("server.jl")   # registers routes (references CONFIG/QUEUE at call time)
 
@@ -71,6 +76,11 @@ function run(; overrides...)
     CONFIG[] = cfg
     QUEUE[]  = queue
 
+    # Load the classifier before serving. Fail fast: a server that silently
+    # doesn't classify is a worse surprise than a clear startup error.
+    CLASSIFIER[] = load_classifier(cfg.model_path)
+    @info "loaded classifier" path=cfg.model_path
+
     recovered = recover_spool!(cfg, queue)
     @info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count capacity=cfg.queue_capacity recovered=recovered
 

src/classify.jl

@@ -0,0 +1,49 @@
+# Runtime inference side of the classifier: load the committed artifact once at
+# startup and score files. Server-only (bin/train.jl doesn't include this). The
+# architecture and feature extraction live in model.jl, which must be included
+# first.
+
+"""
+    Classifier
+
+An in-memory, ready-to-run classifier: the (stateless) Lux `model` plus the
+learned parameters `ps` and states `st` loaded from the artifact. Immutable and
+shared read-only across all worker threads — Lux inference is a pure function
+over `ps`/`st`, so no locking is needed.
+"""
+struct Classifier{M,P,S}
+    model::M
+    ps::P
+    st::S
+end
+
+"""
+    load_classifier(path) -> Classifier
+
+Rebuild the fixed architecture and load `ps`/`st` from the JLD2 artifact at
+`path`. Throws if the file is missing or unreadable — the server fails fast at
+startup rather than run silently without classification.
+"""
+function load_classifier(path::AbstractString)
+    isfile(path) || error("model artifact not found at $path (run bin/train.jl to build it)")
+    data = JLD2.load(path)
+    (haskey(data, "ps") && haskey(data, "st")) ||
+        error("model artifact at $path is missing ps/st (was it written by bin/train.jl?)")
+    return Classifier(build_model(), data["ps"], data["st"])
+end
+
+"""
+    classify(clf, path) -> Symbol
+
+Return `:known` or `:unknown` for the file at `path`. Files shorter than
+`MIN_FILE_BYTES` short-circuit to `:unknown` without touching the model
+(they can't form a 16+16 feature window). Otherwise: scale bytes, run the net,
+argmax the two logits (class 1 = known, class 2 = unknown).
+"""
+function classify(clf::Classifier, path::AbstractString)
+    feats = read_features(path)
+    feats === nothing && return :unknown
+    x = reshape(feats, FEATURE_DIM, 1)
+    logits, _ = Lux.apply(clf.model, x, clf.ps, clf.st)
+    return argmax(vec(logits)) == 1 ? :known : :unknown
+end

src/config.jl

@@ -10,6 +10,7 @@ Base.@kwdef struct Config
     spool_dir::String   = "data/spool"   # files land here on intake (pending)
     done_dir::String    = "data/done"    # files move here after successful processing
     failed_dir::String  = "data/failed"  # files move here if a worker throws
+    model_path::String  = "model/classifier.jld2"  # committed classifier artifact, loaded at startup
 end
 
 """
@@ -21,11 +22,11 @@ and for `FileServer.run(; port=...)`).
 
 Recognised variables:
   FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
-  FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR
+  FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH
 """
 function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
                           queue_capacity=nothing, spool_dir=nothing,
-                          done_dir=nothing, failed_dir=nothing)
+                          done_dir=nothing, failed_dir=nothing, model_path=nothing)
     Config(
         host           = something(host,           get(ENV, "FS_HOST", "127.0.0.1")),
         port           = something(port,           parse(Int, get(ENV, "FS_PORT", "8080"))),
@@ -34,6 +35,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
         spool_dir      = something(spool_dir,      get(ENV, "FS_SPOOL_DIR", "data/spool")),
         done_dir       = something(done_dir,       get(ENV, "FS_DONE_DIR", "data/done")),
         failed_dir     = something(failed_dir,     get(ENV, "FS_FAILED_DIR", "data/failed")),
+        model_path     = something(model_path,     get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
     )
 end
 

src/model.jl

@@ -0,0 +1,51 @@
+# Shared model definition + feature extraction, used by BOTH the training script
+# (bin/train.jl) and the server (worker inference). Keeping the architecture and
+# the byte->feature mapping in one place guarantees train and inference can never
+# drift out of sync (same input layout, same scaling, same net shape).
+#
+# The including scope is expected to have already done `using Lux` and
+# `using JLD2` (FileServer.jl and bin/train.jl both do), so this file adds no
+# top-level `using` of its own and works whether it's pulled into the FileServer
+# module or a bare script.
+
+"Number of input features: first 16 bytes + last 16 bytes of a file."
+const FEATURE_DIM = 32
+
+"Minimum file size (bytes) the model can accept: needs a disjoint 16+16 window."
+const MIN_FILE_BYTES = 32
+
+"""
+    build_model()
+
+The fixed-structure classifier: 32-dim byte features → two logits
+(class 1 = known, class 2 = unknown). Raw logits out (no softmax layer); the
+loss applies log-softmax and inference just takes an argmax.
+"""
+build_model() = Chain(
+    Dense(FEATURE_DIM => 64, relu),
+    Dense(64 => 16, relu),
+    Dense(16 => 2),
+)
+
+"""
+    read_features(path) -> Union{Vector{Float32}, Nothing}
+
+Read the first 16 and last 16 bytes of the file at `path`, concatenate to a
+32-element vector, and scale each byte from 0–255 into [0,1]. Returns `nothing`
+for files shorter than `MIN_FILE_BYTES` (the caller decides what that means:
+training drops them, the server classifies them as unknown without the model).
+
+Memory stays flat: we read 16 bytes from the front and `seek` to end-16 for the
+tail rather than slurping the whole file.
+"""
+function read_features(path::AbstractString)
+    sz = filesize(path)
+    sz < MIN_FILE_BYTES && return nothing
+    bytes = open(path) do io
+        head = read(io, 16)
+        seek(io, sz - 16)
+        tail = read(io, 16)
+        vcat(head, tail)
+    end
+    return Float32.(bytes) ./ 255f0
+end

src/worker.jl

@@ -10,9 +10,11 @@ For now the "work" is just logging the received filename to prove the flow —
 this is the seam where real heavy-lifting will go later.
 """
 function handle_job(job::Job, cfg::Config, worker_id::Int)
-    # --- placeholder for real heavy-lifting work -------------------------
-    @info "received file" worker=worker_id id=job.id name=job.original_name size=job.size
-    # --------------------------------------------------------------------
+    # Classify the spooled file (annotate-only for now: the result is logged but
+    # every file still moves to done/ regardless of known/unknown). Sub-32-byte
+    # files short-circuit to :unknown inside classify without touching the model.
+    classification = classify(CLASSIFIER[], job.path)
+    @info "received file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification
     dest = move_to(cfg.done_dir, job)
     @info "completed" worker=worker_id id=job.id dest=dest
     return nothing