Add stage-2 metadata enrichment pipeline for known files

Known-classified files now flow to a second queue with its own worker pool
that extracts metadata via exiftool and writes a normalized JSON sidecar
next to the file in done/, leaving the original bytes untouched.

- Two-stage pipeline: spool/ → classify → known/ → enrich → done/;
  unknowns park in unknown/ as a seam for a future pool
- src/metadata.jl: exiftool -json -G -n with timeout, normalized schema
  (file_type, mime_type, author, created_by, dimensions, ...) + raw dump;
  degraded sidecar on extraction failure rather than quarantine
- Sidecar-first commit so a file in done/ always has its sidecar
- Parametrized worker_loop with classify/enrich handlers; blocking
  backpressure on a full known queue (never drop a classified file)
- Stage-aware recovery: spool/ and known/ resume at their correct stage
- Ordered drain: close stage-1 and wait its workers (the known queue's
  only producer) before closing the known queue
- exiftool required at startup (fail-fast); new FS_KNOWN_*/FS_UNKNOWN_DIR/
  FS_EXIFTOOL_TIMEOUT config knobs; combined-pool thread warning

src/FileServer.jl

@@ -14,13 +14,15 @@ include("queue.jl")
 include("spool.jl")
 include("model.jl")     # build_model() + read_features(); shared with bin/train.jl
 include("classify.jl")  # Classifier + load_classifier/classify (needs model.jl)
+include("metadata.jl")  # exiftool extraction + sidecar enrichment (stage 2)
 include("worker.jl")
 
 # Globals the HTTP handlers read at request time. Set once in `run`, before the
 # server starts accepting connections. Declared after the includes above so the
 # `Config`/`ChannelQueue` types exist.
 const CONFIG = Ref{Config}()
-const QUEUE  = Ref{ChannelQueue}()
+const QUEUE  = Ref{ChannelQueue}()          # stage-1 (classification) queue; HTTP intake enqueues here
+const KNOWN_QUEUE = Ref{ChannelQueue}()     # stage-2 (enrichment) queue; stage-1 workers enqueue here
 const CLASSIFIER = Ref{Classifier}()  # loaded once at startup, shared read-only across workers
 
 include("server.jl")   # registers routes (references CONFIG/QUEUE at call time)
@@ -68,23 +70,40 @@ function run(; overrides...)
     cfg = config_from_env(; overrides...)
     ensure_dirs(cfg)
 
-    if cfg.worker_count > Threads.nthreads()
-        @warn "worker_count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" worker_count=cfg.worker_count nthreads=Threads.nthreads()
+    # Both pools draw from the same OS threads. Warn on the *combined* size (still
+    # allowed): oversubscription just means tasks share threads, not a failure.
+    total_workers = cfg.worker_count + cfg.known_worker_count
+    if total_workers > Threads.nthreads()
+        @warn "combined worker count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" classify_workers=cfg.worker_count known_workers=cfg.known_worker_count total=total_workers nthreads=Threads.nthreads()
     end
 
-    queue = ChannelQueue(cfg.queue_capacity)
-    CONFIG[] = cfg
-    QUEUE[]  = queue
+    # exiftool is a hard prerequisite for stage-2 enrichment. Fail fast at
+    # startup rather than discover it missing on the first known file.
+    assert_exiftool()
+
+    queue       = ChannelQueue(cfg.queue_capacity)
+    known_queue = ChannelQueue(cfg.known_queue_capacity)
+    CONFIG[]      = cfg
+    QUEUE[]       = queue
+    KNOWN_QUEUE[] = known_queue
 
     # Load the classifier before serving. Fail fast: a server that silently
     # doesn't classify is a worse surprise than a clear startup error.
     CLASSIFIER[] = load_classifier(cfg.model_path)
     @info "loaded classifier" path=cfg.model_path
 
-    recovered = recover_spool!(cfg, queue)
-    @info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count capacity=cfg.queue_capacity recovered=recovered
+    # Stage-aware recovery: re-drive each stage's leftovers onto its own queue so
+    # files resume where they were, not from scratch. (unknown/ has no consumer
+    # yet, so it isn't recovered — it just accumulates for a future pool.)
+    recovered       = recover_dir!(cfg.spool_dir, queue)
+    recovered_known = recover_dir!(cfg.known_dir, known_queue)
+    @info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity recovered=recovered recovered_known=recovered_known
 
-    workers = [Threads.@spawn worker_loop(i, cfg, queue) for i in 1:cfg.worker_count]
+    workers = [Threads.@spawn worker_loop(i, cfg, queue,
+                   (job, c, wid) -> handle_classify_job(job, c, wid, known_queue))
+               for i in 1:cfg.worker_count]
+    known_workers = [Threads.@spawn worker_loop(i, cfg, known_queue, handle_known_job)
+                     for i in 1:cfg.known_worker_count]
 
     register_routes()
     serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false)
@@ -99,10 +118,13 @@ function run(; overrides...)
     drained = Threads.Atomic{Bool}(false)
     function drain()
         Threads.atomic_xchg!(drained, true) && return   # run at most once
-        @info "draining queue and stopping workers"
-        terminate()               # stop accepting new HTTP requests
-        close!(queue)             # let workers drain buffered jobs, then exit
-        foreach(wait, workers)
+        @info "draining queues and stopping workers"
+        terminate()                    # 1. stop accepting new HTTP requests
+        close!(queue)                  # 2. no new classify jobs; stage-1 drains buffered
+        foreach(wait, workers)         # 3. wait out stage-1 — the ONLY producer of the
+                                       #    known queue — so nothing else will be enqueued
+        close!(known_queue)            # 4. now safe to close stage-2's queue
+        foreach(wait, known_workers)   # 5. wait out stage-2
         @info "shutdown complete"
     end
     atexit(drain)