Renamed to DarkStruct

Dockerfile

@@ -1,12 +1,12 @@
-# Multi-stage build for the FileServer pipeline.
+# Multi-stage build for the DarkStruct pipeline.
 #
 # The runtime image carries only what serving needs: the Julia runtime, a depot
 # that is already instantiated *and* precompiled (so startup is load-only, no
 # compilation), exiftool, and optionally github-linguist. Nothing from the build
 # survives into it: no compilers, no package registry, no gem toolchain.
 #
-#   docker build -t file-server .
+#   docker build -t darkstruct .
-#   docker build --build-arg WITH_LINGUIST=false -t file-server .   # ~200MB smaller
+#   docker build --build-arg WITH_LINGUIST=false -t darkstruct .   # ~200MB smaller
 #
 # github-linguist (stage-4 programming-language detection) is the one heavy
 # optional dependency: it needs a Ruby toolchain to build rugged. It degrades
@@ -28,19 +28,19 @@ ENV JULIA_DEPOT_PATH=/opt/julia-depot \
 WORKDIR /app
 
 # Manifest-first so a source-only edit doesn't re-resolve or rebuild the whole
-# dependency tree. FileServer is its own project's root package, so Pkg insists
+# dependency tree. DarkStruct is its own project's root package, so Pkg insists
 # on a module file being there; a stub satisfies it while the (slow, rarely
 # invalidated) dependency layer is built; the real source lands below.
 COPY Project.toml Manifest.toml ./
 RUN mkdir -p src \
- && printf 'module FileServer\nend\n' > src/FileServer.jl \
+ && printf 'module DarkStruct\nend\n' > src/DarkStruct.jl \
  && julia --project=/app -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
 
-# Precompiling FileServer itself means the runtime image loads cached code
+# Precompiling DarkStruct itself means the runtime image loads cached code
 # instead of compiling Lux/Zygote on first request.
 COPY src/ ./src/
 RUN julia --project=/app -e 'using Pkg; Pkg.precompile()' \
- && julia --project=/app -e 'using FileServer'
+ && julia --project=/app -e 'using DarkStruct'
 
 # Everything the depot keeps for *resolving* packages is dead weight once the
 # Manifest is pinned: the registry, git clones, scratch and log dirs.

Manifest.toml

@@ -314,7 +314,7 @@ weakdeps = ["HTTP"]
     [deps.FileIO.extensions]
     HTTPExt = "HTTP"
 
-[[deps.FileServer]]
+[[deps.DarkStruct]]
 deps = ["HTTP", "JLD2", "JSON3", "Languages", "Logging", "Lux", "Optimisers", "Oxygen", "Random", "UUIDs", "Zygote"]
 path = "."
 uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"

Project.toml

@@ -1,4 +1,4 @@
-name = "FileServer"
+name = "DarkStruct"
 uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
 version = "0.1.0"
 authors = ["wardjm@gmail.com"]

README.md

@@ -1,4 +1,4 @@
-# FileServer
+# DarkStruct
 
 A minimal Julia service that receives files over HTTP and hands them off to a
 pool of worker threads for processing. The HTTP endpoint does no real work: it
@@ -367,11 +367,11 @@ docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up
 # 15672 are published for exactly this)
 docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
 FS_QUEUE_BACKEND=rabbitmq \
-FS_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ \
+FS_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ \
   julia --project=. -t auto bin/server.jl
 ```
 
-Queues are named `<FS_AMQP_PREFIX>.<stage>` — `fileserver.classify`,
+Queues are named `<FS_AMQP_PREFIX>.<stage>` — `darkstruct.classify`,
 `.enrich`, `.triage`, `.language` — so the broker's queue list reads like the
 pipeline, and `/stats` reports each one's depth from a once-a-second poll.
 
@@ -419,7 +419,7 @@ branch) run in the normal suite. The round trip against a real broker is gated:
 
 ```bash
 docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
-FS_TEST_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ \
+FS_TEST_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ \
   julia --project=. -e 'using Pkg; Pkg.test()'
 ```
 
@@ -555,7 +555,7 @@ init, so the artifact is exactly regenerable from the same inputs.
 | `FS_NOMINATED_DIR`        | `data/nominated` | One JSON per self-nominated cluster (human promote gate) |
 | `FS_QUEUE_BACKEND`        | `channel`      | `channel` (in-process) or `rabbitmq` (durable); see "Queue backends" |
 | `FS_AMQP_URL`             | `amqp://guest:guest@localhost:5672/` | Broker connection, credentials included |
-| `FS_AMQP_PREFIX`          | `fileserver`   | Queue names are `<prefix>.<stage>`           |
+| `FS_AMQP_PREFIX`          | `darkstruct`   | Queue names are `<prefix>.<stage>`           |
 | `FS_AMQP_PREFETCH`        | worker count   | Unacked messages the broker hands one stage at a time |
 | `FS_AMQP_CONFIRMS`        | `true`         | Wait for a publisher confirm before a `202` (intake only) |
 | `FS_RECOVER_SPOOL`        | `false`        | Re-drive `spool/` at startup even on the broker backend (use after a purged broker) |
@@ -800,7 +800,7 @@ classifier and 6× the rename, and nearly all of it was `ConsoleLogger`
 Demoting them took stage 1 from 8.5k files/s to 35.3k files/s on a single worker,
 a 4.2× speedup for no algorithmic change. The script still prices a formatted
 line, so the cost of turning them back on is visible: running the handler under
-`JULIA_DEBUG=FileServer` measures 133 µs per file, a 4.7× slowdown. That is the
+`JULIA_DEBUG=DarkStruct` measures 133 µs per file, a 4.7× slowdown. That is the
 trade. Per-file tracing is available when you want it and off by default, with
 `GET /stats` giving per-file observability that is counted, not formatted.
 
@@ -970,7 +970,7 @@ Flags: `--model`, `--reps`, `--trials`, `--batches`, `--sizes`, `--no-threads`,
 
 ```
 src/
-  FileServer.jl   module + run() (startup, recovery, workers, serve, shutdown)
+  DarkStruct.jl   module + run() (startup, recovery, workers, serve, shutdown)
   config.jl       Config struct + env parsing
   job.jl          Job (the queue reference)
   queue.jl        JobQueue seam + in-process ChannelQueue

bin/bench.jl

@@ -1,6 +1,6 @@
 #!/usr/bin/env julia
 #
-# bench.jl: measure end-to-end throughput and server memory for a running FileServer.
+# bench.jl: measure end-to-end throughput and server memory for a running DarkStruct.
 #
 # Two things make this pipeline awkward to benchmark with off-the-shelf tools
 # (ab/hey/wrk), and both shape what this script does:
@@ -143,7 +143,7 @@ end
 #
 # Resolved from the same environment variables src/config.jl reads, so a server
 # started with custom dirs is benchmarked correctly. Kept as a standalone table
-# rather than `using FileServer` so the harness doesn't pay to load Lux.
+# rather than `using DarkStruct` so the harness doesn't pay to load Lux.
 
 sinkdirs() = (
     done      = get(ENV, "FS_DONE_DIR",      "data/done"),

bin/bench_stage1.jl

@@ -25,7 +25,7 @@
 #     guess. This sweep is what demoted stage 1's per-file lines to `@debug`
 #     (see the note in src/worker.jl); the standalone `logging (...)` rows below
 #     still price a *formatted* line, i.e. what those lines cost when switched
-#     back on with `JULIA_DEBUG=FileServer`, while the handler rows show what the
+#     back on with `JULIA_DEBUG=DarkStruct`, while the handler rows show what the
 #     stage pays with them off.
 #   * Concurrency. Components that own a lock (the queue's condition, the
 #     logger's stream) don't scale, and the ranking at one worker need not be the
@@ -48,14 +48,14 @@
 # Reported times are the *minimum* over trials: the floor is the signal and
 # everything above it is scheduler, page-cache and GC noise.
 
-using FileServer
+using DarkStruct
 using Lux
 using JSON3
 using Logging
 using Printf
 using Random
 
-const FS = FileServer
+const FS = DarkStruct
 
 # ---------------------------------------------------------------- option parsing
 
@@ -222,10 +222,10 @@ Run `f` under one of the three loggers the cost of logging is bracketed by:
   * `:format`  `ConsoleLogger` to `devnull`: message formatting and key/value
                interpolation, but no I/O.
   * `:flush`   `FlushLogger(ConsoleLogger(io))` to a real file: what
-               `FileServer.run` installs, under the redirect it was written for.
+               `DarkStruct.run` installs, under the redirect it was written for.
   * `:debug`   the same, at `Debug` level: the stage's per-file lines are
                `@debug`, so this is the equivalent of running the server with
-               `JULIA_DEBUG=FileServer` and the only setting under which they
+               `JULIA_DEBUG=DarkStruct` and the only setting under which they
                are emitted at all.
 """
 function with_logger_named(f, which::Symbol, path::AbstractString)

bin/bench_stage2.jl

@@ -65,13 +65,13 @@
 # Reported times are the *minimum* over trials: the floor is the signal and
 # everything above it is scheduler, page-cache and GC noise.
 
-using FileServer
+using DarkStruct
 using JSON3
 using Logging
 using Printf
 using Random
 
-const FS = FileServer
+const FS = DarkStruct
 
 # ---------------------------------------------------------------- option parsing
 
@@ -245,7 +245,7 @@ Run `f` under one of the loggers the cost of logging is bracketed by:
   * `:format`  `ConsoleLogger` to `devnull`: message formatting and key/value
                interpolation, but no I/O.
   * `:flush`   `FlushLogger(ConsoleLogger(io))` to a real file: what
-               `FileServer.run` installs, under the redirect it was written
+               `DarkStruct.run` installs, under the redirect it was written
                for. Stage 2's per-file line is `@info`, not `@debug`, so this
                row is what the deployed server actually pays.
 """

bin/cluster_sweep.jl

@@ -14,6 +14,6 @@
 # auto-promotes to a compaction pass to seed the catalog. Configure via the
 # FS_CLUSTER_* / FS_NOMINATED_DIR env vars (see src/config.jl).
 
-using FileServer
+using DarkStruct
 
-FileServer.cluster_sweep_cli(ARGS)
+DarkStruct.cluster_sweep_cli(ARGS)

bin/send_dir.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# send_dir.sh — upload every file in a directory to a running FileServer.
+# send_dir.sh — upload every file in a directory to a running DarkStruct.
 #
 # Each file is POSTed to /upload as multipart/form-data. By default one file per
 # request (so you get per-file feedback); use -b to batch all files into a

bin/server.jl

@@ -1,4 +1,4 @@
 #!/usr/bin/env julia
 # Entry point. Run with:  julia --project -t auto bin/server.jl
-using FileServer
+using DarkStruct
-FileServer.run()
+DarkStruct.run()

docker-compose.rabbitmq.yml

@@ -18,8 +18,8 @@ services:
       # The built-in guest/guest only authenticates over loopback, so it cannot
       # work across the compose network. Override these for anything real; they
       # are here so the stack comes up with one command, not as a credential.
-      RABBITMQ_DEFAULT_USER: "fileserver"
+      RABBITMQ_DEFAULT_USER: "darkstruct"
-      RABBITMQ_DEFAULT_PASS: "fileserver"
+      RABBITMQ_DEFAULT_PASS: "darkstruct"
     ports:
       # 5672 is published so you can run the server on the host
       # (`julia --project=. -t auto bin/server.jl`) against this broker; 15672 is
@@ -40,10 +40,10 @@ services:
       start_period: 30s
     restart: unless-stopped
 
-  file-server:
+  darkstruct:
     environment:
       FS_QUEUE_BACKEND: "rabbitmq"
-      FS_AMQP_URL: "amqp://fileserver:fileserver@rabbitmq:5672/"
+      FS_AMQP_URL: "amqp://darkstruct:darkstruct@rabbitmq:5672/"
     depends_on:
       rabbitmq:
         # The server retries a broker that isn't up yet (AMQP_CONNECT_RETRY_SECONDS),

docker-compose.yml

@@ -1,5 +1,5 @@
 services:
-  file-server:
+  darkstruct:
     build:
       context: .
       args:
@@ -7,7 +7,7 @@ services:
         # Stage-4 text files then get natural-language enrichment but no
         # programming language; the server warns at startup and carries on.
         WITH_LINGUIST: "true"
-    image: file-server:latest
+    image: darkstruct:latest
     ports:
       - "8080:8080"
     volumes:

src/DarkStruct.jl

@@ -1,4 +1,4 @@
-module FileServer
+module DarkStruct
 
 using Logging
 using Random
@@ -67,7 +67,7 @@ end
 """
     run(; overrides...)
 
-Start the file server: build config, ensure directories, recover any leftover
+Start DarkStruct: build config, ensure directories, recover any leftover
 spooled files, spawn the worker pool, then serve HTTP until interrupted
 (Ctrl-C / SIGINT or SIGTERM). On shutdown it stops accepting uploads, drains the
 queue, waits for workers to finish in-flight files, and exits cleanly.
@@ -134,7 +134,7 @@ function run(; overrides...)
     DETECTOR[] = LanguageDetector()
     @info "loaded language detector"
 
-    @info "starting file-server" backend=cfg.queue_backend host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity
+    @info "starting darkstruct" backend=cfg.queue_backend host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity
 
     # Zero the counters here, not at module load: `since` should mean "serving
     # since", so a scrape's totals cover the run, not the minutes spent loading

src/config.jl

@@ -29,7 +29,7 @@ Base.@kwdef struct Config
     # advisory one against a polled depth on the broker (see `RabbitQueue`).
     queue_backend::Symbol = :channel
     amqp_url::String      = "amqp://guest:guest@localhost:5672/"
-    amqp_prefix::String   = "fileserver"   # queue names are "<prefix>.<stage>"
+    amqp_prefix::String   = "darkstruct"   # queue names are "<prefix>.<stage>"
     # Unacked messages the broker will hand one stage at a time. Defaults to that
     # stage's worker count (0 below means "use it"), so a worker holds at most the
     # one it is working on and a restart redelivers the minimum.
@@ -108,7 +108,7 @@ end
 
 Build a `Config` from environment variables, falling back to the struct
 defaults. Any keyword `overrides` win over the environment (useful for tests
-and for `FileServer.run(; port=...)`).
+and for `DarkStruct.run(; port=...)`).
 
 Recognised variables:
   FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
@@ -171,7 +171,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
         nominated_dir          = something(nominated_dir,          get(ENV, "FS_NOMINATED_DIR", "data/nominated")),
         queue_backend          = something(queue_backend,          parse_backend(get(ENV, "FS_QUEUE_BACKEND", "channel"))),
         amqp_url               = something(amqp_url,               get(ENV, "FS_AMQP_URL", "amqp://guest:guest@localhost:5672/")),
-        amqp_prefix            = something(amqp_prefix,            get(ENV, "FS_AMQP_PREFIX", "fileserver")),
+        amqp_prefix            = something(amqp_prefix,            get(ENV, "FS_AMQP_PREFIX", "darkstruct")),
         amqp_prefetch          = something(amqp_prefetch,          parse(Int, get(ENV, "FS_AMQP_PREFETCH", "0"))),
         amqp_confirms          = something(amqp_confirms,          parse_bool(get(ENV, "FS_AMQP_CONFIRMS", "true"))),
         recover_spool          = something(recover_spool,          parse_bool(get(ENV, "FS_RECOVER_SPOOL", "false"))),

src/model.jl

@@ -4,8 +4,8 @@
 # drift out of sync (same input layout, same scaling, same net shape).
 #
 # The including scope is expected to have already done `using Lux` and
-# `using JLD2` (FileServer.jl and bin/train.jl both do), so this file adds no
+# `using JLD2` (DarkStruct.jl and bin/train.jl both do), so this file adds no
-# top-level `using` of its own and works whether it's pulled into the FileServer
+# top-level `using` of its own and works whether it's pulled into the DarkStruct
 # module or a bare script.
 
 "Number of input features: first 16 bytes + last 16 bytes of a file."

src/worker.jl

@@ -38,7 +38,7 @@
 # the 11.7 µs the whole handler takes with them switched off: six times the rest
 # of the stage put together. Nearly all of it is `ConsoleLogger` formatting
 # (64.2 µs); the FlushLogger's per-message flush is only ~7 µs on top. Switching
-# them back on with `JULIA_DEBUG=FileServer` takes the handler to 106.3 µs, i.e.
+# them back on with `JULIA_DEBUG=DarkStruct` takes the handler to 106.3 µs, i.e.
 # from 85.4k files/s down to 9.4k on one worker.
 #
 # With logging off the stage is the classifier and nothing else: classify 10.30 µs
@@ -50,7 +50,7 @@
 #
 # `@debug` is compiled to a min-level check that doesn't evaluate its arguments,
 # so a disabled line costs ~0.15 µs rather than ~36 µs. The messages are still
-# there when wanted: run with `JULIA_DEBUG=FileServer` to get them back. Errors,
+# there when wanted: run with `JULIA_DEBUG=DarkStruct` to get them back. Errors,
 # quarantines and lifecycle events stay at `@error`/`@info`, since they are rare and
 # their cost doesn't scale with throughput. `GET /stats` (src/stats.jl) is the
 # per-file observability that survives, and it is counted, not formatted.

test/runtests.jl

@@ -1,11 +1,11 @@
 using Test
-using FileServer
+using DarkStruct
 using JSON3
 
 # Pull internals into scope. These aren't exported (only `run` is), but the
 # whole risk profile of this pipeline lives in these functions, so we test them
 # directly rather than only through the HTTP surface.
-using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
+using DarkStruct: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
                   sanitize_filename, recover_dir!, normalize_metadata,
                   build_metadata, finalize_known!, run_exiftool,
                   is_binary, handle_unknown_job, handle_classify_job, worker_loop,
@@ -83,11 +83,11 @@ function tmp_config(root; kwargs...)
         nominated_dir = joinpath(root, "nominated"),
         kwargs...,
     )
-    FileServer.ensure_dirs(cfg)
+    DarkStruct.ensure_dirs(cfg)
     return cfg
 end
 
-@testset "FileServer" begin
+@testset "DarkStruct" begin
 
     @testset "sanitize_filename" begin
         @test sanitize_filename("report.pdf") == "report.pdf"
@@ -101,7 +101,7 @@ end
         @test sanitize_filename("a b&c*.d") == "a_b_c_.d"
         @test sanitize_filename("") == "unnamed"
         # Length is capped.
-        @test Base.length(sanitize_filename("a"^500)) == FileServer.MAX_NAME_LEN
+        @test Base.length(sanitize_filename("a"^500)) == DarkStruct.MAX_NAME_LEN
     end
 
     @testset "multipart_boundary: extraction from Content-Type" begin
@@ -297,9 +297,9 @@ end
         # process that exits instantly, on the hot path of stages 2 and 4. The
         # bound here is deliberately loose (a loaded CI box is slow) but far
         # under the 100 ms floor the polling version could not beat.
-        FileServer.run_with_timeout(`true`, 30)                       # warm up / compile
+        DarkStruct.run_with_timeout(`true`, 30)                       # warm up / compile
         t0 = time()
-        out = FileServer.run_with_timeout(`echo hi`, 30)
+        out = DarkStruct.run_with_timeout(`echo hi`, 30)
         elapsed = time() - t0
         @test out !== nothing
         @test strip(String(out)) == "hi"
@@ -308,7 +308,7 @@ end
 
     @testset "run_with_timeout: kills an overrunning child and reports failure" begin
         t0 = time()
-        out = FileServer.run_with_timeout(`sleep 30`, 1)
+        out = DarkStruct.run_with_timeout(`sleep 30`, 1)
         elapsed = time() - t0
         @test out === nothing            # timed out → no output, caller degrades
         @test elapsed < 5                # killed near the timeout, not after 30 s
@@ -319,11 +319,11 @@ end
         # block on wait(proc) forever and the timeout would be unenforceable.
         cmd = `sh -c "trap '' TERM; sleep 30"`
         t0 = time()
-        out = FileServer.run_with_timeout(cmd, 1)
+        out = DarkStruct.run_with_timeout(cmd, 1)
         elapsed = time() - t0
         @test out === nothing
         # 1 s timeout + up to KILL_GRACE_SECONDS before SIGKILL lands.
-        @test elapsed < 1 + FileServer.KILL_GRACE_SECONDS + 3
+        @test elapsed < 1 + DarkStruct.KILL_GRACE_SECONDS + 3
     end
 
     @testset "run_exiftool: real extraction on a PNG" begin
@@ -403,7 +403,7 @@ end
             # binary: pad to one byte short of the window, then a 2-byte 'é'
             # (0xc3 0xa9) so only its lead byte lands inside the window.
             split = joinpath(root, "split.txt")
-            write(split, vcat(fill(UInt8('a'), FileServer.CONTENT_SNIFF_BYTES - 1),
+            write(split, vcat(fill(UInt8('a'), DarkStruct.CONTENT_SNIFF_BYTES - 1),
                               UInt8[0xc3, 0xa9]))
             @test is_binary(split) == false
 
@@ -414,7 +414,7 @@ end
 
             # Binary garbage past the sniff window is not seen → still text.
             far = joinpath(root, "far.txt")
-            write(far, vcat(fill(UInt8('a'), FileServer.CONTENT_SNIFF_BYTES), UInt8[0x00]))
+            write(far, vcat(fill(UInt8('a'), DarkStruct.CONTENT_SNIFF_BYTES), UInt8[0x00]))
             @test is_binary(far) == false
         end
     end
@@ -503,10 +503,10 @@ end
             # Reads at most LANG_SAMPLE_BYTES, and doesn't choke on a multi-byte
             # char straddling that boundary (trailing 'é' half-in the window).
             big = joinpath(root, "big.txt")
-            write(big, vcat(fill(UInt8('a'), FileServer.LANG_SAMPLE_BYTES - 1),
+            write(big, vcat(fill(UInt8('a'), DarkStruct.LANG_SAMPLE_BYTES - 1),
                             UInt8[0xc3, 0xa9]))              # 'é' split by the edge
             s = read_text_sample(big)
-            @test Base.length(s) == FileServer.LANG_SAMPLE_BYTES - 1   # trailing half-char trimmed
+            @test Base.length(s) == DarkStruct.LANG_SAMPLE_BYTES - 1   # trailing half-char trimmed
             @test all(==('a'), s)
         end
     end
@@ -1240,7 +1240,7 @@ end
 
         # The rest needs a live broker. Point FS_TEST_AMQP_URL at one to run it:
         #   docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
-        #   FS_TEST_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ julia --project=. -e 'using Pkg; Pkg.test()'
+        #   FS_TEST_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ julia --project=. -e 'using Pkg; Pkg.test()'
         amqp_url = get(ENV, "FS_TEST_AMQP_URL", "")
         if isempty(amqp_url)
             @info "skipping RabbitMQ integration tests (set FS_TEST_AMQP_URL to run them)"