jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Renamed to DarkStruct

Browse Source
This commit is contained in:
Jeffrey Ward
2026-08-26 10:11:29 -04:00
parent 630bf5f9a7
commit 4aed713939
17 changed files with 62 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
Dockerfile
View File

@@ -1,12 +1,12 @@
# Multi-stage build for the FileServer pipeline.
# Multi-stage build for the DarkStruct pipeline.
#
# The runtime image carries only what serving needs: the Julia runtime, a depot
# that is already instantiated *and* precompiled (so startup is load-only, no
# compilation), exiftool, and optionally github-linguist. Nothing from the build
# survives into it: no compilers, no package registry, no gem toolchain.
#
# docker build -t file-server .
# docker build --build-arg WITH_LINGUIST=false -t file-server . # ~200MB smaller
# docker build -t darkstruct .
# docker build --build-arg WITH_LINGUIST=false -t darkstruct . # ~200MB smaller
#
# github-linguist (stage-4 programming-language detection) is the one heavy
# optional dependency: it needs a Ruby toolchain to build rugged. It degrades
@@ -28,19 +28,19 @@ ENV JULIA_DEPOT_PATH=/opt/julia-depot \
WORKDIR /app
# Manifest-first so a source-only edit doesn't re-resolve or rebuild the whole
# dependency tree. FileServer is its own project's root package, so Pkg insists
# dependency tree. DarkStruct is its own project's root package, so Pkg insists
# on a module file being there; a stub satisfies it while the (slow, rarely
# invalidated) dependency layer is built; the real source lands below.
COPY Project.toml Manifest.toml ./
RUN mkdir -p src \
&& printf 'module FileServer\nend\n' > src/FileServer.jl \
&& printf 'module DarkStruct\nend\n' > src/DarkStruct.jl \
&& julia --project=/app -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
# Precompiling FileServer itself means the runtime image loads cached code
# Precompiling DarkStruct itself means the runtime image loads cached code
# instead of compiling Lux/Zygote on first request.
COPY src/ ./src/
RUN julia --project=/app -e 'using Pkg; Pkg.precompile()' \
&& julia --project=/app -e 'using FileServer'
&& julia --project=/app -e 'using DarkStruct'
# Everything the depot keeps for *resolving* packages is dead weight once the
# Manifest is pinned: the registry, git clones, scratch and log dirs.

2
Manifest.toml
View File

@@ -314,7 +314,7 @@ weakdeps = ["HTTP"]
[deps.FileIO.extensions]
HTTPExt = "HTTP"
[[deps.FileServer]]
[[deps.DarkStruct]]
deps = ["HTTP", "JLD2", "JSON3", "Languages", "Logging", "Lux", "Optimisers", "Oxygen", "Random", "UUIDs", "Zygote"]
path = "."
uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"

2
Project.toml
View File

@@ -1,4 +1,4 @@
name = "FileServer"
name = "DarkStruct"
uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
version = "0.1.0"
authors = ["wardjm@gmail.com"]

14
README.md
View File

@@ -1,4 +1,4 @@
# FileServer
# DarkStruct
A minimal Julia service that receives files over HTTP and hands them off to a
pool of worker threads for processing. The HTTP endpoint does no real work: it
@@ -367,11 +367,11 @@ docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up
# 15672 are published for exactly this)
docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
FS_QUEUE_BACKEND=rabbitmq \
FS_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ \
FS_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ \
julia --project=. -t auto bin/server.jl
```
Queues are named `<FS_AMQP_PREFIX>.<stage>` — `fileserver.classify`,
Queues are named `<FS_AMQP_PREFIX>.<stage>` — `darkstruct.classify`,
`.enrich`, `.triage`, `.language` — so the broker's queue list reads like the
pipeline, and `/stats` reports each one's depth from a once-a-second poll.
@@ -419,7 +419,7 @@ branch) run in the normal suite. The round trip against a real broker is gated:
```bash
docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
FS_TEST_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ \
FS_TEST_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ \
julia --project=. -e 'using Pkg; Pkg.test()'
```
@@ -555,7 +555,7 @@ init, so the artifact is exactly regenerable from the same inputs.
| `FS_NOMINATED_DIR` | `data/nominated` | One JSON per self-nominated cluster (human promote gate) |
| `FS_QUEUE_BACKEND` | `channel` | `channel` (in-process) or `rabbitmq` (durable); see "Queue backends" |
| `FS_AMQP_URL` | `amqp://guest:guest@localhost:5672/` | Broker connection, credentials included |
| `FS_AMQP_PREFIX` | `fileserver` | Queue names are `<prefix>.<stage>` |
| `FS_AMQP_PREFIX` | `darkstruct` | Queue names are `<prefix>.<stage>` |
| `FS_AMQP_PREFETCH` | worker count | Unacked messages the broker hands one stage at a time |
| `FS_AMQP_CONFIRMS` | `true` | Wait for a publisher confirm before a `202` (intake only) |
| `FS_RECOVER_SPOOL` | `false` | Re-drive `spool/` at startup even on the broker backend (use after a purged broker) |
@@ -800,7 +800,7 @@ classifier and 6× the rename, and nearly all of it was `ConsoleLogger`
Demoting them took stage 1 from 8.5k files/s to 35.3k files/s on a single worker,
a 4.2× speedup for no algorithmic change. The script still prices a formatted
line, so the cost of turning them back on is visible: running the handler under
`JULIA_DEBUG=FileServer` measures 133 µs per file, a 4.7× slowdown. That is the
`JULIA_DEBUG=DarkStruct` measures 133 µs per file, a 4.7× slowdown. That is the
trade. Per-file tracing is available when you want it and off by default, with
`GET /stats` giving per-file observability that is counted, not formatted.
@@ -970,7 +970,7 @@ Flags: `--model`, `--reps`, `--trials`, `--batches`, `--sizes`, `--no-threads`,
```
src/
FileServer.jl module + run() (startup, recovery, workers, serve, shutdown)
DarkStruct.jl module + run() (startup, recovery, workers, serve, shutdown)
config.jl Config struct + env parsing
job.jl Job (the queue reference)
queue.jl JobQueue seam + in-process ChannelQueue

4
bin/bench.jl
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env julia
#
# bench.jl: measure end-to-end throughput and server memory for a running FileServer.
# bench.jl: measure end-to-end throughput and server memory for a running DarkStruct.
#
# Two things make this pipeline awkward to benchmark with off-the-shelf tools
# (ab/hey/wrk), and both shape what this script does:
@@ -143,7 +143,7 @@ end
#
# Resolved from the same environment variables src/config.jl reads, so a server
# started with custom dirs is benchmarked correctly. Kept as a standalone table
# rather than `using FileServer` so the harness doesn't pay to load Lux.
# rather than `using DarkStruct` so the harness doesn't pay to load Lux.
sinkdirs() = (
done = get(ENV, "FS_DONE_DIR", "data/done"),

10
bin/bench_stage1.jl
View File

@@ -25,7 +25,7 @@
# guess. This sweep is what demoted stage 1's per-file lines to `@debug`
# (see the note in src/worker.jl); the standalone `logging (...)` rows below
# still price a *formatted* line, i.e. what those lines cost when switched
# back on with `JULIA_DEBUG=FileServer`, while the handler rows show what the
# back on with `JULIA_DEBUG=DarkStruct`, while the handler rows show what the
# stage pays with them off.
# * Concurrency. Components that own a lock (the queue's condition, the
# logger's stream) don't scale, and the ranking at one worker need not be the
@@ -48,14 +48,14 @@
# Reported times are the *minimum* over trials: the floor is the signal and
# everything above it is scheduler, page-cache and GC noise.
using FileServer
using DarkStruct
using Lux
using JSON3
using Logging
using Printf
using Random
const FS = FileServer
const FS = DarkStruct
# ---------------------------------------------------------------- option parsing
@@ -222,10 +222,10 @@ Run `f` under one of the three loggers the cost of logging is bracketed by:
* `:format` `ConsoleLogger` to `devnull`: message formatting and key/value
interpolation, but no I/O.
* `:flush` `FlushLogger(ConsoleLogger(io))` to a real file: what
`FileServer.run` installs, under the redirect it was written for.
`DarkStruct.run` installs, under the redirect it was written for.
* `:debug` the same, at `Debug` level: the stage's per-file lines are
`@debug`, so this is the equivalent of running the server with
`JULIA_DEBUG=FileServer` and the only setting under which they
`JULIA_DEBUG=DarkStruct` and the only setting under which they
are emitted at all.
"""
function with_logger_named(f, which::Symbol, path::AbstractString)

6
bin/bench_stage2.jl
View File

@@ -65,13 +65,13 @@
# Reported times are the *minimum* over trials: the floor is the signal and
# everything above it is scheduler, page-cache and GC noise.
using FileServer
using DarkStruct
using JSON3
using Logging
using Printf
using Random
const FS = FileServer
const FS = DarkStruct
# ---------------------------------------------------------------- option parsing
@@ -245,7 +245,7 @@ Run `f` under one of the loggers the cost of logging is bracketed by:
* `:format` `ConsoleLogger` to `devnull`: message formatting and key/value
interpolation, but no I/O.
* `:flush` `FlushLogger(ConsoleLogger(io))` to a real file: what
`FileServer.run` installs, under the redirect it was written
`DarkStruct.run` installs, under the redirect it was written
for. Stage 2's per-file line is `@info`, not `@debug`, so this
row is what the deployed server actually pays.
"""

4
bin/cluster_sweep.jl
View File

@@ -14,6 +14,6 @@
# auto-promotes to a compaction pass to seed the catalog. Configure via the
# FS_CLUSTER_* / FS_NOMINATED_DIR env vars (see src/config.jl).
using FileServer
using DarkStruct
FileServer.cluster_sweep_cli(ARGS)
DarkStruct.cluster_sweep_cli(ARGS)

2
bin/send_dir.sh
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
#
# send_dir.sh — upload every file in a directory to a running FileServer.
# send_dir.sh — upload every file in a directory to a running DarkStruct.
#
# Each file is POSTed to /upload as multipart/form-data. By default one file per
# request (so you get per-file feedback); use -b to batch all files into a

4
bin/server.jl
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env julia
# Entry point. Run with: julia --project -t auto bin/server.jl
using FileServer
FileServer.run()
using DarkStruct
DarkStruct.run()

8
docker-compose.rabbitmq.yml
View File

@@ -18,8 +18,8 @@ services:
# The built-in guest/guest only authenticates over loopback, so it cannot
# work across the compose network. Override these for anything real; they
# are here so the stack comes up with one command, not as a credential.
RABBITMQ_DEFAULT_USER: "fileserver"
RABBITMQ_DEFAULT_PASS: "fileserver"
RABBITMQ_DEFAULT_USER: "darkstruct"
RABBITMQ_DEFAULT_PASS: "darkstruct"
ports:
# 5672 is published so you can run the server on the host
# (`julia --project=. -t auto bin/server.jl`) against this broker; 15672 is
@@ -40,10 +40,10 @@ services:
start_period: 30s
restart: unless-stopped
file-server:
darkstruct:
environment:
FS_QUEUE_BACKEND: "rabbitmq"
FS_AMQP_URL: "amqp://fileserver:fileserver@rabbitmq:5672/"
FS_AMQP_URL: "amqp://darkstruct:darkstruct@rabbitmq:5672/"
depends_on:
rabbitmq:
# The server retries a broker that isn't up yet (AMQP_CONNECT_RETRY_SECONDS),

4
docker-compose.yml
View File

@@ -1,5 +1,5 @@
services:
file-server:
darkstruct:
build:
context: .
args:
@@ -7,7 +7,7 @@ services:
# Stage-4 text files then get natural-language enrichment but no
# programming language; the server warns at startup and carries on.
WITH_LINGUIST: "true"
image: file-server:latest
image: darkstruct:latest
ports:
- "8080:8080"
volumes:

6
src/FileServer.jl → src/DarkStruct.jl
View File

@@ -1,4 +1,4 @@
module FileServer
module DarkStruct
using Logging
using Random
@@ -67,7 +67,7 @@ end
"""
run(; overrides...)
Start the file server: build config, ensure directories, recover any leftover
Start DarkStruct: build config, ensure directories, recover any leftover
spooled files, spawn the worker pool, then serve HTTP until interrupted
(Ctrl-C / SIGINT or SIGTERM). On shutdown it stops accepting uploads, drains the
queue, waits for workers to finish in-flight files, and exits cleanly.
@@ -134,7 +134,7 @@ function run(; overrides...)
DETECTOR[] = LanguageDetector()
@info "loaded language detector"
@info "starting file-server" backend=cfg.queue_backend host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity
@info "starting darkstruct" backend=cfg.queue_backend host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity
# Zero the counters here, not at module load: `since` should mean "serving
# since", so a scrape's totals cover the run, not the minutes spent loading

6
src/config.jl
View File

@@ -29,7 +29,7 @@ Base.@kwdef struct Config
# advisory one against a polled depth on the broker (see `RabbitQueue`).
queue_backend::Symbol = :channel
amqp_url::String = "amqp://guest:guest@localhost:5672/"
amqp_prefix::String = "fileserver" # queue names are "<prefix>.<stage>"
amqp_prefix::String = "darkstruct" # queue names are "<prefix>.<stage>"
# Unacked messages the broker will hand one stage at a time. Defaults to that
# stage's worker count (0 below means "use it"), so a worker holds at most the
# one it is working on and a restart redelivers the minimum.
@@ -108,7 +108,7 @@ end
Build a `Config` from environment variables, falling back to the struct
defaults. Any keyword `overrides` win over the environment (useful for tests
and for `FileServer.run(; port=...)`).
and for `DarkStruct.run(; port=...)`).
Recognised variables:
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
@@ -171,7 +171,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
nominated_dir = something(nominated_dir, get(ENV, "FS_NOMINATED_DIR", "data/nominated")),
queue_backend = something(queue_backend, parse_backend(get(ENV, "FS_QUEUE_BACKEND", "channel"))),
amqp_url = something(amqp_url, get(ENV, "FS_AMQP_URL", "amqp://guest:guest@localhost:5672/")),
amqp_prefix = something(amqp_prefix, get(ENV, "FS_AMQP_PREFIX", "fileserver")),
amqp_prefix = something(amqp_prefix, get(ENV, "FS_AMQP_PREFIX", "darkstruct")),
amqp_prefetch = something(amqp_prefetch, parse(Int, get(ENV, "FS_AMQP_PREFETCH", "0"))),
amqp_confirms = something(amqp_confirms, parse_bool(get(ENV, "FS_AMQP_CONFIRMS", "true"))),
recover_spool = something(recover_spool, parse_bool(get(ENV, "FS_RECOVER_SPOOL", "false"))),

4
src/model.jl
View File

@@ -4,8 +4,8 @@
# drift out of sync (same input layout, same scaling, same net shape).
#
# The including scope is expected to have already done `using Lux` and
# `using JLD2` (FileServer.jl and bin/train.jl both do), so this file adds no
# top-level `using` of its own and works whether it's pulled into the FileServer
# `using JLD2` (DarkStruct.jl and bin/train.jl both do), so this file adds no
# top-level `using` of its own and works whether it's pulled into the DarkStruct
# module or a bare script.
"Number of input features: first 16 bytes + last 16 bytes of a file."

4
src/worker.jl
View File

@@ -38,7 +38,7 @@
# the 11.7 µs the whole handler takes with them switched off: six times the rest
# of the stage put together. Nearly all of it is `ConsoleLogger` formatting
# (64.2 µs); the FlushLogger's per-message flush is only ~7 µs on top. Switching
# them back on with `JULIA_DEBUG=FileServer` takes the handler to 106.3 µs, i.e.
# them back on with `JULIA_DEBUG=DarkStruct` takes the handler to 106.3 µs, i.e.
# from 85.4k files/s down to 9.4k on one worker.
#
# With logging off the stage is the classifier and nothing else: classify 10.30 µs
@@ -50,7 +50,7 @@
#
# `@debug` is compiled to a min-level check that doesn't evaluate its arguments,
# so a disabled line costs ~0.15 µs rather than ~36 µs. The messages are still
# there when wanted: run with `JULIA_DEBUG=FileServer` to get them back. Errors,
# there when wanted: run with `JULIA_DEBUG=DarkStruct` to get them back. Errors,
# quarantines and lifecycle events stay at `@error`/`@info`, since they are rare and
# their cost doesn't scale with throughput. `GET /stats` (src/stats.jl) is the
# per-file observability that survives, and it is counted, not formatted.

30
test/runtests.jl
View File

@@ -1,11 +1,11 @@
using Test
using FileServer
using DarkStruct
using JSON3
# Pull internals into scope. These aren't exported (only `run` is), but the
# whole risk profile of this pipeline lives in these functions, so we test them
# directly rather than only through the HTTP surface.
using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
using DarkStruct: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
sanitize_filename, recover_dir!, normalize_metadata,
build_metadata, finalize_known!, run_exiftool,
is_binary, handle_unknown_job, handle_classify_job, worker_loop,
@@ -83,11 +83,11 @@ function tmp_config(root; kwargs...)
nominated_dir = joinpath(root, "nominated"),
kwargs...,
)
FileServer.ensure_dirs(cfg)
DarkStruct.ensure_dirs(cfg)
return cfg
end
@testset "FileServer" begin
@testset "DarkStruct" begin
@testset "sanitize_filename" begin
@test sanitize_filename("report.pdf") == "report.pdf"
@@ -101,7 +101,7 @@ end
@test sanitize_filename("a b&c*.d") == "a_b_c_.d"
@test sanitize_filename("") == "unnamed"
# Length is capped.
@test Base.length(sanitize_filename("a"^500)) == FileServer.MAX_NAME_LEN
@test Base.length(sanitize_filename("a"^500)) == DarkStruct.MAX_NAME_LEN
end
@testset "multipart_boundary: extraction from Content-Type" begin
@@ -297,9 +297,9 @@ end
# process that exits instantly, on the hot path of stages 2 and 4. The
# bound here is deliberately loose (a loaded CI box is slow) but far
# under the 100 ms floor the polling version could not beat.
FileServer.run_with_timeout(`true`, 30) # warm up / compile
DarkStruct.run_with_timeout(`true`, 30) # warm up / compile
t0 = time()
out = FileServer.run_with_timeout(`echo hi`, 30)
out = DarkStruct.run_with_timeout(`echo hi`, 30)
elapsed = time() - t0
@test out !== nothing
@test strip(String(out)) == "hi"
@@ -308,7 +308,7 @@ end
@testset "run_with_timeout: kills an overrunning child and reports failure" begin
t0 = time()
out = FileServer.run_with_timeout(`sleep 30`, 1)
out = DarkStruct.run_with_timeout(`sleep 30`, 1)
elapsed = time() - t0
@test out === nothing # timed out → no output, caller degrades
@test elapsed < 5 # killed near the timeout, not after 30 s
@@ -319,11 +319,11 @@ end
# block on wait(proc) forever and the timeout would be unenforceable.
cmd = `sh -c "trap '' TERM; sleep 30"`
t0 = time()
out = FileServer.run_with_timeout(cmd, 1)
out = DarkStruct.run_with_timeout(cmd, 1)
elapsed = time() - t0
@test out === nothing
# 1 s timeout + up to KILL_GRACE_SECONDS before SIGKILL lands.
@test elapsed < 1 + FileServer.KILL_GRACE_SECONDS + 3
@test elapsed < 1 + DarkStruct.KILL_GRACE_SECONDS + 3
end
@testset "run_exiftool: real extraction on a PNG" begin
@@ -403,7 +403,7 @@ end
# binary: pad to one byte short of the window, then a 2-byte 'é'
# (0xc3 0xa9) so only its lead byte lands inside the window.
split = joinpath(root, "split.txt")
write(split, vcat(fill(UInt8('a'), FileServer.CONTENT_SNIFF_BYTES - 1),
write(split, vcat(fill(UInt8('a'), DarkStruct.CONTENT_SNIFF_BYTES - 1),
UInt8[0xc3, 0xa9]))
@test is_binary(split) == false
@@ -414,7 +414,7 @@ end
# Binary garbage past the sniff window is not seen → still text.
far = joinpath(root, "far.txt")
write(far, vcat(fill(UInt8('a'), FileServer.CONTENT_SNIFF_BYTES), UInt8[0x00]))
write(far, vcat(fill(UInt8('a'), DarkStruct.CONTENT_SNIFF_BYTES), UInt8[0x00]))
@test is_binary(far) == false
end
end
@@ -503,10 +503,10 @@ end
# Reads at most LANG_SAMPLE_BYTES, and doesn't choke on a multi-byte
# char straddling that boundary (trailing 'é' half-in the window).
big = joinpath(root, "big.txt")
write(big, vcat(fill(UInt8('a'), FileServer.LANG_SAMPLE_BYTES - 1),
write(big, vcat(fill(UInt8('a'), DarkStruct.LANG_SAMPLE_BYTES - 1),
UInt8[0xc3, 0xa9])) # 'é' split by the edge
s = read_text_sample(big)
@test Base.length(s) == FileServer.LANG_SAMPLE_BYTES - 1 # trailing half-char trimmed
@test Base.length(s) == DarkStruct.LANG_SAMPLE_BYTES - 1 # trailing half-char trimmed
@test all(==('a'), s)
end
end
@@ -1240,7 +1240,7 @@ end
# The rest needs a live broker. Point FS_TEST_AMQP_URL at one to run it:
# docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
# FS_TEST_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ julia --project=. -e 'using Pkg; Pkg.test()'
# FS_TEST_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ julia --project=. -e 'using Pkg; Pkg.test()'
amqp_url = get(ENV, "FS_TEST_AMQP_URL", "")
if isempty(amqp_url)
@info "skipping RabbitMQ integration tests (set FS_TEST_AMQP_URL to run them)"