jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Route by queue, not by directory: stop moving files between stages

Browse Source 
A file is now written once into spool/ and stays there for its whole time in
flight. Stages 1 and 3 hand work on by enqueueing the same Job reference, so
job.path is constant from intake until commit. The three inter-stage renames
(spool->known, spool->unknown, unknown->text) are gone, along with the
known/, unknown/ and text/ directories and their FS_*_DIR settings.

Terminal moves stay: done/, text_done/, binary/ and failed/ still receive the
file, and binary/ in particular must, since it is the corpus the offline
stage-5 discovery sweep reads.

Measured by bin/bench_stage1.jl (2000 x 64 KiB, min of 5): the removed rename
cost 11.6 us per file, the equal of the classifier itself. One stage-1 worker
goes from ~28.6 us/file (~35k files/s) to 11.70 us (85.4k files/s); 16 workers
reach 333k files/s. What is left is classify 10.30 us, the queue handoff
0.12 us, and the disabled @debug lines 0.29 us.

The stage a file has reached now lives only in the queue holding its
reference, and the queues are in-process, so a crash loses it: everything in
spool/ replays from stage 1. That is safe rather than merely tolerable —
classification and the UTF-8 sniff are pure functions of the file's bytes and
the terminal commits rename with force=true, so a replayed file lands where it
would have landed and overwrites its own sidecar. The per-stage directories
were standing in for a durable queue, and charging every file a rename per
stage to do it; src/queue.jl already defines the seam where a broker-backed
JobQueue restores exact resume properly.

Recovery had to change to match. All leftovers now funnel onto the single
stage-1 queue, so the old non-blocking recover_dir! would have capped a
4000-file recovery at 1000 and abandoned the rest. It now blocks on a full
queue, and run() spawns the worker pools before recovering so the consumers
drain it as we fill; reset_metrics! moves above the spawn accordingly.
enqueue_blocking! takes stats::Union{StageStats,Nothing} so recovery reuses
the never-drop retry without charging its wait to a stage's blocked_ns, which
would drive /stats utilization negative.

Tests assert the new invariant positively (the file stays put, the routed
reference is unchanged, no intermediate directory appears) and cover recovery
of more files than the queue can hold. Verified end to end against a real
server: 40 leftovers, stage-1 capacity 3, all recovered and drained to
text_done/ with spool/ and failed/ empty.
This commit is contained in:
Jeffrey Ward
2026-08-07 13:21:49 -04:00
parent 341b61f806
commit 4a22123001
13 changed files with 352 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
bin/bench.jl
View File

@@ -14,8 +14,9 @@
# 2. End-to-end throughput doesn't name the slow stage. The four stages run
# concurrently behind their own queues, so the pipeline's rate is the
# slowest stage's rate and the others are invisible. Directory polling can't
# recover them either — known/, unknown/ and text/ are transient, and a file
# can cross one between two samples. So the server keeps per-stage counters
# recover them either — every in-flight file sits in spool/ whatever stage it
# is at, since stages route by enqueueing rather than by moving bytes. So the
# server keeps per-stage counters
# (src/stats.jl) and we scrape GET /stats before and after: the deltas give
# each stage's throughput, mean service time, and worker utilization, and
# utilization is what actually names the bottleneck (see `stage_report`).
@@ -151,12 +152,12 @@ sinkdirs() = (
failed = get(ENV, "FS_FAILED_DIR", "data/failed"),
)
stagedirs() = (
spool = get(ENV, "FS_SPOOL_DIR", "data/spool"),
known = get(ENV, "FS_KNOWN_DIR", "data/known"),
unknown = get(ENV, "FS_UNKNOWN_DIR", "data/unknown"),
text = get(ENV, "FS_TEXT_DIR", "data/text"),
)
# One directory, not four. Files no longer move between stages — spool/ holds
# every in-flight file at every stage, and which stage it has reached lives in
# the queue holding its reference (src/worker.jl header). So this depth is
# "files in flight", full stop; per-stage depth comes from /stats, which is the
# only place that can see it at all.
stagedirs() = (spool = get(ENV, "FS_SPOOL_DIR", "data/spool"),)
"Count work items in `dir`, ignoring the .meta.json sidecars stages 2/4 write."
function count_files(dir::AbstractString)::Int
@@ -513,7 +514,7 @@ function main(argv)
# Leftovers mid-pipeline would land in the sinks during our window and be
# counted as our throughput, so say so up front rather than quietly skewing.
pending = total(counts(stages))
pending > 0 && @warn "pipeline is not idle: $pending file(s) in the stage dirs; " *
pending > 0 && @warn "pipeline is not idle: $pending file(s) still in spool/; " *
"throughput will include their completions"
# --- corpus
@@ -570,7 +571,7 @@ function main(argv)
"(the server predates src/stats.jl)"
end
# --- sampler: RSS curve, stage dir depths, and queue depths.
# --- sampler: RSS curve, spool depth, and queue depths.
stop = Threads.Atomic{Bool}(false)
rss_samples = Float64[]
depth_max = Dict(k => 0 for k in keys(stages))
@@ -731,7 +732,7 @@ function main(argv)
println("\nnote: $(sink_delta.failed) file(s) landed in $(sinks.failed) — check the server log.")
timed_out &&
println("\nnote: drain stalled with $(accepted - completed) file(s) outstanding. " *
"Check the server log and the stage dirs; raise --timeout if the pipeline is just slow.")
"Check the server log and spool/; raise --timeout if the pipeline is just slow.")
if opts["json"] !== nothing
result = (

67
bin/bench_stage1.jl
View File

@@ -10,7 +10,6 @@
# classify filesize + two 16-byte reads + a 32x1 forward pass
# read_features open, read head, seek, read tail, scale to Float32
# Lux.apply the network on a feature vector already in memory
# move_to rename spool/<f> -> known/<f> or unknown/<f>
# enqueue push a Job reference onto the downstream bounded queue
# logging two @info lines ("classified file", "routed to ...")
#
@@ -36,8 +35,8 @@
# Usage:
# julia --project=. -t auto bin/bench_stage1.jl [options]
#
# --files N files per timed pass for consuming benchmarks (default: 2000)
# --reps N calls per timed pass for non-consuming benchmarks (default: 20000)
# --files N files per timed pass for whole-corpus benchmarks (default: 2000)
# --reps N calls per timed pass for per-call benchmarks (default: 20000)
# --trials N timed passes; the minimum is reported (default: 5)
# --size SPEC corpus file size (default: 64k)
# --dir PATH working directory for the corpus (default: a temp dir under data/)
@@ -113,8 +112,8 @@ const SINK = Ref{Any}(nothing)
Run `pass()` `trials` times and report the fastest, in nanoseconds per operation
(`pass` returns the number of operations it performed). `prepare()` runs before
each pass and is *not* timed — that is where a consuming benchmark puts the file
back where it started. `pass` comes first so callers can pass it as a `do` block.
each pass and is *not* timed — that is where a benchmark resets whatever its
last pass consumed (today: the queues). `pass` comes first so callers can pass it as a `do` block.
The first pass is thrown away: it pays Julia's JIT compilation, which on calls
this small is orders of magnitude more than the thing being measured.
@@ -198,26 +197,11 @@ function make_corpus(cfg::FS.Config, n::Int, size::Int, rng)
return jobs
end
"""
respool!(cfg, jobs)
Put every corpus file back in `spool/`, wherever the last pass left it (known/,
unknown/, or already home). This is the untimed `prepare` step for benchmarks
that consume their input by moving it.
"""
function respool!(cfg::FS.Config, jobs::Vector{FS.Job})
for job in jobs
isfile(job.path) && continue
for dir in (cfg.known_dir, cfg.unknown_dir, cfg.failed_dir)
candidate = joinpath(dir, basename(job.path))
if isfile(candidate)
mv(candidate, job.path; force = true)
break
end
end
end
return nothing
end
# There is no `respool!` step any more. Stage 1 used to consume its input by
# renaming each file into known/ or unknown/, so every repeated pass had to put
# the corpus back first. It now routes by enqueueing alone and leaves the file in
# spool/, so the corpus is reusable as-is and the only per-pass reset is draining
# the queues.
"Drain a queue without blocking, so the next pass starts from empty."
function drain!(q::FS.ChannelQueue)
@@ -263,10 +247,10 @@ end
"""
component_rows(cfg, clf, jobs, opts) -> Vector
Time each piece of stage 1 on its own. Non-consuming pieces (`filesize`,
Time each piece of stage 1 on its own. Per-call pieces (`filesize`,
`read_features`, `Lux.apply`, `classify`, the log lines) run `reps` times over
the corpus; consuming pieces (`move_to`, the full handler) run once per corpus
file with an untimed reset between passes.
the corpus; whole-corpus pieces (`enqueue_blocking!`, the full handler) run once
per corpus file, with an untimed queue drain between passes.
"""
function component_rows(cfg::FS.Config, clf::FS.Classifier, jobs::Vector{FS.Job}, opts)
reps, trials = opts["reps"], opts["trials"]
@@ -317,14 +301,11 @@ function component_rows(cfg::FS.Config, clf::FS.Classifier, jobs::Vector{FS.Job}
reps
end))
# --- move_to: the rename out of spool/. Consuming: reset before each pass.
push!(rows, (; name = "move_to (rename)", part = "route",
ns = best_of(() -> respool!(cfg, jobs); trials) do
@inbounds for job in jobs
SINK[] = FS.move_to(cfg.unknown_dir, job)
end
nfiles
end))
# Stage 1 used to rename each file into known/ or unknown/ before enqueueing
# it, and that rename was measured here as its own line item. It is gone:
# routing is the enqueue alone, and a file does not move until it is
# committed to a terminal sink (src/worker.jl header). So `route` below is
# now just the queue handoff.
# --- enqueue: lock, push, notify on an uncontended, non-full queue
q = FS.ChannelQueue(nfiles + 1)
@@ -381,7 +362,7 @@ function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
(:flush, "handle_classify_job (flush→file)"),
(:debug, "handle_classify_job (JULIA_DEBUG)"))
ns = with_logger_named(which, logfile) do
best_of(() -> (respool!(cfg, jobs); drain!(known); drain!(unknown)); trials) do
best_of(() -> (drain!(known); drain!(unknown)); trials) do
@inbounds for job in jobs
FS.handle_classify_job(job, cfg, 1, known, unknown, stats)
end
@@ -390,7 +371,7 @@ function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
end
push!(rows, (; name = label, part = "total", ns))
end
respool!(cfg, jobs); drain!(known); drain!(unknown)
drain!(known); drain!(unknown)
rm(logfile; force = true)
return rows
end
@@ -419,7 +400,7 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
base = 0.0
for k in counts
ns = with_logger_named(:flush, logfile) do
best_of(() -> (respool!(cfg, jobs); drain!(known); drain!(unknown)); trials) do
best_of(() -> (drain!(known); drain!(unknown)); trials) do
# Static split: each task takes a contiguous slice, so the only
# sharing between workers is the state the server also shares —
# the classifier, the queues, the logger, the filesystem.
@@ -441,7 +422,7 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
k == counts[1] && (base = r)
push!(rows, (; workers = k, ns, files_per_sec = r, speedup = r / base))
end
respool!(cfg, jobs); drain!(known); drain!(unknown)
drain!(known); drain!(unknown)
rm(logfile; force = true)
return rows
end
@@ -481,12 +462,10 @@ function main(argv)
owned = opts["dir"] === nothing
cfg = FS.Config(
spool_dir = joinpath(root, "spool"),
known_dir = joinpath(root, "known"),
unknown_dir = joinpath(root, "unknown"),
failed_dir = joinpath(root, "failed"),
model_path = modelpath,
)
for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.failed_dir)
for d in (cfg.spool_dir, cfg.failed_dir)
mkpath(d)
end
@@ -524,7 +503,7 @@ function main(argv)
# Stage 1's own log lines are `@debug`, so what the deployed handler pays
# for them is the disabled-macro cost, not a formatted line.
accounted = sum(r.ns for r in comps if r.name in
("classify (total)", "move_to (rename)", "enqueue_blocking!", "logging (NullLogger)"))
("classify (total)", "enqueue_blocking!", "logging (NullLogger)"))
println()
@printf("accounted: %s of %s (%.0f%%); unaccounted overhead %s\n",
human_time(accounted), human_time(total), 100 * accounted / total,

47
bin/bench_stage2.jl
View File

@@ -17,7 +17,7 @@
# JSON3.write serialize the sidecar payload
# write + fsync durably persist the sidecar bytes to a temp name
# mv + fsync_dir commit the sidecar, then persist the rename itself
# move_to rename known/<f> -> done/<f>, the commit point
# move_to rename spool/<f> -> done/<f>, the commit point
# logging one @info line ("enriched")
#
# This script times each of those in isolation, then times the real
@@ -28,7 +28,7 @@
# * The corpus must be real files. exiftool's cost depends on what it finds;
# random bytes exit early and would understate the stage by a lot. The
# default corpus is `data/done` — files that already went through stage 2 on
# this machine — copied back into a scratch known/ dir.
# this machine — copied back into a scratch spool/ dir.
# * Two rows price the *alternatives* to one-fork-per-file, because if the
# fork dominates then the only fixes are to stop paying it per file:
# `exiftool (batched Nx)` runs the whole corpus through one process, and
@@ -173,7 +173,7 @@ end
"""
make_corpus(cfg, corpus_dir, n) -> Vector{Job}
Copy up to `n` real files from `corpus_dir` into `known/` and build the `Job`
Copy up to `n` real files from `corpus_dir` into `spool/` and build the `Job`
references a stage-2 worker would dequeue for them — the exact input
`handle_known_job` sees.
@@ -198,21 +198,23 @@ function make_corpus(cfg::FS.Config, corpus_dir::AbstractString, n::Int)
# corpus the file came from.
id, spooled = FS.spool_path(cfg, @sprintf("s2-%04d-%s", i, basename(name)))
cp(src, spooled; force = true)
dest = joinpath(cfg.known_dir, basename(spooled))
mv(spooled, dest; force = true)
push!(jobs, FS.Job(id, basename(name), dest, filesize(dest), time()))
# Staged in spool/, not a known/ dir: stage 1 routes by enqueueing and
# leaves the bytes where intake put them, so this is the exact on-disk
# state a stage-2 worker dequeues into (src/worker.jl header).
push!(jobs, FS.Job(id, basename(name), spooled, filesize(spooled), time()))
end
return jobs
end
"""
reknown!(cfg, jobs)
respool!(cfg, jobs)
Put every corpus file back in `known/`, wherever the last pass left it (done/ or
Put every corpus file back in `spool/`, wherever the last pass left it (done/ or
already home), and delete any sidecar it produced. This is the untimed `prepare`
step for benchmarks that consume their input by committing it.
step for benchmarks that consume their input by committing it — stage 2 still
moves, because its move is the terminal commit, not an inter-stage hop.
"""
function reknown!(cfg::FS.Config, jobs::Vector{FS.Job})
function respool!(cfg::FS.Config, jobs::Vector{FS.Job})
for job in jobs
base = basename(job.path)
for dir in (cfg.done_dir, cfg.failed_dir)
@@ -463,8 +465,8 @@ function component_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
nio
end)
# --- move_to: the rename known/<f> -> done/<f>. Consuming: reset each pass.
add!("move_to (rename)", "commit", best_of(() -> reknown!(cfg, jobs); trials) do
# --- move_to: the rename spool/<f> -> done/<f>. Consuming: reset each pass.
add!("move_to (rename)", "commit", best_of(() -> respool!(cfg, jobs); trials) do
@inbounds for job in jobs
SINK[] = FS.move_to(cfg.done_dir, job)
end
@@ -474,17 +476,17 @@ function component_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
# --- commit_enriched!: the whole sidecar-first commit, extraction excluded.
committable = [j for (j, _) in good]
add!("commit_enriched! (total)", "commit",
best_of(() -> reknown!(cfg, jobs); trials) do
best_of(() -> respool!(cfg, jobs); trials) do
@inbounds for (k, job) in enumerate(committable)
SINK[] = FS.commit_enriched!(cfg.done_dir, job, metas[k])
end
length(committable)
end)
reknown!(cfg, jobs)
respool!(cfg, jobs)
# --- the one @info line, under each logger.
job1, meta1 = good[1][1], metas[1]
logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
logfile = joinpath(dirname(cfg.spool_dir), "bench_stage2.log")
for (which, label) in ((:null, "logging (NullLogger)"),
(:format, "logging (format only)"),
(:flush, "logging (flush→file)"))
@@ -513,13 +515,13 @@ running server actually pays.
function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
trials = opts["trials"]
nfiles = length(jobs)
logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
logfile = joinpath(dirname(cfg.spool_dir), "bench_stage2.log")
rows = []
for (which, label) in ((:null, "handle_known_job (NullLogger)"),
(:format, "handle_known_job (format only)"),
(:flush, "handle_known_job (flush→file)"))
ns = with_logger_named(which, logfile) do
best_of(() -> reknown!(cfg, jobs); trials) do
best_of(() -> respool!(cfg, jobs); trials) do
@inbounds for job in jobs
FS.handle_known_job(job, cfg, 1)
end
@@ -528,7 +530,7 @@ function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
end
push!(rows, (; name = label, part = "total", ns))
end
reknown!(cfg, jobs)
respool!(cfg, jobs)
rm(logfile; force = true)
return rows
end
@@ -556,13 +558,13 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
[parse(Int, s) for s in split(String(opts["threads"]), ",")]
counts = sort(unique(filter(k -> 1 <= k <= Threads.nthreads(), counts)))
logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
logfile = joinpath(dirname(cfg.spool_dir), "bench_stage2.log")
rows = []
base = 0.0
for k in counts
ns = with_logger_named(:flush, logfile) do
next = Threads.Atomic{Int}(1)
best_of(() -> (reknown!(cfg, jobs); next[] = 1); trials) do
best_of(() -> (respool!(cfg, jobs); next[] = 1); trials) do
# Shared counter, not a contiguous slice: every worker takes the
# next unclaimed file the moment it frees up, exactly as the
# server's pool takes the next job off the known queue.
@@ -582,7 +584,7 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
k == counts[1] && (base = r)
push!(rows, (; workers = k, ns, files_per_sec = r, speedup = r / base))
end
reknown!(cfg, jobs)
respool!(cfg, jobs)
rm(logfile; force = true)
return rows
end
@@ -622,12 +624,11 @@ function main(argv)
owned = opts["dir"] === nothing
cfg = FS.Config(
spool_dir = joinpath(root, "spool"),
known_dir = joinpath(root, "known"),
done_dir = joinpath(root, "done"),
failed_dir = joinpath(root, "failed"),
exiftool_timeout = opts["timeout"],
)
for d in (cfg.spool_dir, cfg.known_dir, cfg.done_dir, cfg.failed_dir)
for d in (cfg.spool_dir, cfg.done_dir, cfg.failed_dir)
mkpath(d)
end