jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Route by queue, not by directory: stop moving files between stages

Browse Source 
A file is now written once into spool/ and stays there for its whole time in
flight. Stages 1 and 3 hand work on by enqueueing the same Job reference, so
job.path is constant from intake until commit. The three inter-stage renames
(spool->known, spool->unknown, unknown->text) are gone, along with the
known/, unknown/ and text/ directories and their FS_*_DIR settings.

Terminal moves stay: done/, text_done/, binary/ and failed/ still receive the
file, and binary/ in particular must, since it is the corpus the offline
stage-5 discovery sweep reads.

Measured by bin/bench_stage1.jl (2000 x 64 KiB, min of 5): the removed rename
cost 11.6 us per file, the equal of the classifier itself. One stage-1 worker
goes from ~28.6 us/file (~35k files/s) to 11.70 us (85.4k files/s); 16 workers
reach 333k files/s. What is left is classify 10.30 us, the queue handoff
0.12 us, and the disabled @debug lines 0.29 us.

The stage a file has reached now lives only in the queue holding its
reference, and the queues are in-process, so a crash loses it: everything in
spool/ replays from stage 1. That is safe rather than merely tolerable —
classification and the UTF-8 sniff are pure functions of the file's bytes and
the terminal commits rename with force=true, so a replayed file lands where it
would have landed and overwrites its own sidecar. The per-stage directories
were standing in for a durable queue, and charging every file a rename per
stage to do it; src/queue.jl already defines the seam where a broker-backed
JobQueue restores exact resume properly.

Recovery had to change to match. All leftovers now funnel onto the single
stage-1 queue, so the old non-blocking recover_dir! would have capped a
4000-file recovery at 1000 and abandoned the rest. It now blocks on a full
queue, and run() spawns the worker pools before recovering so the consumers
drain it as we fill; reset_metrics! moves above the spawn accordingly.
enqueue_blocking! takes stats::Union{StageStats,Nothing} so recovery reuses
the never-drop retry without charging its wait to a stage's blocked_ns, which
would drive /stats utilization negative.

Tests assert the new invariant positively (the file stays put, the routed
reference is unchanged, no intermediate directory appears) and cover recovery
of more files than the queue can hold. Verified end to end against a real
server: 40 leftovers, stage-1 capacity 3, all recovered and drained to
text_done/ with spool/ and failed/ empty.
This commit is contained in:
Jeffrey Ward
2026-08-07 13:21:49 -04:00
parent 341b61f806
commit 4a22123001
13 changed files with 352 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
test/runtests.jl
View File

@@ -8,7 +8,8 @@ using JSON3
using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
sanitize_filename, recover_dir!, normalize_metadata,
build_metadata, finalize_known!, run_exiftool,
is_binary, handle_unknown_job, worker_loop,
is_binary, handle_unknown_job, handle_classify_job, worker_loop,
load_classifier, CLASSIFIER,
capacity, StageStats, IntakeStats, Metrics, METRICS, reset_metrics!,
record_job!, enqueue_blocking!, stats_snapshot, STAGE_KEYS,
detect_natural_language, run_linguist, detect_programming_language,
@@ -70,10 +71,7 @@ end
function tmp_config(root; kwargs...)
cfg = Config(;
spool_dir = joinpath(root, "spool"),
known_dir = joinpath(root, "known"),
unknown_dir = joinpath(root, "unknown"),
binary_dir = joinpath(root, "binary"),
text_dir = joinpath(root, "text"),
done_dir = joinpath(root, "done"),
text_done_dir = joinpath(root, "text_done"),
failed_dir = joinpath(root, "failed"),
@@ -279,7 +277,7 @@ end
mktempdir() do root
cfg = tmp_config(root; exiftool_timeout=5)
# Point at a nonexistent file → exiftool exits non-zero → degraded.
job = Job("id-3", "gone.dat", joinpath(cfg.known_dir, "id-3-gone.dat"), 99, 0.0)
job = Job("id-3", "gone.dat", joinpath(cfg.spool_dir, "id-3-gone.dat"), 99, 0.0)
m = build_metadata(job, cfg)
@test m.error !== nothing
@test m.file_type === nothing
@@ -341,7 +339,7 @@ end
mktempdir() do root
cfg = tmp_config(root)
# A real known-stage file to enrich.
src = joinpath(cfg.known_dir, "id-9-pixel.png")
src = joinpath(cfg.spool_dir, "id-9-pixel.png")
write(src, PNG_1x1)
job = Job("id-9", "pixel.png", src, Base.length(PNG_1x1), 0.0)
@@ -425,7 +423,7 @@ end
stats = StageStats()
# A binary file (embedded NUL) lands in binary/ and is NOT enqueued.
bpath = joinpath(cfg.unknown_dir, "id-b-blob.dat")
bpath = joinpath(cfg.spool_dir, "id-b-blob.dat")
write(bpath, UInt8[0x00, 0xFF, 0x10])
bjob = Job("id-b", "blob.dat", bpath, filesize(bpath), 0.0)
handle_unknown_job(bjob, cfg, 1, text_queue, stats)
@@ -433,19 +431,49 @@ end
@test !isfile(bpath)
@test length(text_queue) == 0
# A text file lands in text/ AND is routed onto the stage-4 queue,
# with its path updated to the new text/ location.
tpath = joinpath(cfg.unknown_dir, "id-t-notes.log")
# A text file is routed onto the stage-4 queue and *does not move*:
# stage 4 is still to come, so the bytes stay in spool/ and the same
# reference is handed on. Regression guard against re-introducing an
# intermediate hop — the queue, not a directory, is what records that
# this file has cleared triage.
tpath = joinpath(cfg.spool_dir, "id-t-notes.log")
write(tpath, "just some log text\n")
tjob = Job("id-t", "notes.log", tpath, filesize(tpath), 0.0)
handle_unknown_job(tjob, cfg, 1, text_queue, stats)
moved = joinpath(cfg.text_dir, "id-t-notes.log")
@test isfile(moved)
@test !isfile(tpath)
@test isfile(tpath) # stayed put
@test length(text_queue) == 1
routed = dequeue!(text_queue)
@test routed.id == "id-t"
@test routed.path == moved
@test routed.path == tpath # reference unchanged
end
end
@testset "handle_classify_job: routes by enqueue only, never moves the file" begin
# Stage 1's entire job is picking a queue. Whichever way the classifier
# votes, the bytes must stay where intake wrote them and the *same* Job
# must be handed on — the queue records the phase, not a directory.
# Asserted against both queues at once so the test doesn't depend on how
# the committed model happens to label these bytes.
CLASSIFIER[] = load_classifier(joinpath(@__DIR__, "..", "model", "classifier.jld2"))
mktempdir() do root
cfg = tmp_config(root)
known_queue, unknown_queue = ChannelQueue(10), ChannelQueue(10)
stats = StageStats()
path = joinpath(cfg.spool_dir, "id-c-pixel.png")
write(path, PNG_1x1)
job = Job("id-c", "pixel.png", path, Base.length(PNG_1x1), 0.0)
handle_classify_job(job, cfg, 1, known_queue, unknown_queue, stats)
@test isfile(path) # stayed put
@test length(known_queue) + length(unknown_queue) == 1 # routed exactly once
routed = length(known_queue) == 1 ? dequeue!(known_queue) : dequeue!(unknown_queue)
@test routed.path == path # reference unchanged
@test routed.id == "id-c"
# No intermediate directory was invented alongside spool/.
@test !isdir(joinpath(root, "known"))
@test !isdir(joinpath(root, "unknown"))
end
end
@@ -503,7 +531,7 @@ end
cfg = tmp_config(root)
d = LanguageDetector()
src = joinpath(cfg.text_dir, "id-x-script.py")
src = joinpath(cfg.spool_dir, "id-x-script.py")
write(src, join(["# a short program in English prose comment",
"import sys",
"def greet(name):",
@@ -542,7 +570,7 @@ end
cfg = tmp_config(root)
d = LanguageDetector()
src = joinpath(cfg.text_dir, "id-h-readme.md")
src = joinpath(cfg.spool_dir, "id-h-readme.md")
write(src, "# Project\n\nThis project does something useful and interesting for everyone.\n")
job = Job("id-h", "readme.md", src, filesize(src), 0.0)
@@ -739,6 +767,39 @@ end
end
end
@testset "recover_dir!: blocks on a full queue, recovers every file" begin
# The regression this guards is data loss, not slowness. `spool/` now
# holds every in-flight file at every stage, so a crash under load leaves
# far more leftovers than one queue's capacity — and the old non-blocking
# recovery warned and abandoned the excess, which is exactly the work
# recovery exists to save. With the pool live, recovery must park until
# the consumer makes room and come back with all of it.
mktempdir() do root
dir = joinpath(root, "spool"); mkpath(dir)
n_files = 25
for i in 1:n_files
write(joinpath(dir, string("id-", lpad(i, 3, '0'), "-f.dat")), "x")
end
q = ChannelQueue(4) # deliberately far smaller than n_files
drained = Job[]
consumer = Threads.@spawn begin
while Base.length(drained) < n_files
job = dequeue!(q)
job === nothing && break
push!(drained, job)
end
end
n = recover_dir!(dir, q) # would strand 21 files if it didn't block
wait(consumer)
@test n == n_files
@test Base.length(drained) == n_files
@test Base.length(Set(j.path for j in drained)) == n_files # no duplicates
end
end
# Helper: write a "file" of raw bytes into a dir with a UUID-ish unique name,
# returning its path. Mirrors what stage-3 deposits into binary/.
function drop_binary(dir, bytes; name=string(rand(UInt128)))
@@ -1012,7 +1073,7 @@ end
text_queue = ChannelQueue(1)
@test enqueue!(text_queue, Job("filler", "f", "/tmp/f", 1, 0.0))
p = joinpath(cfg.unknown_dir, "id-t-notes.log")
p = joinpath(cfg.spool_dir, "id-t-notes.log")
write(p, "plain text\n")
job = Job("id-t", "notes.log", p, filesize(p), 0.0)
@@ -1024,7 +1085,7 @@ end
wait(drainer)
@test stats.blocked_ns[] > 50_000_000
@test length(text_queue) == 1 # the file did get through
@test isfile(joinpath(cfg.text_dir, "id-t-notes.log"))
@test isfile(p) # and stayed in spool/
end
end