jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial text cleanup

Browse Source
This commit is contained in:
Jeffrey Ward
2026-08-19 12:55:19 -04:00
parent 819a4cce7a
commit cc84d70d52
26 changed files with 454 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
bin/bench.jl
View File

@@ -1,12 +1,12 @@
#!/usr/bin/env julia
#
# bench.jl — measure end-to-end throughput and server memory for a running FileServer.
# bench.jl: measure end-to-end throughput and server memory for a running FileServer.
#
# Two things make this pipeline awkward to benchmark with off-the-shelf tools
# (ab/hey/wrk), and both shape what this script does:
#
# 1. HTTP latency is not throughput. /upload returns 202 as soon as the bytes
# are spooled and a reference is enqueued — all four stages run afterwards.
# are spooled and a reference is enqueued, and all four stages run afterwards.
# So real throughput is the *arrival rate at the terminal sinks*
# (done/, text_done/, binary/, failed/), not the response rate. We upload a
# corpus, then poll the sinks until the file count stops moving.
@@ -14,7 +14,7 @@
# 2. End-to-end throughput doesn't name the slow stage. The four stages run
# concurrently behind their own queues, so the pipeline's rate is the
# slowest stage's rate and the others are invisible. Directory polling can't
# recover them either — every in-flight file sits in spool/ whatever stage it
# recover them either: every in-flight file sits in spool/ whatever stage it
# is at, since stages route by enqueueing rather than by moving bytes. So the
# server keeps per-stage counters
# (src/stats.jl) and we scrape GET /stats before and after: the deltas give
@@ -36,7 +36,7 @@
#
# The harness talks to the server only over HTTP (/upload, /health, /stats) and
# reads the on-disk sink layout; it must run on the same machine (sink dirs and
# /proc). If /stats is missing — an older build — everything else still works and
# /proc). If /stats is missing (an older build) everything else still works and
# the per-stage section is skipped.
#
# Usage:
@@ -46,7 +46,7 @@
# --files N number of files to upload (default: 200)
# --size S size of each generated file, e.g. 4k, 512k, 8m, 1g (default: 64k)
# --concurrency J uploads in flight at once (default: 8)
# --kind K binary | text | mixed — what to generate (default: binary)
# --kind K binary | text | mixed: what to generate (default: binary)
# --corpus DIR upload an existing directory instead of generating
# (the only way to exercise stage 2: point it at real known files)
# --keep-corpus don't delete the generated corpus on exit
@@ -152,7 +152,7 @@ sinkdirs() = (
failed = get(ENV, "FS_FAILED_DIR", "data/failed"),
)
# One directory, not four. Files no longer move between stages — spool/ holds
# One directory, not four. Files no longer move between stages: spool/ holds
# every in-flight file at every stage, and which stage it has reached lives in
# the queue holding its reference (src/worker.jl header). So this depth is
# "files in flight", full stop; per-stage depth comes from /stats, which is the
@@ -200,7 +200,7 @@ Find the running server process, or `nothing`.
server: any shell launched with the command in its own argv (`sh -c 'julia …
bin/server.jl > log'`, a `setsid`/`nohup` wrapper, even the terminal running the
benchmark) matches the same pattern. Sampling one of those reports a few MiB of
shell as the server's memory — a wrong answer that looks plausible, which is the
shell as the server's memory: a wrong answer that looks plausible, which is the
worst kind.
So candidates are filtered by what each process *is* (`/proc/<pid>/comm`, the
@@ -309,7 +309,7 @@ function stage_deltas(before, after, peak_depth::Dict{Int,Int})
end
"JSON has no NaN. A stage that completed nothing has no service time, and `null`
is the honest way to say that — writing NaN just makes JSON3 throw."
is the honest way to say that; writing NaN just makes JSON3 throw."
json_num(x::Real) = isfinite(x) ? x : nothing
pad(s, n) = rpad(string(s), n)
@@ -321,7 +321,7 @@ Print the per-stage table and say which stage is the bottleneck.
The verdict reads utilization, not throughput: in a pipeline every stage
completes the same files, so at steady state they all report nearly the same
files/s regardless of which one is the constraint. What separates them is how
hard each pool had to work to keep up — the bottleneck is pinned near 1.0 while
hard each pool had to work to keep up: the bottleneck is pinned near 1.0 while
its neighbours idle.
"""
function stage_report(deltas::Vector{StageDelta}, window::Float64)
@@ -356,14 +356,14 @@ function stage_report(deltas::Vector{StageDelta}, window::Float64)
"$(fmt(utilization(top, window) * 100, 0))% utilization of " *
"$(top.workers) worker(s)")
if utilization(top, window) < 0.5
println(" — but no stage is near saturated: the pipeline is " *
println(" ...but no stage is near saturated: the pipeline is " *
"waiting on intake,\n not on itself. Raise --concurrency " *
"or --files to load it properly.")
end
for d in worked
blocked_share(d) > 0.25 && println(" ! stage $(d.stage) ($(d.name)) spent " *
"$(fmt(blocked_share(d) * 100, 0))% of its time parked on a full downstream " *
"queue —\n it is being held up by the stage after it, not doing that work itself.")
"queue.\n It is being held up by the stage after it, not doing that work itself.")
end
return nothing
end
@@ -459,7 +459,7 @@ end
Upload every path, at most `concurrency` in flight.
A bounded set of worker tasks pulling from a shared index keeps exactly
`concurrency` requests in flight for the whole run — unlike batching, where each
`concurrency` requests in flight for the whole run, unlike batching, where each
batch stalls on its slowest (largest) file and the real concurrency sags.
"""
function upload_all(url::String, paths::Vector{String}, concurrency::Int)
@@ -504,7 +504,7 @@ function main(argv)
try
HTTP.get(string(rstrip(url, '/'), "/health"); retry = false, readtimeout = 5)
catch e
println(stderr, "cannot reach $url/health — is the server running?")
println(stderr, "cannot reach $url/health. Is the server running?")
println(stderr, " start it with: julia --project=. -t auto bin/server.jl")
return 1
end
@@ -548,7 +548,7 @@ function main(argv)
"reporting sampled RSS only"
# Baseline is read *after* the reset, not before. VmHWM restarts
# from whatever RSS is at the moment of the reset, so a baseline
# sampled earlier is measured against a different origin — and if
# sampled earlier is measured against a different origin, and if
# the GC hands memory back in between, the run reports negative
# growth, which is nonsense on its face.
r2 = read_rss(pid)
@@ -587,8 +587,8 @@ function main(argv)
depth_max[k] = max(depth_max[k], getfield(d, k))
end
# Queue depth, unlike directory depth, can't be missed by a slow
# sample in the same way — a file's *reference* sits in the queue
# for the whole time it waits — so this is the depth the stage
# sample in the same way, since a file's *reference* sits in the
# queue for the whole time it waits, so this is the depth the stage
# table reports.
if stats_before !== nothing
s = scrape_stats(url)
@@ -673,7 +673,7 @@ function main(argv)
"$(human(corpusbytes / length(paths))) avg")
println("concurrency $(opts["concurrency"])")
println()
println("INTAKE (HTTP 202 — bytes spooled, not processed)")
println("INTAKE (HTTP 202: bytes spooled, not processed)")
println(" accepted $accepted of $(length(paths)) [202: $n_202, 503: $n_503, error: $n_err]")
println(" wall $(fmt(intake_secs))s")
println(" rate $(fmt(accepted / max(intake_secs, 1e-9))) files/s, " *
@@ -700,7 +700,7 @@ function main(argv)
println("SERVER MEMORY (pid $pid)")
println(" baseline RSS $(human(baseline_rss))")
println(" peak RSS $(human(peak_rss))" *
(peak_reset ? " (kernel VmHWM, reset at start)" : " (sampled — may miss spikes)"))
(peak_reset ? " (kernel VmHWM, reset at start)" : " (sampled; may miss spikes)"))
growth = peak_rss - baseline_rss
if growth <= 0
# RSS never got back to where it started, so the run's own cost
@@ -708,13 +708,13 @@ function main(argv)
# Printing a negative "growth" would invite reading a memory
# *saving* into what is really "too small to measure here".
println(" growth none measurable (peak never exceeded the baseline)")
println(" The baseline was still falling when we sampled it — give the")
println(" The baseline was still falling when we sampled it. Give the")
println(" server ~30s to settle after startup for a comparable figure.")
else
println(" growth $(human(growth))")
println(" per in-flight $(human(growth / opts["concurrency"])) " *
"at $(human(corpusbytes / length(paths))) avg file size")
println(" (should not grow with file size — intake streams to disk)")
println(" (should not grow with file size: intake streams to disk)")
end
# Julia's GC returns memory to the OS lazily, so a second run on the
# same process starts from an inflated baseline and under-reports
@@ -729,7 +729,7 @@ function main(argv)
println("=" ^ 68)
sink_delta.failed > 0 &&
println("\nnote: $(sink_delta.failed) file(s) landed in $(sinks.failed) — check the server log.")
println("\nnote: $(sink_delta.failed) file(s) landed in $(sinks.failed); check the server log.")
timed_out &&
println("\nnote: drain stalled with $(accepted - completed) file(s) outstanding. " *
"Check the server log and spool/; raise --timeout if the pipeline is just slow.")