Demote stage-1 per-file logging to @debug; add stage-1 decomposition benchmark
bin/bench_stage1.jl takes handle_classify_job apart — filesize, read_features, Lux.apply, classify, move_to, enqueue_blocking!, and the log lines — times each in isolation, then times the real handler end to end under four loggers so the parts can be checked against the whole. It found that logging was stage 1's dominant cost: as @info the two per-file lines cost ~71 us of the handler's ~118 us, roughly 6x the classifier (10.6 us) and 6x the rename (11.6 us). Nearly all of it is ConsoleLogger formatting (~64 us), not the FlushLogger's per-message flush (~8 us). Demoting them to @debug takes stage 1 from 8.5k files/s to 35.3k files/s on one worker (4.2x). The messages are still available with JULIA_DEBUG=FileServer, which the benchmark also prices (133 us/file). What remains splits evenly between the rename (11.7 us) and classify (10.7 us, itself 74% feature read), so stage 1 is now filesystem-bound; its thread sweep peaks at ~4 workers.
This commit is contained in:
@@ -18,6 +18,21 @@
|
||||
# this worker-to-worker handoff blocks.
|
||||
const ROUTE_ENQUEUE_RETRY_SECONDS = 0.05
|
||||
|
||||
# Per-file logging in stage 1 is `@debug`, not `@info`, because it is the
|
||||
# stage's dominant cost. Measured by bin/bench_stage1.jl (2000 x 64 KiB files,
|
||||
# min of 5 trials): the two log lines cost ~71 µs of the ~118 µs
|
||||
# `handle_classify_job` spent per file — roughly 6x the classifier (10.6 µs) and
|
||||
# 6x the rename (11.6 µs). Nearly all of it is `ConsoleLogger` formatting
|
||||
# (~64 µs); the FlushLogger's per-message flush is only ~8 µs on top. Demoting
|
||||
# them takes stage 1 from ~8.5k files/s to ~35k files/s on one worker.
|
||||
#
|
||||
# `@debug` is compiled to a min-level check that doesn't evaluate its arguments,
|
||||
# so a disabled line costs ~0.15 µs rather than ~36 µs. The messages are still
|
||||
# there when wanted: run with `JULIA_DEBUG=FileServer` to get them back. Errors,
|
||||
# quarantines and lifecycle events stay at `@error`/`@info` — they are rare and
|
||||
# their cost doesn't scale with throughput. `GET /stats` (src/stats.jl) is the
|
||||
# per-file observability that survives, and it is counted, not formatted.
|
||||
|
||||
"""
|
||||
handle_classify_job(job, cfg, worker_id, known_queue, unknown_queue)
|
||||
|
||||
@@ -35,7 +50,7 @@ function handle_classify_job(job::Job, cfg::Config, worker_id::Int,
|
||||
known_queue::JobQueue, unknown_queue::JobQueue,
|
||||
stats::StageStats)
|
||||
classification = classify(CLASSIFIER[], job.path)
|
||||
@info "classified file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification
|
||||
@debug "classified file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification
|
||||
|
||||
if classification === :known
|
||||
dest = move_to(cfg.known_dir, job)
|
||||
@@ -43,13 +58,13 @@ function handle_classify_job(job::Job, cfg::Config, worker_id::Int,
|
||||
# known queue full → park and retry, don't drop (time charged to blocked_ns)
|
||||
enqueue_blocking!(known_queue, routed, stats;
|
||||
retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
|
||||
@info "routed to enrichment" worker=worker_id id=job.id dest=dest
|
||||
@debug "routed to enrichment" worker=worker_id id=job.id dest=dest
|
||||
else
|
||||
dest = move_to(cfg.unknown_dir, job)
|
||||
routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
|
||||
enqueue_blocking!(unknown_queue, routed, stats;
|
||||
retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
|
||||
@info "routed to content triage" worker=worker_id id=job.id dest=dest
|
||||
@debug "routed to content triage" worker=worker_id id=job.id dest=dest
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user