Compare commits
14 Commits
842668b2ac
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 341b61f806 | |||
| c692d14a2c | |||
| c5d488d9b4 | |||
| 0d8eba05b8 | |||
| e18ac45d70 | |||
| f4e3f5be0b | |||
| 584bad02a7 | |||
| d9f32d9aaf | |||
| 2c8de488a1 | |||
| fac3adbaf6 | |||
| 9fd1bf385b | |||
| e42e8ef8af | |||
| 2a46f5021a | |||
| 1c7d7d6cad |
@@ -2,7 +2,7 @@
|
||||
|
||||
julia_version = "1.12.6"
|
||||
manifest_format = "2.0"
|
||||
project_hash = "1d7ce552eaac13c97732dccac45c97d40527ec58"
|
||||
project_hash = "ed6bd1b772452682c906ce1236b89ccb1b0876fc"
|
||||
|
||||
[[deps.ADTypes]]
|
||||
git-tree-sha1 = "d9aaef7c63466eee4de23b4d9dad03629df54bea"
|
||||
@@ -309,7 +309,7 @@ weakdeps = ["HTTP"]
|
||||
HTTPExt = "HTTP"
|
||||
|
||||
[[deps.FileServer]]
|
||||
deps = ["HTTP", "JSON3", "Logging", "Oxygen", "UUIDs"]
|
||||
deps = ["HTTP", "JLD2", "JSON3", "Languages", "Logging", "Lux", "Optimisers", "Oxygen", "Random", "UUIDs", "Zygote"]
|
||||
path = "."
|
||||
uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
|
||||
version = "0.1.0"
|
||||
@@ -468,6 +468,12 @@ weakdeps = ["Serialization"]
|
||||
[deps.LRUCache.extensions]
|
||||
SerializationExt = ["Serialization"]
|
||||
|
||||
[[deps.Languages]]
|
||||
deps = ["InteractiveUtils", "JSON", "RelocatableFolders"]
|
||||
git-tree-sha1 = "023ac3b12f82da68ed2556c71a134a03e1a11343"
|
||||
uuid = "8ef0a80b-9436-5d2c-a485-80b904378c43"
|
||||
version = "0.4.7"
|
||||
|
||||
[[deps.LibCURL]]
|
||||
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
|
||||
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
|
||||
|
||||
11
Project.toml
11
Project.toml
@@ -7,10 +7,12 @@ authors = ["wardjm@gmail.com"]
|
||||
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
|
||||
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
|
||||
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
|
||||
Languages = "8ef0a80b-9436-5d2c-a485-80b904378c43"
|
||||
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
|
||||
Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
|
||||
Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
|
||||
Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
|
||||
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
|
||||
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
|
||||
|
||||
@@ -18,9 +20,18 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
|
||||
HTTP = "1.11.0"
|
||||
JLD2 = "0.6.4"
|
||||
JSON3 = "1.14.3"
|
||||
Languages = "0.4.7"
|
||||
Logging = "1.11.0"
|
||||
Lux = "1.31.4"
|
||||
Optimisers = "0.4.7"
|
||||
Oxygen = "1.10.2"
|
||||
Random = "1.11.0"
|
||||
UUIDs = "1.11.0"
|
||||
Zygote = "0.7.11"
|
||||
|
||||
[extras]
|
||||
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
|
||||
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
|
||||
|
||||
[targets]
|
||||
test = ["Test", "JSON3"]
|
||||
|
||||
770
README.md
770
README.md
@@ -11,43 +11,305 @@ classifier that labels it **known** (a file type resembling the training set) or
|
||||
|
||||
## Architecture
|
||||
|
||||
The pipeline is four stages, each with its own bounded queue and its own worker
|
||||
pool (tuned independently, since classification is CPU-bound, known-file
|
||||
enrichment is process-/IO-bound, content triage is cheap IO, and language
|
||||
enrichment mixes CPU with a subprocess):
|
||||
|
||||
```
|
||||
POST /upload (multipart)
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐ spool bytes to disk
|
||||
┌─────────────────┐ stream bytes to disk (never buffered)
|
||||
│ HTTP handler │────────────────────────► data/spool/<uuid>-<name>
|
||||
│ (Oxygen.jl) │
|
||||
│ (streaming) │
|
||||
└────────┬─────────┘ enqueue reference (non-blocking)
|
||||
│ │
|
||||
▼ ▼
|
||||
202 + job IDs ┌───────────────┐
|
||||
(503 if full) │ work queue │ bounded, thread-safe
|
||||
│ (Channel-ish)│
|
||||
└───────┬───────┘
|
||||
│ dequeue
|
||||
┌───────────────┼───────────────┐
|
||||
▼ ▼ ▼
|
||||
worker 1 worker 2 … worker N (Threads.@spawn)
|
||||
│
|
||||
success ────┴──► data/done/<uuid>-<name>
|
||||
failure ───────► data/failed/<uuid>-<name>
|
||||
202 + job IDs ┌────────────────────┐
|
||||
(503 if full) │ stage-1 queue │ classification
|
||||
└─────────┬──────────┘
|
||||
│ dequeue
|
||||
┌───────────────────┼───────────────────┐
|
||||
▼ ▼ ▼
|
||||
classify wkr 1 classify wkr 2 … classify wkr N
|
||||
│
|
||||
┌────────────┴────────────┐
|
||||
:unknown :known
|
||||
│ move to data/unknown/, │ move to data/known/, then
|
||||
▼ then enqueue (blocking) ▼ enqueue (blocking backpressure)
|
||||
┌────────────────────┐ ┌────────────────────┐
|
||||
│ unknown queue │ │ known queue │ enrichment
|
||||
└─────────┬──────────┘ └─────────┬──────────┘
|
||||
│ dequeue │ dequeue
|
||||
┌────────┼────────┐ ┌───────────┼───────────┐
|
||||
▼ ▼ ▼ ▼ ▼ ▼
|
||||
unk 1 unk 2 … unk K known wkr 1 known wkr 2 … known wkr M
|
||||
│ binary-vs-text sniff │ exiftool → normalized sidecar
|
||||
├─► data/binary/<uuid>-<name> success ──┴──► data/done/<uuid>-<name>
|
||||
│ (terminal) data/done/<uuid>-<name>.meta.json
|
||||
│ (sidecar-first commit)
|
||||
│ :text move to data/text/, failure ───────► data/failed/<uuid>-<name>
|
||||
▼ then enqueue (blocking backpressure)
|
||||
┌────────────────────┐
|
||||
│ text queue │ language enrichment
|
||||
└─────────┬──────────┘
|
||||
│ dequeue
|
||||
┌────────┼────────┐
|
||||
▼ ▼ ▼
|
||||
txt 1 txt 2 … txt P
|
||||
│ Languages.jl (natural language) + github-linguist (programming language)
|
||||
└─► data/text_done/<uuid>-<name> + data/text_done/<uuid>-<name>.meta.json
|
||||
(sidecar-first commit)
|
||||
```
|
||||
|
||||
Stages 2 (known-file enrichment) and 3 (content triage) run in parallel: stage 1
|
||||
feeds both the known and unknown queues. Stage 3 in turn feeds stage 4 (language
|
||||
enrichment) for every file it sorts as text.
|
||||
|
||||
Key properties:
|
||||
|
||||
- **Fast intake:** the queue only ever carries small references; file bytes live
|
||||
on disk, so memory stays flat regardless of file size.
|
||||
- **Backpressure:** the queue is bounded (default 1000). When full, uploads get
|
||||
`503 Service Unavailable` instead of silently piling up.
|
||||
- **Crash-resilient:** files survive on disk. On startup, anything left in
|
||||
`data/spool/` is re-enqueued (`recovered = N` in the log).
|
||||
on disk, so memory stays flat regardless of file size. This holds end to end:
|
||||
intake **streams** each upload from the socket to the spool file a chunk at a
|
||||
time (`FS_UPLOAD_CHUNK_BYTES`, default 64 KiB) rather than buffering the body,
|
||||
and every worker reads only a bounded prefix. Measured: uploads of 256 MiB,
|
||||
1 GiB and 2 GiB each grow resident memory by ~20 MiB — a flat line in file
|
||||
size. See "Streaming intake" and "Benchmarking" below.
|
||||
- **Backpressure:** each queue is bounded (default 1000). When the *intake* queue
|
||||
is full, uploads get `503 Service Unavailable`. When the *known* queue is full,
|
||||
the stage-1 worker blocks and retries (a classified file is never dropped).
|
||||
- **Crash-resilient:** files survive on disk. On startup, recovery is
|
||||
stage-aware: leftovers in `data/spool/` re-enter classification, `data/known/`
|
||||
re-enter enrichment, `data/unknown/` re-enter content triage, and `data/text/`
|
||||
re-enter language enrichment (`recovered` / `recovered_known` /
|
||||
`recovered_unknown` / `recovered_text` in the log), so a file resumes at its
|
||||
correct stage instead of restarting from scratch.
|
||||
- **Graceful shutdown:** SIGINT (Ctrl-C) and SIGTERM (systemd/Docker/k8s `stop`)
|
||||
both stop accepting uploads, drain the queue, wait for in-flight files to
|
||||
finish, then exit. (See "Shutdown" below for one cosmetic caveat on SIGTERM.)
|
||||
both stop accepting uploads, then drain the stages *in order* — close the
|
||||
stage-1 queue and wait out the classify workers (the only producer of the known
|
||||
*and* unknown queues), then close those queues and wait out the enrich and
|
||||
content-triage workers (content triage being the only producer of the text
|
||||
queue), then close the text queue and wait out the language-enrichment workers.
|
||||
(See "Shutdown" below for one cosmetic caveat on SIGTERM.)
|
||||
- **Safe filenames:** client-supplied names are sanitized and prefixed with a
|
||||
server-minted UUID before touching the filesystem (no path traversal).
|
||||
|
||||
### Streaming intake
|
||||
|
||||
The upload endpoint never holds a file in memory. Bytes go socket → spool file in
|
||||
`FS_UPLOAD_CHUNK_BYTES` chunks, so resident memory per in-flight upload is set by
|
||||
the chunk size, not the file size — a 2 GiB upload costs about what a 2 KiB one
|
||||
does. Two pieces make that work, and both are deliberate:
|
||||
|
||||
- **`src/multipart.jl` — an incremental multipart parser.** HTTP.jl's
|
||||
`parse_multipart_form` takes the *complete* body as a byte vector, so using it
|
||||
means every file in the request is in memory at once (and copied again per
|
||||
part). The reader here pulls fixed-size chunks and hands each part's bytes
|
||||
straight to its spool file. Its interface is two calls in a loop —
|
||||
`next_part!` then `write_part_body!` (or `skip_part_body!`) — so the handler
|
||||
keeps ordinary control flow instead of inverting into callbacks. The subtle
|
||||
part is that a boundary delimiter can straddle two chunks, so the buffer always
|
||||
retains the last `length(delimiter)-1` bytes; the test suite parses the same
|
||||
body at chunk sizes from 1 byte upward to put that split at every offset.
|
||||
- **`/upload` bypasses Oxygen's router.** Oxygen's root handler wraps
|
||||
`HTTP.streamhandler`, which does `request.body = read(stream)` *before*
|
||||
dispatching — even for an Oxygen `@stream` route, so no route can stream an
|
||||
upload. `run` therefore passes its own `handler` to `serve`
|
||||
(`root_stream_handler`), which intercepts `POST /upload` at the stream level and
|
||||
delegates everything else to Oxygen unchanged. The trade-off: `/upload` is
|
||||
absent from Oxygen's built-in metrics and docs.
|
||||
|
||||
Streaming also changes what the endpoint can promise. A buffered handler knows up
|
||||
front how many files a request holds; this one discovers them as they arrive. So
|
||||
when the intake queue fills mid-request it does not abandon the connection: it
|
||||
stops spooling (discarding the remaining parts rather than writing files it can't
|
||||
queue), drains the body, and answers `503` with the `accepted` list of whatever
|
||||
got in first. Files already queued stay queued, and the client can retry the rest.
|
||||
|
||||
A client that hangs up mid-upload is treated as routine: the partial spool file is
|
||||
removed (so restart recovery can never pick up a truncated upload as if it were
|
||||
complete) and the event is logged `upload aborted by client`. One cosmetic caveat,
|
||||
like the SIGTERM one below: when a request body is cut short, HTTP.jl's own
|
||||
`closeread` logs an `EOFError` after the handler returns, because the connection
|
||||
promised more bytes via `Content-Length` than arrived. It's harmless noise from
|
||||
inside HTTP.jl — the partial file is already cleaned up and the connection closed.
|
||||
|
||||
### Metadata enrichment (stage 2)
|
||||
|
||||
Files the classifier labels **known** are handed to a second pool that extracts
|
||||
metadata with [`exiftool`](https://exiftool.org/) (`exiftool -json -G -n`) —
|
||||
chosen because no native Julia library comes close to its multi-format coverage.
|
||||
The output is normalized into a small, stable, documented schema and written as a
|
||||
JSON **sidecar** next to the file in `data/done/`, e.g.
|
||||
`data/done/<uuid>-<name>.meta.json`. The original bytes are never modified.
|
||||
|
||||
> **Prerequisite:** `exiftool` must be on `PATH` (e.g. `apt install
|
||||
> libimage-exiftool-perl`). The server **fails fast at startup** if it's missing.
|
||||
|
||||
Sidecar top-level fields (all nullable — present only when available), plus the
|
||||
complete raw `exiftool` object under `raw`:
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `id`, `original_name` | job id and client-supplied name |
|
||||
| `file_type`, `mime_type` | e.g. `PDF` / `application/pdf` |
|
||||
| `file_size` | bytes (authoritative, from intake — not exiftool) |
|
||||
| `created_date`, `modified_date` | content timestamps |
|
||||
| `author` | person (`Author`/`Artist`/`By-line`) |
|
||||
| `created_by` | authoring app/tool (`Producer`/`CreatorTool`/`Creator`/`Software`/…) |
|
||||
| `dimensions` | `{width, height}` for media |
|
||||
| `duration` | seconds, for audio/video |
|
||||
| `page_count` | for documents |
|
||||
| `error` | set on a *degraded* sidecar (see below) |
|
||||
| `raw` | full `exiftool` output |
|
||||
|
||||
Each normalized field is a coalesce over a priority list of exiftool tags
|
||||
(`src/metadata.jl`); extend a field by appending tag names. If extraction fails
|
||||
or `exiftool` times out (`FS_EXIFTOOL_TIMEOUT`, default 30s), the file still
|
||||
completes to `data/done/` with a **degraded sidecar** — `file_size`/`file_type`
|
||||
plus an `error` note — rather than being quarantined, because it's still a wanted
|
||||
known file. Only genuine I/O errors (can't write the sidecar or move the file)
|
||||
send it to `data/failed/`.
|
||||
|
||||
The sidecar is committed **before** the file is moved into `data/done/`, so a
|
||||
file's presence there always implies its sidecar is already present; a crash in
|
||||
between leaves only a harmless orphan sidecar, and recovery re-enriches
|
||||
idempotently.
|
||||
|
||||
### Content triage (stage 3)
|
||||
|
||||
Files the classifier labels **unknown** are handed to a third pool that sorts
|
||||
them into two coarse buckets so downstream tooling can treat them differently:
|
||||
|
||||
- **`data/binary/`** — the file looks like binary data.
|
||||
- **`data/text/`** — the file looks like text.
|
||||
|
||||
The test is a **UTF-8 sniff**: read the first 8000 bytes and call the file text
|
||||
when that window is valid UTF-8 and holds no control bytes outside the text-safe
|
||||
set (tab, newline, CR, and friends, plus ESC for ANSI-colored logs); otherwise
|
||||
binary. It's cheap (no full read) and Unicode-aware — unlike the older NUL-byte
|
||||
or printable-ASCII heuristics, it keeps non-ASCII text (accents, CJK, emoji) in
|
||||
`text/` instead of misfiling it, while binary formats — which rarely form valid
|
||||
UTF-8 near their start — still land in `binary/`. A NUL byte is valid UTF-8 but
|
||||
not a text control byte, so it still reads as binary. A multi-byte character
|
||||
split by the 8000-byte boundary is trimmed before the check so it isn't mistaken
|
||||
for malformed bytes. An empty file is treated as text. `binary/` is terminal on
|
||||
the live path (but is the input the offline **stage-5 discovery** sweeps — see
|
||||
below); `text/` is handed to stage 4 (`src/content.jl`).
|
||||
|
||||
### Language enrichment (stage 4)
|
||||
|
||||
Files that stage 3 sorts as **text** are handed to a fourth pool that identifies
|
||||
their language and writes a `.meta.json` sidecar, mirroring the stage-2
|
||||
known-file enrichment. Two detectors run per file:
|
||||
|
||||
- **natural language** — [`Languages.jl`](https://github.com/JuliaText/Languages.jl)'s
|
||||
`LanguageDetector` (a Julia port of the `whatlang` n-gram model) reads a bounded
|
||||
prefix (up to `LANG_SAMPLE_BYTES`, 64 KiB) and reports the language's English
|
||||
name, ISO 639-3 code, and a confidence in `[0,1]`. Pure Julia, no subprocess.
|
||||
The detector is built once at startup and shared read-only across the pool.
|
||||
- **programming language** — the [`github-linguist`](https://github.com/github-linguist/linguist)
|
||||
CLI recognizes source and markup by extension + content heuristics (e.g.
|
||||
`Python`, `Markdown`). Plain prose reports as `Text` and unrecognized content as
|
||||
`null`; both collapse to *no programming language*.
|
||||
|
||||
The sidecar schema:
|
||||
|
||||
| field | meaning |
|
||||
|---|---|
|
||||
| `id`, `original_name` | from intake |
|
||||
| `file_size` | bytes (authoritative, from intake) |
|
||||
| `content_type` | always `"text"` |
|
||||
| `language` | natural-language English name (e.g. `English`), or `null` |
|
||||
| `language_code` | ISO 639-3 code (e.g. `eng`), or `null` |
|
||||
| `language_confidence` | detector confidence in `[0,1]`, or `null` |
|
||||
| `programming_language` | e.g. `Python`, `Markdown`, or `null` |
|
||||
| `error` | set if natural-language detection produced nothing |
|
||||
|
||||
> **`github-linguist` and the git-repo quirk:** run against a path *inside* a git
|
||||
> repository, linguist reads the file's committed git blob, not the on-disk bytes
|
||||
> — and an untracked file (which everything under `data/` is) has no blob, so it
|
||||
> crashes. Stage 4 sidesteps this by copying each file to a fresh temp dir under
|
||||
> `/tmp` (outside any repo, preserving the name so extension heuristics still
|
||||
> fire) and pointing linguist there.
|
||||
>
|
||||
> Programming-language detection is **best-effort**: if `github-linguist` is
|
||||
> missing (a startup warning, not a fatal error, unlike `exiftool`), fails, or
|
||||
> times out (`FS_LINGUIST_TIMEOUT`, default 30s), `programming_language` is simply
|
||||
> `null` and the file still completes. Natural-language detection failing produces
|
||||
> a **degraded sidecar** (with an `error` note) rather than a quarantine, because
|
||||
> the file is still wanted text.
|
||||
|
||||
Like stage 2, the sidecar is committed **before** the file is moved into
|
||||
`data/text_done/`, so the file's presence there always implies its sidecar is
|
||||
present; recovery re-enriches idempotently (`src/language.jl`).
|
||||
|
||||
### Unknown-format discovery (stage 5, offline)
|
||||
|
||||
The `binary/` sink from stage 3 is the pile of genuinely *unrecognized* files.
|
||||
Stage 5 mines it for **recurring new file formats** by clustering files on their
|
||||
header bytes — a growing catalog of discovered formats, each with a magic-byte
|
||||
signature that can eventually be promoted into the classifier's fast path. Unlike
|
||||
stages 1–4 it is **not on the request hot path**: it is a single-owner *batch*
|
||||
process (the catalog is mutable shared state, the opposite of the stateless
|
||||
classifier), and because promotion is human-gated nothing here is
|
||||
latency-sensitive. The full rationale — and the assumptions we deliberately
|
||||
rejected — live in [`model/DESIGN_clustering.md`](model/DESIGN_clustering.md).
|
||||
|
||||
The model (`src/cluster.jl`, base-Julia, no extra deps) is a Dirichlet-process
|
||||
mixture of **per-position categoricals** over the first 32 header bytes, on a
|
||||
257-symbol alphabet (byte `0–255` plus a `past-EOF` symbol so short fixed-length
|
||||
formats are modeled honestly). Bytes are treated as **categorical, not numeric**
|
||||
— `0x89` and `0x88` are not "close" — so this deliberately does *not* reuse the
|
||||
classifier's `[0,1]` byte scaling. A fixed uniform **background** component
|
||||
absorbs structureless (compressed/encrypted) blobs so they don't mint spurious
|
||||
clusters. A cluster's spiked positions become a libmagic-style signature;
|
||||
clusters with enough members and enough fixed positions self-**nominate** for
|
||||
promotion (a human does the one irreversible step, redefining "known").
|
||||
|
||||
**Status:** both phases are implemented and calibrated. Phase A (offline Gibbs)
|
||||
is the science; phase B (`src/catalog.jl`) is the live catalog: a durable
|
||||
single-owner state that sweeps `binary/`, folds each new file into a cluster with
|
||||
the deterministic CRP-predictive rule, and writes promotion nominations.
|
||||
|
||||
The catalog process is run periodically (cron), single-threaded — it is the
|
||||
**only** writer of the catalog, so it needs no locking:
|
||||
|
||||
```bash
|
||||
julia --project=. bin/cluster_sweep.jl # incremental live sweep of new binary/ files
|
||||
julia --project=. bin/cluster_sweep.jl --compact # offline Gibbs re-cluster (seed / recompact)
|
||||
```
|
||||
|
||||
The catalog is a single durable file (`FS_CLUSTER_CATALOG`, default
|
||||
`data/catalog.json`) committed with the same sidecar-first
|
||||
temp→fsync→rename→fsync-dir discipline as the stage-2 sidecars, so a crash can
|
||||
neither corrupt it nor lose a write. On the **first** run (empty catalog) the
|
||||
sweep auto-promotes to a `--compact` pass to seed clusters; later runs assign
|
||||
incrementally, touching only files they have not seen. A cluster that clears the
|
||||
member/magic thresholds writes a nomination — a hex magic template, member count,
|
||||
and example filenames — into `FS_NOMINATED_DIR` (default `data/nominated/`) for a
|
||||
human to glance at and promote. Under the calibrated `bg_mass > α`, the live
|
||||
sweep never mints single-file clusters; genuinely new formats surface from the
|
||||
periodic `--compact` re-clustering of the background residue, not the live path.
|
||||
|
||||
Calibration is its own offline script (like training — never in the request
|
||||
path), scored against magic-collapsed ground truth (so `docx`≡`zip` and the whole
|
||||
ELF family count as one format each, which is the *correct* answer, not an error):
|
||||
|
||||
```bash
|
||||
julia --project=. bin/cluster_calibrate.jl [training_set_dir] # defaults to ../training_set
|
||||
```
|
||||
|
||||
It grid-tunes the hyperparameters to maximize Adjusted Rand Index against known
|
||||
formats and cross-checks against a model-free NCD (gzip) baseline. On the 700-file
|
||||
training corpus the calibrated defaults (`n=32`, `α=1.0`, `β=0.1`) recover the
|
||||
known formats at **ARI 0.77** (0.885 excluding tar), with `gzip`, `pkzip`
|
||||
(`docx`+`zip` merged), and `jpeg` forming clean, promotable clusters; the NCD
|
||||
baseline agrees. See `DESIGN_clustering.md` §11 for the full results, including the
|
||||
one known limitation (ELF and these tarballs share a long run of header zero-
|
||||
padding and merge — the v2 fix is inverse-entropy position weighting).
|
||||
|
||||
## The queue seam (→ RabbitMQ later)
|
||||
|
||||
The HTTP handler and workers only ever call `enqueue!`, `dequeue!`, and
|
||||
@@ -62,6 +324,11 @@ or worker code changes.
|
||||
# install deps (first time)
|
||||
julia --project=. -e 'using Pkg; Pkg.instantiate()'
|
||||
|
||||
# external tools: exiftool (stage 2, required) and github-linguist (stage 4,
|
||||
# optional — programming-language detection). e.g. on Debian/Ubuntu:
|
||||
# apt install libimage-exiftool-perl
|
||||
# gem install github-linguist
|
||||
|
||||
# start the server; -t sets the number of OS threads available to workers
|
||||
julia --project=. -t auto bin/server.jl
|
||||
```
|
||||
@@ -98,9 +365,12 @@ tell you "PDF", just "this looks like something I was trained on, or not".
|
||||
- **Artifact:** trained weights live in `model/classifier.jld2` (committed), so
|
||||
the server just loads them at startup. Missing/unreadable ⇒ the server fails
|
||||
fast rather than run without classification.
|
||||
- **Effect today:** *annotate-only*. The class is logged
|
||||
(`classification=known|unknown`) but every file still moves to `done/`; the
|
||||
classifier can't misroute real files while it's unproven.
|
||||
- **Effect today:** *active routing*. The class is logged
|
||||
(`classification=known|unknown`) and drives the pipeline split: `:known` files
|
||||
go to `known/` for metadata enrichment (stage 2), `:unknown` files go to
|
||||
`unknown/` for content triage (stage 3). The class chooses the downstream
|
||||
stage; what's still unproven is the model's *accuracy*, not whether the routing
|
||||
path runs.
|
||||
|
||||
The architecture and byte→feature mapping are defined once in `src/model.jl` and
|
||||
shared by the trainer and the server, so they can't drift apart.
|
||||
@@ -130,18 +400,42 @@ init, so the artifact is exactly regenerable from the same inputs.
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---------------------|----------------|------------------------------------------|
|
||||
| `FS_HOST` | `127.0.0.1` | Bind address |
|
||||
| `FS_PORT` | `8080` | Port |
|
||||
| `FS_WORKERS` | `nthreads()` | Number of worker tasks |
|
||||
| `FS_QUEUE_CAPACITY` | `1000` | Max pending jobs before `503` |
|
||||
| `FS_SPOOL_DIR` | `data/spool` | Incoming files (pending) |
|
||||
| `FS_DONE_DIR` | `data/done` | Files after successful processing |
|
||||
| `FS_FAILED_DIR` | `data/failed` | Files whose processing threw |
|
||||
| `FS_MODEL_PATH` | `model/classifier.jld2` | Classifier artifact loaded at startup |
|
||||
| `FS_HOST` | `127.0.0.1` | Bind address |
|
||||
| `FS_PORT` | `8080` | Port |
|
||||
| `FS_WORKERS` | `nthreads()` | Stage-1 (classification) worker tasks |
|
||||
| `FS_QUEUE_CAPACITY` | `1000` | Max pending intake jobs before `503` |
|
||||
| `FS_KNOWN_WORKERS` | `nthreads()` | Stage-2 (enrichment) worker tasks |
|
||||
| `FS_KNOWN_QUEUE_CAPACITY` | `1000` | Max pending enrichment jobs (then backpressure) |
|
||||
| `FS_UNKNOWN_WORKERS` | `nthreads()` | Stage-3 (content triage) worker tasks |
|
||||
| `FS_UNKNOWN_QUEUE_CAPACITY` | `1000` | Max pending triage jobs (then backpressure) |
|
||||
| `FS_TEXT_WORKERS` | `nthreads()` | Stage-4 (language enrichment) worker tasks |
|
||||
| `FS_TEXT_QUEUE_CAPACITY` | `1000` | Max pending language jobs (then backpressure) |
|
||||
| `FS_SPOOL_DIR` | `data/spool` | Incoming files (pending classification) |
|
||||
| `FS_KNOWN_DIR` | `data/known` | Classified-known, awaiting enrichment |
|
||||
| `FS_UNKNOWN_DIR` | `data/unknown` | Classified-unknown, awaiting content triage |
|
||||
| `FS_BINARY_DIR` | `data/binary` | Stage-3 sink: unknown files that look binary |
|
||||
| `FS_TEXT_DIR` | `data/text` | Classified-text, awaiting language enrichment |
|
||||
| `FS_DONE_DIR` | `data/done` | Enriched known files (+ `.meta.json`) |
|
||||
| `FS_TEXT_DONE_DIR` | `data/text_done` | Enriched text files (+ `.meta.json`) |
|
||||
| `FS_FAILED_DIR` | `data/failed` | Files whose processing threw |
|
||||
| `FS_MODEL_PATH` | `model/classifier.jld2` | Classifier artifact loaded at startup |
|
||||
| `FS_UPLOAD_CHUNK_BYTES` | `65536` | Socket read size at intake; bounds intake memory per in-flight upload |
|
||||
| `FS_EXIFTOOL_TIMEOUT` | `30` | Seconds before a stuck exiftool is killed |
|
||||
| `FS_LINGUIST_TIMEOUT` | `30` | Seconds before a stuck github-linguist is killed |
|
||||
| `FS_CLUSTER_DIR` | `data/binary` | Stage-5 input: the unknown/binary pile to sweep |
|
||||
| `FS_CLUSTER_N` | `32` | Header bytes modeled per file |
|
||||
| `FS_CLUSTER_ALPHA` | `1.0` | CRP concentration (propensity to spawn new formats) |
|
||||
| `FS_CLUSTER_PSEUDOCOUNT` | `0.1` | Dirichlet pseudocount β (calibrated) |
|
||||
| `FS_CLUSTER_BG_MASS` | `5.0` | Mass of the uniform background component |
|
||||
| `FS_PROMOTE_MIN_MEMBERS` | `20` | Cluster size threshold for promotion nomination |
|
||||
| `FS_PROMOTE_MIN_MAGIC` | `3` | Required fixed signature positions to nominate |
|
||||
| `FS_CLUSTER_CATALOG` | `data/catalog.json` | Durable stage-5 catalog file (single-owner) |
|
||||
| `FS_NOMINATED_DIR` | `data/nominated` | One JSON per self-nominated cluster (human promote gate) |
|
||||
|
||||
> To get real parallelism, start Julia with enough threads (`-t N`) to match
|
||||
> `FS_WORKERS`. If `FS_WORKERS` exceeds available threads you'll get a warning
|
||||
> and workers will share threads.
|
||||
> To get real parallelism, start Julia with enough threads (`-t N`) to cover all
|
||||
> pools. If `FS_WORKERS + FS_KNOWN_WORKERS + FS_UNKNOWN_WORKERS + FS_TEXT_WORKERS`
|
||||
> exceeds available threads you'll get a warning (non-fatal) and workers will
|
||||
> share threads.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -153,6 +447,9 @@ curl http://127.0.0.1:8080/health
|
||||
# upload one or more files (multipart/form-data)
|
||||
curl -F "a=@report.pdf" -F "b=@data.csv" http://127.0.0.1:8080/upload
|
||||
# 202 {"accepted":[{"id":"<uuid>","name":"report.pdf"}, ...]}
|
||||
|
||||
# per-stage counters
|
||||
curl http://127.0.0.1:8080/stats
|
||||
```
|
||||
|
||||
Each file in a request becomes its own job. Responses:
|
||||
@@ -162,6 +459,386 @@ Each file in a request becomes its own job. Responses:
|
||||
- `503 Service Unavailable` — queue full, retry later
|
||||
- `500 Internal Server Error` — failed to write a file to disk
|
||||
|
||||
### `GET /stats` — per-stage counters
|
||||
|
||||
The pipeline counts its own work (`src/stats.jl`), because nothing outside it
|
||||
can: `known/`, `unknown/` and `text/` are *transient*, so a file can cross one
|
||||
between two directory polls and an external sampler will miss exactly the stages
|
||||
you most want to measure.
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"now": 1785725300.5, "since": 1785725291.9, "uptime_seconds": 8.5,
|
||||
"intake": { "requests": 400, "files": 400, "bytes": 6553600, "rejected": 0 },
|
||||
"stages": [
|
||||
{ "stage": 4, "name": "language", "workers": 16,
|
||||
"queue_depth": 184, "queue_capacity": 1000,
|
||||
"completed": 200, "failed": 0, "bytes": 3276800,
|
||||
"busy_seconds": 170.5, // summed handler time across the pool
|
||||
"blocked_seconds": 0.0, // of that, time parked on a full downstream queue
|
||||
"in_flight": 3 }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Counters are monotonic since startup, Prometheus-style — rates are the reader's
|
||||
job, so a scrape is stateless and two readers can't disturb each other. Take two
|
||||
scrapes Δt apart and subtract:
|
||||
|
||||
```
|
||||
throughput = Δcompleted / Δt
|
||||
utilization = (Δbusy_seconds − Δblocked_seconds) / (Δt × workers)
|
||||
```
|
||||
|
||||
**Utilization is the number that names the bottleneck.** In a pipeline every
|
||||
stage completes the same files, so at steady state they all report near-identical
|
||||
files/s no matter which one is the constraint; what separates them is how hard
|
||||
each pool worked to keep up. The bottleneck sits near 1.0 with its queue backing
|
||||
up while its neighbours idle.
|
||||
|
||||
`blocked_seconds` is what keeps that true. Stages 1 and 3 apply *blocking*
|
||||
backpressure — a full downstream queue means the handler parks and retries rather
|
||||
than dropping the file — and that wait happens inside the handler. Counting it as
|
||||
busy would pin stage 1 at 1.0 whenever stage 2 is the real jam, making every
|
||||
stage upstream of a jam look like the jam. Subtracted out, utilization means
|
||||
"doing its own work", and a high blocked share becomes its own signal: a stage
|
||||
blocked 90% of the time is naming its successor.
|
||||
|
||||
The counters are a handful of atomic adds per file, recorded in `worker_loop` —
|
||||
the one place every stage's work passes through, so a new stage is instrumented
|
||||
the moment it is wired up, and never on the read path.
|
||||
|
||||
## Benchmarking (throughput + memory)
|
||||
|
||||
There are five harnesses. Only the first needs a running server:
|
||||
|
||||
| script | measures | server? |
|
||||
|---|---|---|
|
||||
| `bin/bench.jl` (below) | intake, end-to-end and per-stage throughput; server RSS | **yes** |
|
||||
| [`bin/bench_stage1.jl`](#stage-1-component-benchmark-binbench_stage1jl) | stage 1 taken apart: classify vs. rename vs. enqueue vs. logging | no |
|
||||
| [`bin/bench_stage2.jl`](#stage-2-component-benchmark-binbench_stage2jl) | stage 2 taken apart: exiftool spawn vs. extraction vs. commit | no |
|
||||
| [`bin/bench_model.jl`](#model-microbenchmark-binbench_modeljl) | the classifier alone: inference, feature reads, thread scaling | no |
|
||||
| [`bin/cluster_calibrate.jl`](#unknown-format-discovery-stage-5-offline) | stage-5 clustering quality vs. an NCD baseline | no |
|
||||
|
||||
Running all of them from a clean checkout:
|
||||
|
||||
```bash
|
||||
julia --project=. -e 'using Pkg; Pkg.instantiate()' # once
|
||||
|
||||
# 1. the model, on its own — no server involved
|
||||
julia --project=. -t auto bin/bench_model.jl
|
||||
|
||||
# 1b. stage 1 taken apart — also no server
|
||||
julia --project=. -t auto bin/bench_stage1.jl
|
||||
|
||||
# 1c. stage 2 taken apart — needs a directory of real files, not generated ones
|
||||
julia --project=. -t auto bin/bench_stage2.jl
|
||||
|
||||
# 2. the pipeline. Start the server in one terminal…
|
||||
julia --project=. -t auto bin/server.jl
|
||||
|
||||
# …and drive it from another. Restart the server between memory runs: Julia's
|
||||
# GC returns memory to the OS lazily, so a second run starts inflated.
|
||||
julia --project=. -t auto bin/bench.jl --files 2000 --size 8k --concurrency 32
|
||||
julia --project=. -t auto bin/bench.jl --files 2 --size 1g --concurrency 1 # memory
|
||||
julia --project=. -t auto bin/bench.jl --corpus ../training_set --concurrency 16
|
||||
|
||||
# 3. stage-5 clustering quality (offline, needs a labelled corpus)
|
||||
julia --project=. bin/cluster_calibrate.jl ../training_set
|
||||
```
|
||||
|
||||
The test suite is `julia --project=. -t auto -e 'using Pkg; Pkg.test()'`.
|
||||
|
||||
`bin/bench.jl` must run on the same machine as the server (it reads the sink dirs
|
||||
and `/proc`); otherwise it only speaks HTTP — `/upload`, `/health`, and `/stats`
|
||||
for the per-stage numbers.
|
||||
|
||||
Three properties of this design dictate how it measures:
|
||||
|
||||
- **HTTP latency is not throughput.** `/upload` returns `202` once the bytes are
|
||||
spooled and a reference is enqueued — all four stages run *after* the response,
|
||||
so `ab`/`hey`/`wrk` would only ever measure intake. The bench instead uploads a
|
||||
corpus and polls the terminal sinks (`done/`, `text_done/`, `binary/`,
|
||||
`failed/`) until the count stops moving, and reports both numbers separately:
|
||||
intake rate *and* end-to-end completion rate. It also samples the intermediate
|
||||
stage dirs, so the peak depth of `spool/`/`known/`/`unknown/`/`text/` shows
|
||||
where work piles up.
|
||||
- **End-to-end throughput doesn't name the slow stage.** The four stages run
|
||||
concurrently behind their own queues, so the pipeline's rate *is* the slowest
|
||||
stage's rate and the others are invisible in it. The bench scrapes
|
||||
[`/stats`](#get-stats--per-stage-counters) before and after the run and
|
||||
subtracts, giving each stage its own throughput, mean service time and
|
||||
utilization:
|
||||
|
||||
```
|
||||
PER-STAGE (server counters, delta over the end-to-end window)
|
||||
stage files/s MiB/s svc ms util blocked peak queue failed
|
||||
1 classify 32.52 0.51 38.7 0.08 0.0% 209/1000 0
|
||||
2 enrich 0.24 0.0 541.0 0.01 0.0% 0/1000 0
|
||||
3 triage 32.27 0.5 33.9 0.07 0.0% 83/1000 0
|
||||
4 language 16.26 0.25 852.6 0.87 0.0% 184/1000 0
|
||||
bottleneck stage 4 (language) at 87.0% utilization of 16 worker(s)
|
||||
```
|
||||
|
||||
(400 mixed files, 16 KiB each, concurrency 16. Stage 4 is the constraint —
|
||||
`github-linguist` is a process spawn per file at ~850 ms — and the 200 text
|
||||
files queue up behind it while stages 1 and 3 idle at under 10%.) Read *down*
|
||||
the util column, not the files/s column: each stage sees a different subset of
|
||||
the corpus, so a low rate can just mean little work was routed there.
|
||||
- **Memory should be flat in file size, and the sweep is what proves it.** Both
|
||||
halves of the pipeline are bounded: the workers read bounded prefixes (16+16
|
||||
bytes to classify, 8 KB to sniff, 64 KB to language-detect), and intake streams
|
||||
each upload to disk a chunk at a time. So peak RSS should track *concurrency*,
|
||||
not size. Measured on this machine (16 threads, 64 KiB chunk):
|
||||
|
||||
| upload size | concurrency | RSS growth |
|
||||
|---|---|---|
|
||||
| 256 MiB × 4 | 1 | 14.0 / 20.4 MiB (two runs) |
|
||||
| 1 GiB × 2 | 1 | 22.3 MiB |
|
||||
| 2 GiB × 1 | 1 | 31.3 MiB |
|
||||
| 256 MiB × 4 | 4 | none measurable (peak stayed under the baseline) |
|
||||
|
||||
**Read the shape, not the digits.** Growth is flat across an 8× range of file
|
||||
sizes — 14–31 MiB whether the upload is 256 MiB or 2 GiB — which is the claim
|
||||
that matters: nothing scales with file size, so nothing is buffering. The
|
||||
absolute figures are *not* precise to the megabyte. A freshly started server
|
||||
settles anywhere in an ~860–985 MiB band, so run-to-run variance in the
|
||||
baseline is comparable to the growth being measured; the residual is GC churn
|
||||
from the chunk reads, not retained buffers.
|
||||
|
||||
Two consequences worth knowing before quoting these numbers:
|
||||
|
||||
- **Let the server settle ~30s after startup** before a memory run, or the
|
||||
baseline is sampled mid-fall and the run reports less growth than it caused
|
||||
(or none at all, as the concurrency-4 row above did).
|
||||
- **Linear-in-concurrency is not currently demonstrated.** An earlier
|
||||
measurement put 4 concurrent 256 MiB uploads at 86.9 MiB (21.7 per in-flight
|
||||
upload), but that run predates a fix to how the baseline was sampled — it was
|
||||
read *before* the peak counter was reset, against a different origin — and the
|
||||
re-measurement above cannot reproduce it: the growth sits under the noise
|
||||
floor. Expect concurrency to cost memory; don't trust a specific coefficient
|
||||
without a quieter machine or many more runs.
|
||||
|
||||
Before intake
|
||||
was streamed, the same 256 MiB upload grew RSS by ~700 MiB and 4 concurrent ones
|
||||
pushed a 950 MiB baseline past 2 GiB, so **a `--size` sweep that slopes upward is
|
||||
the regression signal** — it means something has started buffering bodies again.
|
||||
|
||||
Flags: `--files`, `--size` (`8k`/`64m`/`1g`), `--concurrency`, `--kind`
|
||||
(`binary`/`text`/`mixed` — chooses which stages get loaded), `--corpus DIR` (real
|
||||
files, the only way to exercise stage 2's exiftool path), `--pid`, `--no-mem`,
|
||||
`--no-stats`, `--sample-ms`, `--timeout`, `--json PATH`. Full list in the script
|
||||
header. The `--json` output carries the per-stage numbers too, so a sweep can be
|
||||
compared run to run.
|
||||
|
||||
Two caveats the script reports rather than hides: it counts sink *deltas*, so it
|
||||
warns if the pipeline isn't idle at the start (in-flight leftovers would be
|
||||
counted as its own throughput); and because Julia's GC returns memory to the OS
|
||||
lazily, a second run in the same process starts from an inflated baseline — it
|
||||
resets the kernel's peak-RSS counter (`/proc/<pid>/clear_refs`) per run and flags
|
||||
a drifted baseline, but for a clean growth figure restart the server between
|
||||
memory runs.
|
||||
|
||||
### Stage-1 component benchmark (`bin/bench_stage1.jl`)
|
||||
|
||||
`bin/bench.jl` reports stage 1 as one number and `bin/bench_model.jl` takes the
|
||||
*classifier* apart — but stage 1 is more than the model. Per file it also
|
||||
renames the file into its stage directory, pushes a reference onto the
|
||||
downstream queue, and logs. `bin/bench_stage1.jl` times each of those in
|
||||
isolation, then times the real `handle_classify_job` end to end so the parts can
|
||||
be checked against the whole:
|
||||
|
||||
```bash
|
||||
julia --project=. -t auto bin/bench_stage1.jl
|
||||
```
|
||||
|
||||
Measured on this machine (Ryzen 7 2700X, 8 cores/16 threads, Julia 1.12; 2000 ×
|
||||
64 KiB files, minimum of 5 trials):
|
||||
|
||||
| component | per file | share of the handler |
|
||||
|---|---|---|
|
||||
| `classify()` | 10.7 µs | 38% |
|
||||
| ↳ `read_features` | 7.9 µs | 28% |
|
||||
| ↳ `Lux.apply` | 2.3 µs | 8% |
|
||||
| `move_to` (rename) | 11.7 µs | 41% |
|
||||
| `enqueue_blocking!` | 0.12 µs | 0.4% |
|
||||
| per-file logging (disabled `@debug`) | 0.29 µs | 1% |
|
||||
| **`handle_classify_job`** | **28.3 µs** | 100% |
|
||||
|
||||
**This benchmark is why stage 1's per-file log lines are `@debug` rather than
|
||||
`@info`.** As `@info` they cost ~71 µs of the handler's ~118 µs — about 6× the
|
||||
classifier and 6× the rename — and nearly all of it was `ConsoleLogger`
|
||||
*formatting* (~64 µs), not the `FlushLogger`'s per-message flush (~8 µs on top).
|
||||
Demoting them took stage 1 from 8.5k files/s to 35.3k files/s on a single worker,
|
||||
a 4.2× speedup for no algorithmic change. The script still prices a formatted
|
||||
line, so the cost of turning them back on is visible: running the handler under
|
||||
`JULIA_DEBUG=FileServer` measures 133 µs per file, a 4.7× slowdown. That is the
|
||||
trade — per-file tracing is available when you want it, and off by default,
|
||||
with `GET /stats` giving per-file observability that is counted rather than
|
||||
formatted.
|
||||
|
||||
What's left is evenly split between the rename and the classifier, and neither
|
||||
has an easy 2×. Two things worth knowing:
|
||||
|
||||
- **The rename, not the model, is the single largest component** (11.7 µs), and
|
||||
it's a plain `mv` within one filesystem. Inside `classify`, the same pattern
|
||||
holds: 7.9 µs of the 10.7 µs is `read_features` — the `open`, the two reads
|
||||
and the `seek` — against 2.3 µs of actual inference. Stage 1 is now a
|
||||
filesystem-bound stage with a neural network attached, not the reverse.
|
||||
- **Stage 1 now peaks at ~4 workers.** With the logger removed from the hot path
|
||||
the sweep reads 35.0k/s at 1 worker, 66.6k/s at 2, **73.3k/s at 4**, then
|
||||
*falls back* to 60.1k/s at 8 and 53.3k/s at 16 — every worker renaming into the
|
||||
same two directories contends on the same directory inode. That ceiling
|
||||
coincides with the one `bin/bench_model.jl` finds for inference, so ~4 is the
|
||||
number from both directions: raising `FS_WORKERS` past it costs throughput.
|
||||
|
||||
Reported times are the **minimum** over trials. Flags: `--files`, `--reps`,
|
||||
`--trials`, `--size`, `--dir`, `--model`, `--threads`, `--no-threads`,
|
||||
`--json PATH`.
|
||||
|
||||
### Stage-2 component benchmark (`bin/bench_stage2.jl`)
|
||||
|
||||
Stage 2 is the one stage whose cost is dominated by something outside Julia
|
||||
entirely: it forks `exiftool`, a Perl program, once per file. `bin/bench.jl`
|
||||
reports the stage as a single throughput number, which can't distinguish "the
|
||||
extraction is slow" from "the *spawn* is slow" — and those have opposite fixes.
|
||||
`bin/bench_stage2.jl` times each piece in isolation, then times the real
|
||||
`handle_known_job` end to end:
|
||||
|
||||
```bash
|
||||
julia --project=. -t auto bin/bench_stage2.jl
|
||||
```
|
||||
|
||||
Two things make this benchmark different from the stage-1 one:
|
||||
|
||||
- **The corpus must be real files.** exiftool's cost depends on what it finds; a
|
||||
file of random bytes bails out early and understates the stage by ~10×. The
|
||||
default corpus is `data/done` — files that already went through stage 2 on this
|
||||
machine. `--corpus PATH` points it elsewhere.
|
||||
- **It prices the alternatives to one-fork-per-file**, because if the fork
|
||||
dominates then the only fixes are to stop paying it per file. `exiftool
|
||||
(batched Nx)` runs the whole corpus through one process; `exiftool
|
||||
(-stay_open)` keeps one process alive and feeds it one file at a time over a
|
||||
pipe — the shape a streaming pipeline could actually adopt. Both are measured,
|
||||
not assumed.
|
||||
|
||||
Measured on this machine (Ryzen 7 2700X, 8 cores/16 threads, Julia 1.12,
|
||||
exiftool 12.40; 150 real files / 102 MiB, minimum of 2 trials):
|
||||
|
||||
| component | per file | share of the handler |
|
||||
|---|---|---|
|
||||
| `run_exiftool()` | 135.9 ms | 98% |
|
||||
| ↳ bare fork + Perl boot (`exiftool -ver`) | 76.7 ms | 55% |
|
||||
| ↳ `JSON3.read` + tag map | 10 µs | 0.0% |
|
||||
| `normalize_metadata` | 1.7 µs | 0.0% |
|
||||
| `commit_enriched!` (sidecar + fsyncs + rename) | 2.0 ms | 1.5% |
|
||||
| per-file logging (`@info`, flush→file) | 47 µs | 0.0% |
|
||||
| **`handle_known_job`** | **138.3 ms** | 100% |
|
||||
| *alt:* `exiftool -stay_open` | 40.1 ms | 29% |
|
||||
| *alt:* `exiftool` batched 150× | 37.3 ms | 27% |
|
||||
|
||||
**Stage 2 is exiftool and nothing else.** Everything the Julia code does —
|
||||
parsing, normalizing, the durable sidecar-first commit, the log line — sums to
|
||||
about 1.5% of the stage. There is no point optimizing any of it.
|
||||
|
||||
**More than half the stage is interpreter startup, not metadata extraction.**
|
||||
The bare `exiftool -ver` (fork, Perl boot, module loads, read no file) costs
|
||||
76.7 ms against a 135.9 ms full call. Both fork-free alternatives agree on what's
|
||||
left: ~37–40 ms of actual work per file. So a persistent exiftool would cut the
|
||||
stage by ~70%, and `-stay_open` gets there without giving up the one-file-in,
|
||||
one-result-out shape the pipeline needs. That remains the single biggest
|
||||
available win in this stage; it is measured here but not yet implemented.
|
||||
|
||||
**This benchmark is also why `run_with_timeout` no longer polls.** The original
|
||||
watchdog polled with `sleep(0.1)` and then joined the polling task, so every call
|
||||
paid the remainder of an in-flight sleep *after* the child had already exited —
|
||||
~25 ms per file here, and a measured 101 ms on a process that exits instantly.
|
||||
Replacing it with a one-shot `Timer` took the stage from 164.6 ms to 138.3 ms per
|
||||
file (6→8 files/s on one worker) and cost nothing in behavior. Stage 4 shares the
|
||||
wrapper and got the same fix for free.
|
||||
|
||||
Writing the missing tests for that wrapper turned up a second, worse problem:
|
||||
**the timeout was never enforceable.** `wait(proc)` returns only once the
|
||||
captured stdout pipe closes, and any grandchild inherits that pipe — so
|
||||
signalling the child alone left the worker blocked until the whole process tree
|
||||
finished on its own (a `sh -c "trap '' TERM; sleep 30"` child ran the full 30 s
|
||||
against a 1 s timeout, under both the old and new watchdog). The child now runs
|
||||
in its own process group and the timeout signals the group. The trade is that a
|
||||
hard crash of the server orphans an in-flight child rather than taking it down;
|
||||
these children are short-lived and timeout-bounded, which is the cheaper side of
|
||||
it.
|
||||
|
||||
**Stage 2 scales to ~8 workers, then flattens**: 8 files/s at 1 worker, 14 at 2,
|
||||
28 at 4, **49 at 8**, and 49 at 16 — the machine runs out of cores to run Perl
|
||||
on, which is exactly what you'd expect of a stage that is ~100% subprocess. Note
|
||||
that the sweep pulls from a shared counter rather than splitting the corpus into
|
||||
contiguous slices: per-file exiftool time spans two orders of magnitude on a real
|
||||
corpus (one 2.1 s archive among 48 files), and a static split reports a scaling
|
||||
ceiling that is really just load imbalance.
|
||||
|
||||
One caveat the numbers raise but don't answer: **`fsync_dir` measures 1.75 µs**,
|
||||
which is far too fast to be a real disk flush. The durability that
|
||||
`commit_enriched!` is written for may not survive power loss on this filesystem,
|
||||
even though the code is correct. That's a correctness question, not a speed one,
|
||||
and it is not yet resolved.
|
||||
|
||||
Reported times are the **minimum** over trials. Flags: `--files`, `--reps`,
|
||||
`--trials`, `--corpus`, `--dir`, `--timeout`, `--threads`, `--no-threads`,
|
||||
`--no-stay-open`, `--json PATH`.
|
||||
|
||||
### Model microbenchmark (`bin/bench_model.jl`)
|
||||
|
||||
`bin/bench.jl` reports stage 1 as a single number — the wall time of
|
||||
`handle_classify_job`, which is a feature read, an inference, a rename, a
|
||||
(disabled) debug line, and whatever contention the other three pools create.
|
||||
That's the right number for capacity planning and the wrong one for "is the
|
||||
model slow?".
|
||||
`bin/bench_model.jl` answers that separately, with no server, queue, or HTTP
|
||||
involved:
|
||||
|
||||
```bash
|
||||
julia --project=. -t auto bin/bench_model.jl
|
||||
```
|
||||
|
||||
Measured on this machine (Ryzen 7 2700X, 8 cores/16 threads, Julia 1.12):
|
||||
|
||||
| what | per file | notes |
|
||||
|---|---|---|
|
||||
| `Lux.apply`, batch 1 | **2.3 µs** | 768 B allocated per call |
|
||||
| `read_features` | **4.2–6.0 µs** | flat across a 262,144× size range (1 KiB → 256 MiB) |
|
||||
| `classify()` | **8.4 µs** | 73% feature read, 27% inference |
|
||||
|
||||
So the model is **not** the pipeline's problem, by three orders of magnitude: the
|
||||
same run measured stage 1 at 38.7 ms per file, ~4,500× the 8.4 µs `classify()`
|
||||
costs. Whatever stage 1 spends its time on, it isn't the network. (That 38.7 ms
|
||||
predates the `@debug` demotion above and is a whole-pipeline figure — it includes
|
||||
time the stage-1 worker spends *blocked* on a full downstream queue, which is why
|
||||
it is three orders of magnitude above the 28.3 µs the handler costs in
|
||||
isolation. For the uncontended split, see
|
||||
[the stage-1 decomposition](#stage-1-component-benchmark-binbench_stage1jl).)
|
||||
|
||||
Two findings worth acting on if stage 1 ever *does* become the constraint:
|
||||
|
||||
- **Batching would buy ~13×.** A 32×1 matmul wastes most of a BLAS call:
|
||||
batch 64 costs 280 ns/file and batch 512 costs 179 ns/file, against 2.34 µs
|
||||
one at a time. The pipeline classifies strictly one file per job today, so it
|
||||
pays the worst row in that table.
|
||||
- **Inference does not scale past ~4 threads.** Concurrent `Lux.apply` on the
|
||||
shared read-only `Classifier` peaks around 1.2M files/s at 4 tasks and then
|
||||
*falls back* to single-thread throughput at 16. The script runs a pure-compute
|
||||
control kernel through the same sweep to place the blame: the control reaches
|
||||
14.3× at 16 tasks (90% efficiency) on the same box, so the machine
|
||||
parallelizes and `Lux.apply` doesn't. GC is only ~1% of it, so allocation
|
||||
pressure isn't the explanation either — the cause is inside Lux/BLAS and is
|
||||
not diagnosed here. The practical consequence: raising `FS_WORKERS` past ~4
|
||||
adds no classification throughput.
|
||||
|
||||
Reported times are the **minimum** over trials — for a microbenchmark the floor
|
||||
is the signal and everything above it is scheduler and GC noise — and every timed
|
||||
loop stores its result in a sink so a pure call can't be hoisted out of the loop.
|
||||
Flags: `--model`, `--reps`, `--trials`, `--batches`, `--sizes`, `--no-threads`,
|
||||
`--json PATH`.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
@@ -170,14 +847,27 @@ src/
|
||||
config.jl Config struct + env parsing
|
||||
job.jl Job (the queue reference)
|
||||
queue.jl JobQueue seam + in-process ChannelQueue
|
||||
spool.jl filename sanitizing, spool/move, startup recovery
|
||||
stats.jl per-stage counters behind GET /stats (throughput, utilization)
|
||||
multipart.jl streaming multipart/form-data reader (intake never buffers a file)
|
||||
spool.jl filename sanitizing, streaming spool/move, startup recovery
|
||||
model.jl NN architecture + byte→feature mapping (shared with trainer)
|
||||
classify.jl load artifact + classify a file at inference time
|
||||
worker.jl worker loop + per-job processing (classify + move)
|
||||
server.jl HTTP routes/handlers
|
||||
metadata.jl exiftool extraction + normalized sidecar (stage 2)
|
||||
content.jl binary-vs-text sniff for unknown files (stage 3)
|
||||
language.jl natural + programming language enrichment for text (stage 4)
|
||||
cluster.jl header-byte clustering model + Gibbs + scoring core (stage 5, science)
|
||||
catalog.jl durable single-owner format catalog + sweep + nominations (stage 5, phase B)
|
||||
worker.jl parametrized worker loop + classify/enrich/triage/language handlers
|
||||
server.jl HTTP routes + the streaming /upload handler
|
||||
bin/
|
||||
server.jl entry point
|
||||
train.jl offline training script → model/classifier.jld2
|
||||
server.jl entry point
|
||||
bench.jl throughput + memory harness against a running server
|
||||
bench_model.jl classifier microbenchmark (inference, feature reads, scaling)
|
||||
bench_stage1.jl stage-1 decomposition (classify vs. rename vs. enqueue vs. logging)
|
||||
train.jl offline training script → model/classifier.jld2
|
||||
cluster_calibrate.jl offline stage-5 hyperparameter calibration + NCD baseline
|
||||
cluster_sweep.jl stage-5 phase-B runner: sweep binary/, update catalog, write nominations
|
||||
model/
|
||||
classifier.jld2 committed trained weights (loaded at startup)
|
||||
classifier.jld2 committed trained weights (loaded at startup)
|
||||
DESIGN_clustering.md stage-5 design rationale + calibration results
|
||||
```
|
||||
|
||||
783
bin/bench.jl
Executable file
783
bin/bench.jl
Executable file
@@ -0,0 +1,783 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# bench.jl — measure end-to-end throughput and server memory for a running FileServer.
|
||||
#
|
||||
# Two things make this pipeline awkward to benchmark with off-the-shelf tools
|
||||
# (ab/hey/wrk), and both shape what this script does:
|
||||
#
|
||||
# 1. HTTP latency is not throughput. /upload returns 202 as soon as the bytes
|
||||
# are spooled and a reference is enqueued — all four stages run afterwards.
|
||||
# So real throughput is the *arrival rate at the terminal sinks*
|
||||
# (done/, text_done/, binary/, failed/), not the response rate. We upload a
|
||||
# corpus, then poll the sinks until the file count stops moving.
|
||||
#
|
||||
# 2. End-to-end throughput doesn't name the slow stage. The four stages run
|
||||
# concurrently behind their own queues, so the pipeline's rate is the
|
||||
# slowest stage's rate and the others are invisible. Directory polling can't
|
||||
# recover them either — known/, unknown/ and text/ are transient, and a file
|
||||
# can cross one between two samples. So the server keeps per-stage counters
|
||||
# (src/stats.jl) and we scrape GET /stats before and after: the deltas give
|
||||
# each stage's throughput, mean service time, and worker utilization, and
|
||||
# utilization is what actually names the bottleneck (see `stage_report`).
|
||||
#
|
||||
# 3. Memory should be flat in file size, and that claim needs checking on two
|
||||
# axes. The workers read bounded prefixes (16+16 bytes to classify, 8 KB to
|
||||
# sniff, 64 KB to language-detect), and intake streams each upload from the
|
||||
# socket to the spool file a chunk at a time (FS_UPLOAD_CHUNK_BYTES, see
|
||||
# src/multipart.jl). So peak RSS should track *concurrency*, not file size:
|
||||
# sweeping --size at fixed --concurrency should be a flat line, and that is
|
||||
# the regression this measures. We sample the server's RSS throughout and
|
||||
# report the high-water mark.
|
||||
#
|
||||
# (Before intake was streamed it buffered each upload whole, several times
|
||||
# over, and a 256 MiB upload grew RSS by ~700 MiB. If a --size sweep ever
|
||||
# slopes upward again, something has started buffering.)
|
||||
#
|
||||
# The harness talks to the server only over HTTP (/upload, /health, /stats) and
|
||||
# reads the on-disk sink layout; it must run on the same machine (sink dirs and
|
||||
# /proc). If /stats is missing — an older build — everything else still works and
|
||||
# the per-stage section is skipped.
|
||||
#
|
||||
# Usage:
|
||||
# julia --project=. -t auto bin/bench.jl [options]
|
||||
#
|
||||
# --url URL server base URL (default: $FS_URL or http://127.0.0.1:8080)
|
||||
# --files N number of files to upload (default: 200)
|
||||
# --size S size of each generated file, e.g. 4k, 512k, 8m, 1g (default: 64k)
|
||||
# --concurrency J uploads in flight at once (default: 8)
|
||||
# --kind K binary | text | mixed — what to generate (default: binary)
|
||||
# --corpus DIR upload an existing directory instead of generating
|
||||
# (the only way to exercise stage 2: point it at real known files)
|
||||
# --keep-corpus don't delete the generated corpus on exit
|
||||
# --pid PID server pid for memory sampling (default: autodetect)
|
||||
# --no-mem skip memory sampling entirely
|
||||
# --no-stats skip the per-stage /stats scrape and its table
|
||||
# --sample-ms MS sink/RSS sampling interval (default: 200)
|
||||
# --timeout SEC give up after this long with no drain progress (default: 120)
|
||||
# --json PATH also write the results as JSON
|
||||
# --force skip the corpus-size safety check
|
||||
#
|
||||
# Examples:
|
||||
# # throughput: many small files, high concurrency
|
||||
# julia --project=. -t auto bin/bench.jl --files 2000 --size 8k --concurrency 32
|
||||
#
|
||||
# # memory: flat in file size? sweep --size with concurrency pinned
|
||||
# julia --project=. -t auto bin/bench.jl --files 4 --size 256m --concurrency 1
|
||||
# julia --project=. -t auto bin/bench.jl --files 2 --size 1g --concurrency 1
|
||||
# julia --project=. -t auto bin/bench.jl --files 8 --size 1g --concurrency 8
|
||||
#
|
||||
# # stage 2 (exiftool) with real known files
|
||||
# julia --project=. -t auto bin/bench.jl --corpus ../training_set --concurrency 16
|
||||
|
||||
using HTTP
|
||||
using JSON3
|
||||
using Random
|
||||
|
||||
# ---------------------------------------------------------------- option parsing
|
||||
|
||||
const DEFAULTS = Dict{String,Any}(
|
||||
"url" => get(ENV, "FS_URL", "http://127.0.0.1:8080"),
|
||||
"files" => 200,
|
||||
"size" => 64 * 1024,
|
||||
"concurrency" => 8,
|
||||
"kind" => "binary",
|
||||
"corpus" => nothing,
|
||||
"keep-corpus" => false,
|
||||
"pid" => nothing,
|
||||
"no-mem" => false,
|
||||
"no-stats" => false,
|
||||
"sample-ms" => 200,
|
||||
"timeout" => 120,
|
||||
"json" => nothing,
|
||||
"force" => false,
|
||||
)
|
||||
|
||||
const FLAGS = ("keep-corpus", "no-mem", "no-stats", "force")
|
||||
|
||||
# Approximate RSS of a freshly started server (Lux + the loaded classifier + the
|
||||
# language detector, measured on Julia 1.12 / -t auto). Only used to notice that
|
||||
# a baseline is inflated by a previous run's un-returned GC memory, so a rough
|
||||
# figure is enough.
|
||||
const FRESH_RSS_HINT = 950 * 1024^2
|
||||
|
||||
"Parse `4k`/`8M`/`1g`/`4096` into a byte count."
|
||||
function parse_size(s::AbstractString)::Int
|
||||
m = match(r"^(\d+(?:\.\d+)?)\s*([kKmMgG]?)[bB]?$", strip(s))
|
||||
m === nothing && error("bad --size: $s (expected e.g. 512, 64k, 8m, 1g)")
|
||||
mult = Dict('k' => 1024, 'm' => 1024^2, 'g' => 1024^3)
|
||||
scale = isempty(m[2]) ? 1 : mult[lowercase(m[2])[1]]
|
||||
return round(Int, parse(Float64, m[1]) * scale)
|
||||
end
|
||||
|
||||
function parse_args(argv)::Dict{String,Any}
|
||||
opts = copy(DEFAULTS)
|
||||
i = 1
|
||||
while i <= length(argv)
|
||||
a = argv[i]
|
||||
startswith(a, "--") || error("unexpected argument: $a (see the header of $(PROGRAM_FILE))")
|
||||
key = a[3:end]
|
||||
haskey(opts, key) || error("unknown option: $a")
|
||||
if key in FLAGS
|
||||
opts[key] = true
|
||||
i += 1
|
||||
continue
|
||||
end
|
||||
i + 1 <= length(argv) || error("option --$key needs a value")
|
||||
val = argv[i+1]
|
||||
opts[key] = key == "size" ? parse_size(val) :
|
||||
key in ("files", "concurrency", "sample-ms") ? parse(Int, val) :
|
||||
key == "timeout" ? parse(Float64, val) :
|
||||
key == "pid" ? parse(Int, val) :
|
||||
val
|
||||
i += 2
|
||||
end
|
||||
opts["kind"] in ("binary", "text", "mixed") ||
|
||||
error("--kind must be binary, text or mixed (got $(opts["kind"]))")
|
||||
opts["files"] >= 1 || error("--files must be >= 1")
|
||||
opts["concurrency"] >= 1 || error("--concurrency must be >= 1")
|
||||
return opts
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- dirs
|
||||
#
|
||||
# Resolved from the same environment variables src/config.jl reads, so a server
|
||||
# started with custom dirs is benchmarked correctly. Kept as a standalone table
|
||||
# rather than `using FileServer` so the harness doesn't pay to load Lux.
|
||||
|
||||
sinkdirs() = (
|
||||
done = get(ENV, "FS_DONE_DIR", "data/done"),
|
||||
text_done = get(ENV, "FS_TEXT_DONE_DIR", "data/text_done"),
|
||||
binary = get(ENV, "FS_BINARY_DIR", "data/binary"),
|
||||
failed = get(ENV, "FS_FAILED_DIR", "data/failed"),
|
||||
)
|
||||
|
||||
stagedirs() = (
|
||||
spool = get(ENV, "FS_SPOOL_DIR", "data/spool"),
|
||||
known = get(ENV, "FS_KNOWN_DIR", "data/known"),
|
||||
unknown = get(ENV, "FS_UNKNOWN_DIR", "data/unknown"),
|
||||
text = get(ENV, "FS_TEXT_DIR", "data/text"),
|
||||
)
|
||||
|
||||
"Count work items in `dir`, ignoring the .meta.json sidecars stages 2/4 write."
|
||||
function count_files(dir::AbstractString)::Int
|
||||
isdir(dir) || return 0
|
||||
n = 0
|
||||
for name in readdir(dir)
|
||||
endswith(name, ".meta.json") && continue
|
||||
isfile(joinpath(dir, name)) && (n += 1)
|
||||
end
|
||||
return n
|
||||
end
|
||||
|
||||
counts(dirs) = NamedTuple{keys(dirs)}(map(count_files, values(dirs)))
|
||||
total(c) = sum(values(c))
|
||||
deltas(now_, base) = NamedTuple{keys(now_)}(map(-, values(now_), values(base)))
|
||||
|
||||
# ----------------------------------------------------------------------- memory
|
||||
|
||||
"Read (VmRSS, VmHWM) in bytes for `pid`, or `nothing` if unreadable."
|
||||
function read_rss(pid::Int)
|
||||
rss = hwm = nothing
|
||||
try
|
||||
for line in eachline("/proc/$pid/status")
|
||||
if startswith(line, "VmRSS:")
|
||||
rss = parse(Int, split(line)[2]) * 1024
|
||||
elseif startswith(line, "VmHWM:")
|
||||
hwm = parse(Int, split(line)[2]) * 1024
|
||||
end
|
||||
end
|
||||
catch
|
||||
return nothing
|
||||
end
|
||||
return (rss === nothing || hwm === nothing) ? nothing : (rss, hwm)
|
||||
end
|
||||
|
||||
"""
|
||||
Find the running server process, or `nothing`.
|
||||
|
||||
`pgrep -f` matches against the whole command line, which catches more than the
|
||||
server: any shell launched with the command in its own argv (`sh -c 'julia …
|
||||
bin/server.jl > log'`, a `setsid`/`nohup` wrapper, even the terminal running the
|
||||
benchmark) matches the same pattern. Sampling one of those reports a few MiB of
|
||||
shell as the server's memory — a wrong answer that looks plausible, which is the
|
||||
worst kind.
|
||||
|
||||
So candidates are filtered by what each process *is* (`/proc/<pid>/comm`, the
|
||||
executable name) rather than by what its arguments say. No pattern over argv can
|
||||
make that distinction.
|
||||
"""
|
||||
function detect_pid()
|
||||
out = try
|
||||
readchomp(`pgrep -f "bin/server.jl"`)
|
||||
catch
|
||||
return nothing
|
||||
end
|
||||
candidates = parse.(Int, split(out))
|
||||
pids = filter(candidates) do pid
|
||||
comm = try
|
||||
readchomp("/proc/$pid/comm")
|
||||
catch
|
||||
return false # exited between pgrep and here
|
||||
end
|
||||
startswith(comm, "julia")
|
||||
end
|
||||
isempty(pids) && return nothing
|
||||
length(pids) > 1 && @warn "multiple server processes matched; sampling the first" pids
|
||||
return first(pids)
|
||||
end
|
||||
|
||||
"""
|
||||
Reset the kernel's peak-RSS counter so VmHWM reflects only this run.
|
||||
|
||||
Without it, VmHWM carries the high-water mark from startup (model load) or from
|
||||
an earlier benchmark, which would silently dominate a small run's result.
|
||||
Requires the server to run as the same user; on failure we say so and fall back
|
||||
to sampled VmRSS, which can miss a spike between samples.
|
||||
"""
|
||||
function reset_peak_rss(pid::Int)::Bool
|
||||
try
|
||||
write("/proc/$pid/clear_refs", "5")
|
||||
return true
|
||||
catch
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------- per-stage counters
|
||||
#
|
||||
# The server exposes monotonic counters at GET /stats (src/stats.jl). Rates are
|
||||
# ours to compute: scrape once before the run and once after, subtract, divide by
|
||||
# the elapsed *server* clock so a slow scrape doesn't distort the window.
|
||||
|
||||
"Fetch and parse GET /stats, or `nothing` if the server doesn't serve it."
|
||||
function scrape_stats(url::String)
|
||||
try
|
||||
resp = HTTP.get(string(rstrip(url, '/'), "/stats");
|
||||
status_exception = false, retry = false, readtimeout = 5)
|
||||
resp.status == 200 || return nothing
|
||||
return JSON3.read(String(resp.body))
|
||||
catch
|
||||
return nothing
|
||||
end
|
||||
end
|
||||
|
||||
"One stage's activity between two scrapes."
|
||||
struct StageDelta
|
||||
stage::Int
|
||||
name::String
|
||||
workers::Int
|
||||
completed::Int
|
||||
failed::Int
|
||||
bytes::Int
|
||||
busy::Float64 # summed handler seconds across all workers in the pool
|
||||
blocked::Float64 # of `busy`, seconds parked on a full downstream queue
|
||||
peak_depth::Int # deepest its queue got, from the sampler
|
||||
capacity::Int
|
||||
end
|
||||
|
||||
files_per_sec(d::StageDelta, window) = d.completed / max(window, 1e-9)
|
||||
mib_per_sec(d::StageDelta, window) = d.bytes / max(window, 1e-9) / 1024^2
|
||||
"Mean wall time one file spends in one worker of this stage."
|
||||
service_ms(d::StageDelta) = d.completed == 0 ? NaN :
|
||||
(d.busy - d.blocked) / d.completed * 1000
|
||||
"""
|
||||
Fraction of the pool's capacity spent doing this stage's own work.
|
||||
|
||||
Blocked time is subtracted first: a stage parked on a full downstream queue is
|
||||
waiting, not working, and leaving it in would light up every stage upstream of a
|
||||
jam as though each were the jam.
|
||||
"""
|
||||
utilization(d::StageDelta, window) =
|
||||
(d.busy - d.blocked) / max(window * d.workers, 1e-9)
|
||||
blocked_share(d::StageDelta) = d.busy <= 0 ? 0.0 : d.blocked / d.busy
|
||||
|
||||
"Subtract two scrapes into per-stage deltas, folding in sampled peak depths."
|
||||
function stage_deltas(before, after, peak_depth::Dict{Int,Int})
|
||||
out = StageDelta[]
|
||||
for (b, a) in zip(before.stages, after.stages)
|
||||
push!(out, StageDelta(a.stage, String(a.name), a.workers,
|
||||
a.completed - b.completed,
|
||||
a.failed - b.failed,
|
||||
a.bytes - b.bytes,
|
||||
a.busy_seconds - b.busy_seconds,
|
||||
a.blocked_seconds - b.blocked_seconds,
|
||||
get(peak_depth, Int(a.stage), 0),
|
||||
a.queue_capacity))
|
||||
end
|
||||
return out
|
||||
end
|
||||
|
||||
"JSON has no NaN. A stage that completed nothing has no service time, and `null`
|
||||
is the honest way to say that — writing NaN just makes JSON3 throw."
|
||||
json_num(x::Real) = isfinite(x) ? x : nothing
|
||||
|
||||
pad(s, n) = rpad(string(s), n)
|
||||
lpad_(s, n) = lpad(string(s), n)
|
||||
|
||||
"""
|
||||
Print the per-stage table and say which stage is the bottleneck.
|
||||
|
||||
The verdict reads utilization, not throughput: in a pipeline every stage
|
||||
completes the same files, so at steady state they all report nearly the same
|
||||
files/s regardless of which one is the constraint. What separates them is how
|
||||
hard each pool had to work to keep up — the bottleneck is pinned near 1.0 while
|
||||
its neighbours idle.
|
||||
"""
|
||||
function stage_report(deltas::Vector{StageDelta}, window::Float64)
|
||||
println("PER-STAGE (server counters, delta over the end-to-end window)")
|
||||
println(" stage files/s MiB/s svc ms util blocked peak queue failed")
|
||||
for d in deltas
|
||||
svc = service_ms(d)
|
||||
println(" $(d.stage) $(pad(d.name, 10)) " *
|
||||
lpad_(fmt(files_per_sec(d, window)), 8) * " " *
|
||||
lpad_(fmt(mib_per_sec(d, window)), 8) * " " *
|
||||
lpad_(isnan(svc) ? "—" : fmt(svc, 1), 8) * " " *
|
||||
lpad_(fmt(utilization(d, window)), 6) * " " *
|
||||
lpad_(fmt(blocked_share(d) * 100, 0) * "%", 7) * " " *
|
||||
lpad_("$(d.peak_depth)/$(d.capacity)", 11) * " " *
|
||||
lpad_(d.failed, 7))
|
||||
end
|
||||
|
||||
# Per-stage files/s are not comparable across rows and saying so costs one
|
||||
# line: the stages process different subsets (stage 2 only known files,
|
||||
# stage 4 only text), so a low rate can mean "little work arrived here"
|
||||
# rather than "slow". Utilization is the column that compares.
|
||||
println(" (files/s counts only files routed to that stage; svc is per-file wall time in")
|
||||
println(" one worker; util = (busy − blocked) / (window × workers))")
|
||||
|
||||
worked = filter(d -> d.completed > 0, deltas)
|
||||
if isempty(worked)
|
||||
println(" (no stage completed a file in this window)")
|
||||
return nothing
|
||||
end
|
||||
top = argmax(d -> utilization(d, window), worked)
|
||||
println(" bottleneck stage $(top.stage) ($(top.name)) at " *
|
||||
"$(fmt(utilization(top, window) * 100, 0))% utilization of " *
|
||||
"$(top.workers) worker(s)")
|
||||
if utilization(top, window) < 0.5
|
||||
println(" — but no stage is near saturated: the pipeline is " *
|
||||
"waiting on intake,\n not on itself. Raise --concurrency " *
|
||||
"or --files to load it properly.")
|
||||
end
|
||||
for d in worked
|
||||
blocked_share(d) > 0.25 && println(" ! stage $(d.stage) ($(d.name)) spent " *
|
||||
"$(fmt(blocked_share(d) * 100, 0))% of its time parked on a full downstream " *
|
||||
"queue —\n it is being held up by the stage after it, not doing that work itself.")
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------ corpus
|
||||
|
||||
const WORDS = split("the quick brown fox jumps over a lazy dog while parsing " *
|
||||
"headers and spooling bytes onto disk for later enrichment " *
|
||||
"because throughput matters more than latency here")
|
||||
|
||||
"Write one file of exactly `size` bytes, in bounded chunks so the generator
|
||||
itself never holds a whole 1 GB file in memory."
|
||||
function write_file(path::AbstractString, size::Int, kind::Symbol, rng)
|
||||
chunk = 1024 * 1024
|
||||
open(path, "w") do io
|
||||
remaining = size
|
||||
while remaining > 0
|
||||
n = min(chunk, remaining)
|
||||
if kind === :binary
|
||||
write(io, rand(rng, UInt8, n))
|
||||
else
|
||||
buf = IOBuffer()
|
||||
while buf.size < n
|
||||
print(buf, rand(rng, WORDS), rand(rng) < 0.06 ? ".\n" : " ")
|
||||
end
|
||||
write(io, take!(buf)[1:n])
|
||||
end
|
||||
remaining -= n
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"Generate the corpus and return (dir, paths, total_bytes)."
|
||||
function make_corpus(opts)
|
||||
n, size, kind = opts["files"], opts["size"], opts["kind"]
|
||||
totalbytes = n * size
|
||||
if totalbytes > 16 * 1024^3 && !opts["force"]
|
||||
error("corpus would be $(human(totalbytes)) on disk; pass --force if that's intended")
|
||||
end
|
||||
dir = mktempdir(; prefix = "fsbench-")
|
||||
rng = MersenneTwister(1234)
|
||||
paths = String[]
|
||||
for i in 1:n
|
||||
k = kind == "mixed" ? (isodd(i) ? :binary : :text) :
|
||||
kind == "text" ? :text : :binary
|
||||
ext = k === :text ? "txt" : "bin"
|
||||
path = joinpath(dir, "bench-$(lpad(i, 6, '0')).$ext")
|
||||
write_file(path, size, k, rng)
|
||||
push!(paths, path)
|
||||
end
|
||||
return dir, paths, totalbytes
|
||||
end
|
||||
|
||||
function existing_corpus(dir::AbstractString)
|
||||
isdir(dir) || error("--corpus is not a directory: $dir")
|
||||
paths = sort(filter(isfile, readdir(dir; join = true)))
|
||||
isempty(paths) && error("--corpus directory is empty: $dir")
|
||||
return paths, sum(filesize, paths)
|
||||
end
|
||||
|
||||
# ----------------------------------------------------------------------- upload
|
||||
|
||||
struct Upload
|
||||
status::Int # HTTP status, or 0 if the request threw
|
||||
accepted::Int # jobs the server actually queued (from the 202/503 body)
|
||||
seconds::Float64
|
||||
end
|
||||
|
||||
"POST one file as multipart/form-data and report what the server accepted."
|
||||
function upload_one(url::String, path::String)::Upload
|
||||
t0 = time()
|
||||
try
|
||||
form = HTTP.Form(["file" => HTTP.Multipart(basename(path), open(path, "r"),
|
||||
"application/octet-stream")])
|
||||
resp = HTTP.post(url, [], form; status_exception = false, retry = false)
|
||||
# 202 and 503 both carry an `accepted` array: a partially-accepted batch
|
||||
# still queued those jobs, and they will show up in the sinks.
|
||||
acc = try
|
||||
length(JSON3.read(String(resp.body)).accepted)
|
||||
catch
|
||||
resp.status == 202 ? 1 : 0
|
||||
end
|
||||
return Upload(resp.status, acc, time() - t0)
|
||||
catch e
|
||||
e isa InterruptException && rethrow()
|
||||
@warn "upload failed" file = basename(path) exception = e
|
||||
return Upload(0, 0, time() - t0)
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
Upload every path, at most `concurrency` in flight.
|
||||
|
||||
A bounded set of worker tasks pulling from a shared index keeps exactly
|
||||
`concurrency` requests in flight for the whole run — unlike batching, where each
|
||||
batch stalls on its slowest (largest) file and the real concurrency sags.
|
||||
"""
|
||||
function upload_all(url::String, paths::Vector{String}, concurrency::Int)
|
||||
endpoint = string(rstrip(url, '/'), "/upload")
|
||||
results = Vector{Upload}(undef, length(paths))
|
||||
next = Threads.Atomic{Int}(1)
|
||||
@sync for _ in 1:min(concurrency, length(paths))
|
||||
Threads.@spawn while true
|
||||
i = Threads.atomic_add!(next, 1)
|
||||
i > length(paths) && break
|
||||
results[i] = upload_one(endpoint, paths[i])
|
||||
end
|
||||
end
|
||||
return results
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- formatting
|
||||
|
||||
function human(bytes::Real)
|
||||
b = Float64(bytes)
|
||||
for unit in ("B", "KiB", "MiB", "GiB", "TiB")
|
||||
(abs(b) < 1024 || unit == "TiB") && return "$(round(b; digits = 2)) $unit"
|
||||
b /= 1024
|
||||
end
|
||||
end
|
||||
|
||||
fmt(x::Real, digits::Int = 2) = string(round(Float64(x); digits = digits))
|
||||
|
||||
function percentile(sorted::Vector{Float64}, p::Float64)
|
||||
isempty(sorted) && return NaN
|
||||
idx = clamp(ceil(Int, p * length(sorted)), 1, length(sorted))
|
||||
return sorted[idx]
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- main
|
||||
|
||||
function main(argv)
|
||||
opts = parse_args(argv)
|
||||
url = string(opts["url"])
|
||||
|
||||
# Fail fast and clearly if there's no server, rather than reporting a run of zeros.
|
||||
try
|
||||
HTTP.get(string(rstrip(url, '/'), "/health"); retry = false, readtimeout = 5)
|
||||
catch e
|
||||
println(stderr, "cannot reach $url/health — is the server running?")
|
||||
println(stderr, " start it with: julia --project=. -t auto bin/server.jl")
|
||||
return 1
|
||||
end
|
||||
|
||||
sinks, stages = sinkdirs(), stagedirs()
|
||||
|
||||
# Leftovers mid-pipeline would land in the sinks during our window and be
|
||||
# counted as our throughput, so say so up front rather than quietly skewing.
|
||||
pending = total(counts(stages))
|
||||
pending > 0 && @warn "pipeline is not idle: $pending file(s) in the stage dirs; " *
|
||||
"throughput will include their completions"
|
||||
|
||||
# --- corpus
|
||||
generated = opts["corpus"] === nothing
|
||||
corpusdir, paths, corpusbytes = if generated
|
||||
print("generating corpus: $(opts["files"]) × $(human(opts["size"])) ($(opts["kind"]))… ")
|
||||
t = time()
|
||||
d, p, b = make_corpus(opts)
|
||||
println("done in $(fmt(time() - t))s → $d")
|
||||
d, p, b
|
||||
else
|
||||
p, b = existing_corpus(String(opts["corpus"]))
|
||||
println("using corpus: $(length(p)) file(s), $(human(b)) from $(opts["corpus"])")
|
||||
String(opts["corpus"]), p, b
|
||||
end
|
||||
|
||||
try
|
||||
pid = opts["no-mem"] ? nothing : something(opts["pid"], detect_pid(), Some(nothing))
|
||||
if pid === nothing && !opts["no-mem"]
|
||||
@warn "could not find the server process; skipping memory (pass --pid PID)"
|
||||
end
|
||||
|
||||
baseline_rss = nothing
|
||||
peak_reset = false
|
||||
if pid !== nothing
|
||||
r = read_rss(pid)
|
||||
r === nothing && (@warn "cannot read /proc/$pid/status; skipping memory"; pid = nothing)
|
||||
if pid !== nothing
|
||||
peak_reset = reset_peak_rss(pid)
|
||||
peak_reset || @warn "could not reset the peak-RSS counter (/proc/$pid/clear_refs); " *
|
||||
"reporting sampled RSS only"
|
||||
# Baseline is read *after* the reset, not before. VmHWM restarts
|
||||
# from whatever RSS is at the moment of the reset, so a baseline
|
||||
# sampled earlier is measured against a different origin — and if
|
||||
# the GC hands memory back in between, the run reports negative
|
||||
# growth, which is nonsense on its face.
|
||||
r2 = read_rss(pid)
|
||||
baseline_rss = r2 === nothing ? r[1] : r2[1]
|
||||
end
|
||||
end
|
||||
|
||||
if Threads.nthreads() == 1 && opts["size"] > 64 * 1024^2
|
||||
@warn "running with 1 thread and large files: the sampler shares a thread with " *
|
||||
"blocking file reads, so the RSS curve will be coarse. Prefer -t auto."
|
||||
end
|
||||
|
||||
base_sinks = counts(sinks)
|
||||
interval = opts["sample-ms"] / 1000
|
||||
|
||||
# --- per-stage counters: the "before" half of the delta.
|
||||
stats_before = opts["no-stats"] ? nothing : scrape_stats(url)
|
||||
if stats_before === nothing && !opts["no-stats"]
|
||||
@warn "no /stats endpoint on this server; skipping the per-stage table " *
|
||||
"(the server predates src/stats.jl)"
|
||||
end
|
||||
|
||||
# --- sampler: RSS curve, stage dir depths, and queue depths.
|
||||
stop = Threads.Atomic{Bool}(false)
|
||||
rss_samples = Float64[]
|
||||
depth_max = Dict(k => 0 for k in keys(stages))
|
||||
queue_peak = Dict{Int,Int}()
|
||||
sampler = Threads.@spawn begin
|
||||
while !stop[]
|
||||
if pid !== nothing
|
||||
r = read_rss(pid)
|
||||
r !== nothing && push!(rss_samples, Float64(r[1]))
|
||||
end
|
||||
d = counts(stages)
|
||||
for k in keys(d)
|
||||
depth_max[k] = max(depth_max[k], getfield(d, k))
|
||||
end
|
||||
# Queue depth, unlike directory depth, can't be missed by a slow
|
||||
# sample in the same way — a file's *reference* sits in the queue
|
||||
# for the whole time it waits — so this is the depth the stage
|
||||
# table reports.
|
||||
if stats_before !== nothing
|
||||
s = scrape_stats(url)
|
||||
s === nothing || for st in s.stages
|
||||
k = Int(st.stage)
|
||||
queue_peak[k] = max(get(queue_peak, k, 0), Int(st.queue_depth))
|
||||
end
|
||||
end
|
||||
sleep(interval)
|
||||
end
|
||||
end
|
||||
|
||||
# --- intake
|
||||
println("uploading $(length(paths)) file(s) at concurrency $(opts["concurrency"])…")
|
||||
t_start = time()
|
||||
ups = upload_all(url, paths, opts["concurrency"])
|
||||
t_intake_end = time()
|
||||
|
||||
accepted = sum(u.accepted for u in ups)
|
||||
n_202 = count(u -> u.status == 202, ups)
|
||||
n_503 = count(u -> u.status == 503, ups)
|
||||
n_err = count(u -> !(u.status in (202, 503)), ups)
|
||||
intake_secs = t_intake_end - t_start
|
||||
lat = sort([u.seconds for u in ups])
|
||||
|
||||
println(" intake: $accepted job(s) accepted in $(fmt(intake_secs))s " *
|
||||
"($(fmt(accepted / max(intake_secs, 1e-9))) files/s, " *
|
||||
"$(fmt(corpusbytes / max(intake_secs, 1e-9) / 1024^2)) MiB/s)")
|
||||
n_503 > 0 && println(" backpressure: $n_503 request(s) got 503 (intake queue full)")
|
||||
n_err > 0 && println(" errors: $n_err request(s) failed or returned an unexpected status")
|
||||
|
||||
# --- drain: poll the terminal sinks until they stop moving.
|
||||
println("draining (polling sinks every $(opts["sample-ms"])ms)…")
|
||||
completed = 0
|
||||
t_last_progress = time()
|
||||
t_last_completion = t_intake_end
|
||||
timed_out = false
|
||||
while completed < accepted
|
||||
sleep(interval)
|
||||
c = total(deltas(counts(sinks), base_sinks))
|
||||
if c > completed
|
||||
completed = c
|
||||
t_last_completion = time()
|
||||
t_last_progress = t_last_completion
|
||||
elseif time() - t_last_progress > opts["timeout"]
|
||||
timed_out = true
|
||||
break
|
||||
end
|
||||
end
|
||||
# Scrape before stopping the sampler, so the window closes as near the
|
||||
# last completion as we can manage.
|
||||
stats_after = stats_before === nothing ? nothing : scrape_stats(url)
|
||||
stop[] = true
|
||||
wait(sampler)
|
||||
|
||||
sink_delta = deltas(counts(sinks), base_sinks)
|
||||
e2e_secs = t_last_completion - t_start
|
||||
|
||||
# The stage window is the server's own clock across the two scrapes, not
|
||||
# e2e_secs: it starts a scrape earlier and ends a scrape later, and using
|
||||
# our wall time against its counters would misattribute the difference.
|
||||
stage_window, stage_delta = if stats_after === nothing
|
||||
(0.0, StageDelta[])
|
||||
else
|
||||
(Float64(stats_after.now - stats_before.now),
|
||||
stage_deltas(stats_before, stats_after, queue_peak))
|
||||
end
|
||||
|
||||
final_rss = pid === nothing ? nothing : read_rss(pid)
|
||||
peak_rss = if final_rss !== nothing && peak_reset
|
||||
final_rss[2] # kernel VmHWM: catches spikes between samples
|
||||
elseif !isempty(rss_samples)
|
||||
maximum(rss_samples)
|
||||
else
|
||||
nothing
|
||||
end
|
||||
|
||||
# --- report
|
||||
println()
|
||||
println("=" ^ 68)
|
||||
println("corpus $(length(paths)) file(s), $(human(corpusbytes)) total, " *
|
||||
"$(human(corpusbytes / length(paths))) avg")
|
||||
println("concurrency $(opts["concurrency"])")
|
||||
println()
|
||||
println("INTAKE (HTTP 202 — bytes spooled, not processed)")
|
||||
println(" accepted $accepted of $(length(paths)) [202: $n_202, 503: $n_503, error: $n_err]")
|
||||
println(" wall $(fmt(intake_secs))s")
|
||||
println(" rate $(fmt(accepted / max(intake_secs, 1e-9))) files/s, " *
|
||||
"$(fmt(corpusbytes / max(intake_secs, 1e-9) / 1024^2)) MiB/s")
|
||||
println(" latency p50 $(fmt(percentile(lat, 0.5) * 1000, 1))ms " *
|
||||
"p95 $(fmt(percentile(lat, 0.95) * 1000, 1))ms " *
|
||||
"max $(fmt(percentile(lat, 1.0) * 1000, 1))ms")
|
||||
println()
|
||||
println("END-TO-END (files reaching a terminal sink)")
|
||||
println(" completed $completed of $accepted accepted" * (timed_out ? " ** TIMED OUT **" : ""))
|
||||
println(" wall $(fmt(e2e_secs))s (first upload → last completion)")
|
||||
println(" throughput $(fmt(completed / max(e2e_secs, 1e-9))) files/s, " *
|
||||
"$(fmt(corpusbytes / max(e2e_secs, 1e-9) / 1024^2)) MiB/s")
|
||||
println(" sinks done $(sink_delta.done) text_done $(sink_delta.text_done) " *
|
||||
"binary $(sink_delta.binary) failed $(sink_delta.failed)")
|
||||
println(" peak dir depth spool $(depth_max[:spool]) known $(depth_max[:known]) " *
|
||||
"unknown $(depth_max[:unknown]) text $(depth_max[:text])")
|
||||
println()
|
||||
if !isempty(stage_delta)
|
||||
stage_report(stage_delta, stage_window)
|
||||
println()
|
||||
end
|
||||
if peak_rss !== nothing
|
||||
println("SERVER MEMORY (pid $pid)")
|
||||
println(" baseline RSS $(human(baseline_rss))")
|
||||
println(" peak RSS $(human(peak_rss))" *
|
||||
(peak_reset ? " (kernel VmHWM, reset at start)" : " (sampled — may miss spikes)"))
|
||||
growth = peak_rss - baseline_rss
|
||||
if growth <= 0
|
||||
# RSS never got back to where it started, so the run's own cost
|
||||
# is below the noise floor of the server settling after startup.
|
||||
# Printing a negative "growth" would invite reading a memory
|
||||
# *saving* into what is really "too small to measure here".
|
||||
println(" growth none measurable (peak never exceeded the baseline)")
|
||||
println(" The baseline was still falling when we sampled it — give the")
|
||||
println(" server ~30s to settle after startup for a comparable figure.")
|
||||
else
|
||||
println(" growth $(human(growth))")
|
||||
println(" per in-flight $(human(growth / opts["concurrency"])) " *
|
||||
"at $(human(corpusbytes / length(paths))) avg file size")
|
||||
println(" (should not grow with file size — intake streams to disk)")
|
||||
end
|
||||
# Julia's GC returns memory to the OS lazily, so a second run on the
|
||||
# same process starts from an inflated baseline and under-reports
|
||||
# growth. Absolute peak is the number to trust across runs.
|
||||
baseline_rss > 1.3 * FRESH_RSS_HINT &&
|
||||
println(" ! baseline is well above a fresh start ($(human(FRESH_RSS_HINT))): the GC " *
|
||||
"has not\n returned memory from earlier work. Compare " *
|
||||
"absolute peak, or restart\n the server for a clean growth figure.")
|
||||
else
|
||||
println("SERVER MEMORY not sampled")
|
||||
end
|
||||
println("=" ^ 68)
|
||||
|
||||
sink_delta.failed > 0 &&
|
||||
println("\nnote: $(sink_delta.failed) file(s) landed in $(sinks.failed) — check the server log.")
|
||||
timed_out &&
|
||||
println("\nnote: drain stalled with $(accepted - completed) file(s) outstanding. " *
|
||||
"Check the server log and the stage dirs; raise --timeout if the pipeline is just slow.")
|
||||
|
||||
if opts["json"] !== nothing
|
||||
result = (
|
||||
url, concurrency = opts["concurrency"], kind = opts["kind"],
|
||||
files = length(paths), corpus_bytes = corpusbytes,
|
||||
avg_file_bytes = corpusbytes / length(paths),
|
||||
intake = (; accepted, n_202, n_503, n_err, seconds = intake_secs,
|
||||
files_per_sec = accepted / max(intake_secs, 1e-9),
|
||||
p50_ms = json_num(percentile(lat, 0.5) * 1000),
|
||||
p95_ms = json_num(percentile(lat, 0.95) * 1000),
|
||||
max_ms = json_num(percentile(lat, 1.0) * 1000)),
|
||||
end_to_end = (; completed, seconds = e2e_secs, timed_out,
|
||||
files_per_sec = completed / max(e2e_secs, 1e-9),
|
||||
mib_per_sec = corpusbytes / max(e2e_secs, 1e-9) / 1024^2,
|
||||
sinks = sink_delta, peak_stage_depth = depth_max),
|
||||
stages = [(; stage = d.stage, name = d.name, workers = d.workers,
|
||||
completed = d.completed, failed = d.failed, bytes = d.bytes,
|
||||
busy_seconds = d.busy, blocked_seconds = d.blocked,
|
||||
window_seconds = stage_window,
|
||||
files_per_sec = files_per_sec(d, stage_window),
|
||||
mib_per_sec = mib_per_sec(d, stage_window),
|
||||
service_ms = json_num(service_ms(d)),
|
||||
utilization = utilization(d, stage_window),
|
||||
blocked_share = blocked_share(d),
|
||||
peak_queue_depth = d.peak_depth,
|
||||
queue_capacity = d.capacity) for d in stage_delta],
|
||||
memory = (; pid, baseline_rss, peak_rss, peak_is_kernel_hwm = peak_reset,
|
||||
growth = peak_rss === nothing ? nothing : peak_rss - baseline_rss,
|
||||
samples = rss_samples),
|
||||
)
|
||||
open(String(opts["json"]), "w") do io
|
||||
JSON3.write(io, result)
|
||||
end
|
||||
println("\nwrote $(opts["json"])")
|
||||
end
|
||||
|
||||
return timed_out ? 1 : 0
|
||||
finally
|
||||
if generated && !opts["keep-corpus"]
|
||||
rm(corpusdir; recursive = true, force = true)
|
||||
elseif generated
|
||||
println("\nkept corpus: $corpusdir")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
exit(main(ARGS))
|
||||
end
|
||||
427
bin/bench_model.jl
Executable file
427
bin/bench_model.jl
Executable file
@@ -0,0 +1,427 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# bench_model.jl — microbenchmark the classifier in isolation, with no server,
|
||||
# no queue, and no disk in the way.
|
||||
#
|
||||
# bin/bench.jl measures the *pipeline*: it reports stage 1 as one number, the
|
||||
# wall time of `handle_classify_job`, which is feature reads + inference + a
|
||||
# rename + a log line, under whatever thread contention the other three pools are
|
||||
# creating. That number is the right one for capacity planning and the wrong one
|
||||
# for answering "is the model slow?". This script answers that question by taking
|
||||
# the model apart:
|
||||
#
|
||||
# read_features open, read 16 bytes, seek, read 16 bytes, scale
|
||||
# Lux.apply the network itself, on a feature vector already in memory
|
||||
# classify both together — what stage 1 actually calls per file
|
||||
#
|
||||
# Three properties are worth checking beyond the raw per-file cost:
|
||||
#
|
||||
# * Feature reads should be flat in file size. read_features seeks to the tail
|
||||
# rather than slurping, so a 1 GiB file should cost the same as a 1 KiB one.
|
||||
# (This is the same claim bin/bench.jl makes about memory, on the CPU axis.)
|
||||
# * Batching should be much cheaper per file. A 32x1 matmul wastes most of a
|
||||
# BLAS call; if batch-64 inference is many times cheaper per file, that is
|
||||
# the headroom a batching stage-1 would buy — worth knowing before building
|
||||
# one, since today the pipeline classifies strictly one file at a time.
|
||||
# * Inference should scale across threads. `Classifier` is shared read-only by
|
||||
# the whole stage-1 pool on the claim that Lux inference is pure. If per-file
|
||||
# cost degrades as tasks are added, that claim holds but BLAS threading is
|
||||
# fighting the worker pool, and stage-1 workers are contending, not scaling.
|
||||
#
|
||||
# Usage:
|
||||
# julia --project=. -t auto bin/bench_model.jl [options]
|
||||
#
|
||||
# --model PATH classifier artifact (default: $FS_MODEL_PATH or model/classifier.jld2)
|
||||
# --reps N inference calls per timed trial (default: 20000)
|
||||
# --trials N timed trials; the minimum is reported (default: 5)
|
||||
# --batches LIST batch sizes to sweep, comma-separated (default: 1,8,64,512)
|
||||
# --sizes LIST file sizes for the read_features sweep (default: 1k,64k,4m,256m)
|
||||
# --no-threads skip the thread-scaling sweep
|
||||
# --json PATH also write the results as JSON
|
||||
#
|
||||
# Reported times are the *minimum* over trials: for a microbenchmark the floor is
|
||||
# the signal and everything above it is scheduler and GC noise.
|
||||
|
||||
using JSON3
|
||||
using Random
|
||||
using Statistics
|
||||
using Printf
|
||||
|
||||
# The script is run directly, not as part of the package, so pull in exactly the
|
||||
# pieces the classifier needs. `Lux`/`JLD2` first — model.jl and classify.jl both
|
||||
# assume the including scope already has them (see the note at the top of model.jl).
|
||||
using Lux
|
||||
using JLD2
|
||||
using LinearAlgebra
|
||||
|
||||
const SRC = joinpath(dirname(@__DIR__), "src")
|
||||
include(joinpath(SRC, "model.jl"))
|
||||
include(joinpath(SRC, "classify.jl"))
|
||||
|
||||
# ---------------------------------------------------------------- option parsing
|
||||
|
||||
const DEFAULTS = Dict{String,Any}(
|
||||
"model" => get(ENV, "FS_MODEL_PATH", "model/classifier.jld2"),
|
||||
"reps" => 20_000,
|
||||
"trials" => 5,
|
||||
"batches" => "1,8,64,512",
|
||||
"sizes" => "1k,64k,4m,256m",
|
||||
"no-threads" => false,
|
||||
"json" => nothing,
|
||||
)
|
||||
|
||||
const FLAGS = ("no-threads",)
|
||||
|
||||
function parse_size(s::AbstractString)::Int
|
||||
m = match(r"^(\d+(?:\.\d+)?)\s*([kKmMgG]?)[bB]?$", strip(s))
|
||||
m === nothing && error("bad size: $s (expected e.g. 512, 64k, 8m, 1g)")
|
||||
mult = Dict('k' => 1024, 'm' => 1024^2, 'g' => 1024^3)
|
||||
scale = isempty(m[2]) ? 1 : mult[lowercase(m[2])[1]]
|
||||
return round(Int, parse(Float64, m[1]) * scale)
|
||||
end
|
||||
|
||||
function parse_args(argv)
|
||||
opts = copy(DEFAULTS)
|
||||
i = 1
|
||||
while i <= length(argv)
|
||||
a = argv[i]
|
||||
startswith(a, "--") || error("unexpected argument: $a")
|
||||
key = a[3:end]
|
||||
haskey(opts, key) || error("unknown option: $a")
|
||||
if key in FLAGS
|
||||
opts[key] = true; i += 1; continue
|
||||
end
|
||||
i + 1 <= length(argv) || error("option --$key needs a value")
|
||||
opts[key] = key in ("reps", "trials") ? parse(Int, argv[i+1]) : argv[i+1]
|
||||
i += 2
|
||||
end
|
||||
return opts
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- measurement
|
||||
|
||||
# Every timed loop stores its result here. Without a visible side effect the
|
||||
# compiler is free to hoist a pure call out of the loop and we would be timing an
|
||||
# empty `for`.
|
||||
const SINK = Ref{Any}(nothing)
|
||||
|
||||
"""
|
||||
measure(f, reps; trials) -> (ns_per_op, bytes_per_op)
|
||||
|
||||
Time `f` over `reps` calls, `trials` times, and report the fastest trial.
|
||||
|
||||
The first call is thrown away: it pays Julia's JIT compilation, which on a
|
||||
function this small is orders of magnitude more than the thing being measured.
|
||||
"""
|
||||
function measure(f, reps::Int; trials::Int = 5)
|
||||
SINK[] = f() # warm up (compile), and keep the result
|
||||
best = Inf
|
||||
for _ in 1:trials
|
||||
GC.gc()
|
||||
t0 = time_ns()
|
||||
for _ in 1:reps
|
||||
SINK[] = f()
|
||||
end
|
||||
best = min(best, (time_ns() - t0) / reps)
|
||||
end
|
||||
bytes = @allocated(SINK[] = f()) # one call, after warmup
|
||||
return (Float64(best), Float64(bytes))
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- formatting
|
||||
|
||||
function human_time(ns::Real)
|
||||
ns < 1_000 && return @sprintf("%.0f ns", ns)
|
||||
ns < 1_000_000 && return @sprintf("%.2f µs", ns / 1e3)
|
||||
ns < 1e9 && return @sprintf("%.2f ms", ns / 1e6)
|
||||
return @sprintf("%.2f s", ns / 1e9)
|
||||
end
|
||||
|
||||
human_bytes(b::Real) = b < 1024 ? @sprintf("%.0f B", b) :
|
||||
b < 1024^2 ? @sprintf("%.1f KiB", b / 1024) :
|
||||
@sprintf("%.1f MiB", b / 1024^2)
|
||||
|
||||
rate(ns::Real) = 1e9 / max(ns, 1e-9) # calls per second
|
||||
|
||||
function human_rate(r::Real)
|
||||
r >= 1e6 && return @sprintf("%.2fM/s", r / 1e6)
|
||||
r >= 1e3 && return @sprintf("%.1fk/s", r / 1e3)
|
||||
return @sprintf("%.0f/s", r)
|
||||
end
|
||||
|
||||
fmt2(x::Real) = @sprintf("%.2f", x)
|
||||
|
||||
# ------------------------------------------------------------- scaling control
|
||||
|
||||
"""
|
||||
control_kernel(x) -> Float64
|
||||
|
||||
Pure arithmetic, no allocation, no library call — deliberately dependent
|
||||
(each step needs the last) so the compiler can't vectorize it away, and sized to
|
||||
land in the same microsecond neighbourhood as one `Lux.apply`.
|
||||
"""
|
||||
function control_kernel(x::Float64)
|
||||
a = x
|
||||
@inbounds for i in 1:600
|
||||
a = sqrt(a + i)
|
||||
end
|
||||
return a
|
||||
end
|
||||
|
||||
"""
|
||||
control_scaling(trials) -> Vector
|
||||
|
||||
Run `control_kernel` over the same task counts as the model sweep. This is the
|
||||
machine's own ceiling for perfectly parallel work: if the control scales and the
|
||||
model doesn't, the shortfall is the model's, and no amount of `FS_WORKERS` will
|
||||
recover it.
|
||||
"""
|
||||
function control_scaling(trials::Int)
|
||||
rows = []
|
||||
base = 0.0
|
||||
per_task = 200_000
|
||||
for k in unique([1; 2; 4; 8; Threads.nthreads()])
|
||||
k > Threads.nthreads() && continue
|
||||
best = Inf
|
||||
for _ in 1:trials
|
||||
GC.gc()
|
||||
t0 = time_ns()
|
||||
@sync for _ in 1:k
|
||||
Threads.@spawn begin
|
||||
local acc = 0.0
|
||||
for i in 1:per_task
|
||||
acc += control_kernel(i % 97 + 1.0)
|
||||
end
|
||||
SINK[] = acc
|
||||
end
|
||||
end
|
||||
best = min(best, Float64(time_ns() - t0))
|
||||
end
|
||||
r = k * per_task / (best / 1e9)
|
||||
k == 1 && (base = r)
|
||||
push!(rows, (; tasks = k, ops_per_sec = r, speedup = r / base))
|
||||
end
|
||||
return rows
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- corpus
|
||||
|
||||
"""
|
||||
Write a file of exactly `size` random bytes, in bounded chunks.
|
||||
|
||||
Content is random rather than zeros so the classifier sees a realistic input —
|
||||
and so the filesystem can't cheat with a sparse file, which would make the tail
|
||||
`seek` unrepresentatively fast.
|
||||
"""
|
||||
function write_file(path::AbstractString, size::Int, rng)
|
||||
chunk = 1024 * 1024
|
||||
open(path, "w") do io
|
||||
remaining = size
|
||||
while remaining > 0
|
||||
n = min(chunk, remaining)
|
||||
write(io, rand(rng, UInt8, n))
|
||||
remaining -= n
|
||||
end
|
||||
end
|
||||
return path
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- main
|
||||
|
||||
function main(argv)
|
||||
opts = parse_args(argv)
|
||||
modelpath = String(opts["model"])
|
||||
isfile(modelpath) || (println(stderr, "model artifact not found: $modelpath"); return 1)
|
||||
|
||||
reps, trials = opts["reps"], opts["trials"]
|
||||
batches = [parse(Int, s) for s in split(String(opts["batches"]), ",")]
|
||||
sizes = [parse_size(s) for s in split(String(opts["sizes"]), ",")]
|
||||
|
||||
clf = load_classifier(modelpath)
|
||||
println("model $modelpath")
|
||||
println("architecture $(FEATURE_DIM) → 64 → 16 → 2 (Dense/relu, raw logits)")
|
||||
println("julia threads $(Threads.nthreads()) BLAS threads $(BLAS.get_num_threads())")
|
||||
println("timing min of $trials trials × $reps reps")
|
||||
println("=" ^ 72)
|
||||
|
||||
results = Dict{String,Any}()
|
||||
|
||||
# --- 1. inference alone, one file at a time: the number the pipeline pays.
|
||||
x1 = rand(Float32, FEATURE_DIM, 1)
|
||||
infer_ns, infer_bytes = measure(reps; trials) do
|
||||
Lux.apply(clf.model, x1, clf.ps, clf.st)
|
||||
end
|
||||
println()
|
||||
println("INFERENCE (Lux.apply, batch 1 — features already in memory)")
|
||||
println(" per call $(human_time(infer_ns)) $(human_rate(rate(infer_ns)))")
|
||||
println(" allocations $(human_bytes(infer_bytes)) per call")
|
||||
results["inference_batch1"] = (; ns = infer_ns, bytes = infer_bytes, per_sec = rate(infer_ns))
|
||||
|
||||
# --- 2. batching: how much of that is per-call overhead rather than math?
|
||||
println()
|
||||
println("INFERENCE BATCHED (same net, N files per apply)")
|
||||
println(" batch per batch per file files/s speedup")
|
||||
batch_rows = []
|
||||
for b in batches
|
||||
xb = rand(Float32, FEATURE_DIM, b)
|
||||
ns, _ = measure(max(1, reps ÷ b); trials) do
|
||||
Lux.apply(clf.model, xb, clf.ps, clf.st)
|
||||
end
|
||||
per_file = ns / b
|
||||
@printf(" %8d %13s %12s %13s %7.1fx\n",
|
||||
b, human_time(ns), human_time(per_file),
|
||||
human_rate(rate(per_file)), infer_ns / per_file)
|
||||
push!(batch_rows, (; batch = b, ns_per_batch = ns, ns_per_file = per_file,
|
||||
files_per_sec = rate(per_file), speedup = infer_ns / per_file))
|
||||
end
|
||||
results["batched"] = batch_rows
|
||||
println(" (a large speedup is headroom a batching stage 1 could claim; the pipeline")
|
||||
println(" classifies one file per job today, so it pays the batch-1 row above)")
|
||||
|
||||
# --- 3. feature reads: should be flat in file size (seek, not slurp).
|
||||
println()
|
||||
println("FEATURE READS (read_features: 16 head + 16 tail bytes, scaled)")
|
||||
println(" file size per call calls/s allocations")
|
||||
read_rows = []
|
||||
dir = mktempdir(; prefix = "fsmodel-")
|
||||
try
|
||||
rng = MersenneTwister(1234)
|
||||
for sz in sizes
|
||||
path = write_file(joinpath(dir, "f-$sz.bin"), sz, rng)
|
||||
# Fewer reps for the big files: this touches the page cache, and the
|
||||
# point is the shape of the curve, not another digit of precision.
|
||||
r = max(200, reps ÷ 20)
|
||||
ns, bytes = measure(() -> read_features(path), r; trials)
|
||||
@printf(" %13s %14s %14s %14s\n",
|
||||
human_bytes(sz), human_time(ns), human_rate(rate(ns)), human_bytes(bytes))
|
||||
push!(read_rows, (; size_bytes = sz, ns, bytes, per_sec = rate(ns)))
|
||||
end
|
||||
finally
|
||||
rm(dir; recursive = true, force = true)
|
||||
end
|
||||
results["read_features"] = read_rows
|
||||
flat = length(read_rows) > 1 ?
|
||||
maximum(r.ns for r in read_rows) / minimum(r.ns for r in read_rows) : 1.0
|
||||
@printf(" spread across a %.0fx size range: %.1fx — %s\n",
|
||||
maximum(sizes) / minimum(sizes), flat,
|
||||
flat < 3 ? "flat, as designed (it seeks to the tail)" :
|
||||
"NOT flat: something is reading more than 32 bytes")
|
||||
|
||||
# --- 4. classify(): what stage 1 calls, I/O and inference together.
|
||||
println()
|
||||
println("CLASSIFY (read_features + Lux.apply — one whole stage-1 file)")
|
||||
dir2 = mktempdir(; prefix = "fsmodel-")
|
||||
classify_ns = 0.0
|
||||
try
|
||||
path = write_file(joinpath(dir2, "sample.bin"), 64 * 1024, MersenneTwister(7))
|
||||
classify_ns, classify_bytes = measure(() -> classify(clf, path), max(200, reps ÷ 20); trials)
|
||||
println(" per file $(human_time(classify_ns)) $(human_rate(rate(classify_ns)))")
|
||||
println(" allocations $(human_bytes(classify_bytes)) per file")
|
||||
@printf(" split %.0f%% feature read, %.0f%% inference\n",
|
||||
100 * (classify_ns - infer_ns) / classify_ns, 100 * infer_ns / classify_ns)
|
||||
results["classify"] = (; ns = classify_ns, bytes = classify_bytes, per_sec = rate(classify_ns))
|
||||
finally
|
||||
rm(dir2; recursive = true, force = true)
|
||||
end
|
||||
|
||||
# --- 5. thread scaling: does the shared read-only Classifier actually scale?
|
||||
if !opts["no-threads"] && Threads.nthreads() > 1
|
||||
println()
|
||||
println("THREAD SCALING (concurrent Lux.apply on the one shared Classifier)")
|
||||
println(" tasks files/s per file speedup efficiency GC")
|
||||
thread_rows = []
|
||||
base = 0.0
|
||||
for k in unique([1; 2; 4; 8; Threads.nthreads()])
|
||||
k > Threads.nthreads() && continue
|
||||
# Work per task is held *constant* as tasks are added, so total work
|
||||
# scales with `k`. Splitting a fixed total instead would shrink each
|
||||
# task as the pool grows until `@spawn`/`@sync` overhead dominated,
|
||||
# and the resulting curve would show a collapse that is the
|
||||
# measurement's fault rather than the model's.
|
||||
per_task = max(reps, 20_000)
|
||||
# Each task gets its own input so we measure the model, not cache
|
||||
# line ping-pong on a shared buffer.
|
||||
xs = [rand(Float32, FEATURE_DIM, 1) for _ in 1:k]
|
||||
best, best_gc = Inf, 0.0
|
||||
for _ in 1:trials
|
||||
GC.gc()
|
||||
# Julia's GC stops the world, so it is the one cost that cannot
|
||||
# be parallelized away: measuring its share here is what turns a
|
||||
# bad efficiency number into a diagnosis (see the note below).
|
||||
gc0 = Base.gc_num().total_time
|
||||
t0 = time_ns()
|
||||
@sync for t in 1:k
|
||||
Threads.@spawn begin
|
||||
local acc = 0.0f0
|
||||
for _ in 1:per_task
|
||||
y, _ = Lux.apply(clf.model, xs[t], clf.ps, clf.st)
|
||||
acc += y[1] # consume the result
|
||||
end
|
||||
SINK[] = acc
|
||||
end
|
||||
end
|
||||
elapsed = Float64(time_ns() - t0)
|
||||
if elapsed < best
|
||||
best = elapsed
|
||||
best_gc = Float64(Base.gc_num().total_time - gc0)
|
||||
end
|
||||
end
|
||||
files = k * per_task
|
||||
fps = files / (best / 1e9)
|
||||
k == 1 && (base = fps)
|
||||
@printf(" %8d %13s %11s %7.2fx %10.0f%% %5.0f%%\n",
|
||||
k, human_rate(fps), human_time(best / files), fps / base,
|
||||
100 * fps / base / k, 100 * best_gc / best)
|
||||
push!(thread_rows, (; tasks = k, files_per_sec = fps,
|
||||
ns_per_file = best / files, speedup = fps / base,
|
||||
gc_fraction = best_gc / best))
|
||||
end
|
||||
results["thread_scaling"] = thread_rows
|
||||
|
||||
# A poor scaling curve has two possible authors — the model or the box —
|
||||
# and the table alone can't tell them apart. So run the same sweep on a
|
||||
# kernel that is pure arithmetic with no allocation and no library
|
||||
# underneath: whatever *it* achieves is this machine's ceiling for
|
||||
# embarrassingly parallel work, and the gap between the two curves is
|
||||
# the part that belongs to Lux.apply.
|
||||
ctrl = control_scaling(trials)
|
||||
results["control_scaling"] = ctrl
|
||||
top = last(ctrl)
|
||||
println(" control pure-compute kernel, same sweep: " *
|
||||
"$(fmt2(top.speedup))x at $(top.tasks) tasks " *
|
||||
"($(round(Int, 100 * top.speedup / top.tasks))% efficiency)")
|
||||
model_top = last(thread_rows)
|
||||
if model_top.speedup < 0.6 * top.speedup
|
||||
println(" → the machine parallelizes; Lux.apply does not. Stage-1")
|
||||
println(" workers past ~4 buy little, whatever FS_WORKERS says.")
|
||||
else
|
||||
println(" → inference tracks the machine's own scaling ceiling.")
|
||||
end
|
||||
gc_top = maximum(r.gc_fraction for r in thread_rows)
|
||||
gc_top > 0.15 && println(" ! GC is $(round(Int, 100 * gc_top))% of the " *
|
||||
"worst case: apply allocates per call, and\n" *
|
||||
" collection stops every thread.")
|
||||
end
|
||||
|
||||
println()
|
||||
println("=" ^ 72)
|
||||
println("Stage 1's cost per file in bin/bench.jl is this classify() figure plus a")
|
||||
println("rename, a log line, and whatever contention the other three pools create.")
|
||||
println("A large gap between the two is pipeline overhead, not the model.")
|
||||
|
||||
if opts["json"] !== nothing
|
||||
results["meta"] = (; model = modelpath, feature_dim = FEATURE_DIM,
|
||||
julia_threads = Threads.nthreads(),
|
||||
blas_threads = BLAS.get_num_threads(),
|
||||
reps, trials)
|
||||
open(String(opts["json"]), "w") do io
|
||||
JSON3.write(io, results)
|
||||
end
|
||||
println("\nwrote $(opts["json"])")
|
||||
end
|
||||
return 0
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
exit(main(ARGS))
|
||||
end
|
||||
556
bin/bench_stage1.jl
Normal file
556
bin/bench_stage1.jl
Normal file
@@ -0,0 +1,556 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# bench_stage1.jl — take stage 1 apart and find the slowest component.
|
||||
#
|
||||
# bin/bench.jl reports stage 1 as a single number (the wall time of
|
||||
# `handle_classify_job` under whole-pipeline contention) and bin/bench_model.jl
|
||||
# takes the *classifier* apart. Neither answers "which part of stage 1 costs the
|
||||
# most?", because stage 1 is more than the model. Per file it does:
|
||||
#
|
||||
# classify filesize + two 16-byte reads + a 32x1 forward pass
|
||||
# read_features open, read head, seek, read tail, scale to Float32
|
||||
# Lux.apply the network on a feature vector already in memory
|
||||
# move_to rename spool/<f> -> known/<f> or unknown/<f>
|
||||
# enqueue push a Job reference onto the downstream bounded queue
|
||||
# logging two @info lines ("classified file", "routed to ...")
|
||||
#
|
||||
# This script times each of those in isolation, then times the real
|
||||
# `handle_classify_job` end to end so the parts can be checked against the whole.
|
||||
# The two knobs that most change the answer get their own sweeps:
|
||||
#
|
||||
# * Logger. The server runs `FlushLogger(ConsoleLogger(stderr))` — it formats
|
||||
# and flushes every message. Under redirect (a log file, journald) that is a
|
||||
# syscall per line, two lines per file, on the hot path. We time the handler
|
||||
# under a null logger, a formatting-but-discarding logger, and the real
|
||||
# flushing-to-file logger, so the cost of logging is a subtraction, not a
|
||||
# guess. This sweep is what demoted stage 1's per-file lines to `@debug`
|
||||
# (see the note in src/worker.jl); the standalone `logging (...)` rows below
|
||||
# still price a *formatted* line, i.e. what those lines cost when switched
|
||||
# back on with `JULIA_DEBUG=FileServer`, while the handler rows show what the
|
||||
# stage pays with them off.
|
||||
# * Concurrency. Components that own a lock (the queue's condition, the
|
||||
# logger's stream) don't scale, and the ranking at one worker need not be the
|
||||
# ranking at sixteen. The `--threads` sweep runs the full handler across
|
||||
# worker counts.
|
||||
#
|
||||
# Usage:
|
||||
# julia --project=. -t auto bin/bench_stage1.jl [options]
|
||||
#
|
||||
# --files N files per timed pass for consuming benchmarks (default: 2000)
|
||||
# --reps N calls per timed pass for non-consuming benchmarks (default: 20000)
|
||||
# --trials N timed passes; the minimum is reported (default: 5)
|
||||
# --size SPEC corpus file size (default: 64k)
|
||||
# --dir PATH working directory for the corpus (default: a temp dir under data/)
|
||||
# --model PATH classifier artifact (default: $FS_MODEL_PATH or model/classifier.jld2)
|
||||
# --threads LIST worker counts for the concurrency sweep (default: 1,2,4,8,nthreads)
|
||||
# --no-threads skip the concurrency sweep
|
||||
# --json PATH also write the results as JSON
|
||||
#
|
||||
# Reported times are the *minimum* over trials: the floor is the signal and
|
||||
# everything above it is scheduler, page-cache and GC noise.
|
||||
|
||||
using FileServer
|
||||
using Lux
|
||||
using JSON3
|
||||
using Logging
|
||||
using Printf
|
||||
using Random
|
||||
|
||||
const FS = FileServer
|
||||
|
||||
# ---------------------------------------------------------------- option parsing
|
||||
|
||||
const DEFAULTS = Dict{String,Any}(
|
||||
"files" => 2000,
|
||||
"reps" => 20_000,
|
||||
"trials" => 5,
|
||||
"size" => "64k",
|
||||
"dir" => nothing,
|
||||
"model" => get(ENV, "FS_MODEL_PATH", "model/classifier.jld2"),
|
||||
"threads" => nothing,
|
||||
"no-threads" => false,
|
||||
"json" => nothing,
|
||||
)
|
||||
|
||||
const FLAGS = ("no-threads",)
|
||||
const INTS = ("files", "reps", "trials")
|
||||
|
||||
function parse_size(s::AbstractString)::Int
|
||||
m = match(r"^(\d+(?:\.\d+)?)\s*([kKmMgG]?)[bB]?$", strip(s))
|
||||
m === nothing && error("bad size: $s (expected e.g. 512, 64k, 8m, 1g)")
|
||||
mult = Dict('k' => 1024, 'm' => 1024^2, 'g' => 1024^3)
|
||||
scale = isempty(m[2]) ? 1 : mult[lowercase(m[2])[1]]
|
||||
return round(Int, parse(Float64, m[1]) * scale)
|
||||
end
|
||||
|
||||
function parse_args(argv)
|
||||
opts = copy(DEFAULTS)
|
||||
i = 1
|
||||
while i <= length(argv)
|
||||
a = argv[i]
|
||||
startswith(a, "--") || error("unexpected argument: $a")
|
||||
key = a[3:end]
|
||||
haskey(opts, key) || error("unknown option: $a")
|
||||
if key in FLAGS
|
||||
opts[key] = true; i += 1; continue
|
||||
end
|
||||
i + 1 <= length(argv) || error("option --$key needs a value")
|
||||
opts[key] = key in INTS ? parse(Int, argv[i+1]) : argv[i+1]
|
||||
i += 2
|
||||
end
|
||||
return opts
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- measurement
|
||||
|
||||
# Every timed loop stores its result here. Without a visible side effect the
|
||||
# compiler is free to hoist a pure call out of the loop and we would be timing an
|
||||
# empty `for`.
|
||||
const SINK = Ref{Any}(nothing)
|
||||
|
||||
"""
|
||||
best_of(pass, prepare; trials) -> ns_per_op
|
||||
|
||||
Run `pass()` `trials` times and report the fastest, in nanoseconds per operation
|
||||
(`pass` returns the number of operations it performed). `prepare()` runs before
|
||||
each pass and is *not* timed — that is where a consuming benchmark puts the file
|
||||
back where it started. `pass` comes first so callers can pass it as a `do` block.
|
||||
|
||||
The first pass is thrown away: it pays Julia's JIT compilation, which on calls
|
||||
this small is orders of magnitude more than the thing being measured.
|
||||
"""
|
||||
function best_of(pass, prepare; trials::Int)
|
||||
best = Inf
|
||||
for t in 0:trials
|
||||
prepare()
|
||||
GC.gc()
|
||||
t0 = time_ns()
|
||||
n = pass()
|
||||
dt = Float64(time_ns() - t0)
|
||||
t == 0 && continue # warm-up: compiled, not measured
|
||||
best = min(best, dt / n)
|
||||
end
|
||||
return best
|
||||
end
|
||||
|
||||
noop() = nothing
|
||||
|
||||
# ------------------------------------------------------------------- formatting
|
||||
|
||||
function human_time(ns::Real)
|
||||
ns < 1_000 && return @sprintf("%.0f ns", ns)
|
||||
ns < 1_000_000 && return @sprintf("%.2f µs", ns / 1e3)
|
||||
ns < 1e9 && return @sprintf("%.2f ms", ns / 1e6)
|
||||
return @sprintf("%.2f s", ns / 1e9)
|
||||
end
|
||||
|
||||
function human_rate(r::Real)
|
||||
r >= 1e6 && return @sprintf("%.2fM/s", r / 1e6)
|
||||
r >= 1e3 && return @sprintf("%.1fk/s", r / 1e3)
|
||||
return @sprintf("%.0f/s", r)
|
||||
end
|
||||
|
||||
rate(ns::Real) = 1e9 / max(ns, 1e-9)
|
||||
|
||||
rule(n = 78) = println("-" ^ n)
|
||||
|
||||
function header(title)
|
||||
println()
|
||||
println(title)
|
||||
rule()
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- corpus
|
||||
|
||||
"""
|
||||
Write a file of exactly `size` random bytes, in bounded chunks.
|
||||
|
||||
Content is random rather than zeros so the classifier sees a realistic input —
|
||||
and so the filesystem can't cheat with a sparse file, which would make the tail
|
||||
`seek` unrepresentatively fast.
|
||||
"""
|
||||
function write_file(path::AbstractString, size::Int, rng)
|
||||
chunk = 1024 * 1024
|
||||
open(path, "w") do io
|
||||
remaining = size
|
||||
while remaining > 0
|
||||
n = min(chunk, remaining)
|
||||
write(io, rand(rng, UInt8, n))
|
||||
remaining -= n
|
||||
end
|
||||
end
|
||||
return path
|
||||
end
|
||||
|
||||
"""
|
||||
make_corpus(cfg, n, size, rng) -> Vector{Job}
|
||||
|
||||
Create `n` spooled files and the `Job` references a stage-1 worker would dequeue
|
||||
for them — the exact input `handle_classify_job` sees.
|
||||
"""
|
||||
function make_corpus(cfg::FS.Config, n::Int, size::Int, rng)
|
||||
jobs = FS.Job[]
|
||||
for i in 1:n
|
||||
id, path = FS.spool_path(cfg, @sprintf("bench-%06d.bin", i))
|
||||
write_file(path, size, rng)
|
||||
push!(jobs, FS.Job(id, basename(path), path, size, time()))
|
||||
end
|
||||
return jobs
|
||||
end
|
||||
|
||||
"""
|
||||
respool!(cfg, jobs)
|
||||
|
||||
Put every corpus file back in `spool/`, wherever the last pass left it (known/,
|
||||
unknown/, or already home). This is the untimed `prepare` step for benchmarks
|
||||
that consume their input by moving it.
|
||||
"""
|
||||
function respool!(cfg::FS.Config, jobs::Vector{FS.Job})
|
||||
for job in jobs
|
||||
isfile(job.path) && continue
|
||||
for dir in (cfg.known_dir, cfg.unknown_dir, cfg.failed_dir)
|
||||
candidate = joinpath(dir, basename(job.path))
|
||||
if isfile(candidate)
|
||||
mv(candidate, job.path; force = true)
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"Drain a queue without blocking, so the next pass starts from empty."
|
||||
function drain!(q::FS.ChannelQueue)
|
||||
while length(q) > 0
|
||||
FS.dequeue!(q)
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# --------------------------------------------------------------------- loggers
|
||||
|
||||
"""
|
||||
with_logger_named(name, path, f)
|
||||
|
||||
Run `f` under one of the three loggers the cost of logging is bracketed by:
|
||||
|
||||
* `:null` — `NullLogger`: the `@info` macro's own overhead, nothing else.
|
||||
* `:format` — `ConsoleLogger` to `devnull`: message formatting and key/value
|
||||
interpolation, but no I/O.
|
||||
* `:flush` — `FlushLogger(ConsoleLogger(io))` to a real file: what
|
||||
`FileServer.run` installs, under the redirect it was written for.
|
||||
* `:debug` — the same, at `Debug` level: the stage's per-file lines are
|
||||
`@debug`, so this is the equivalent of running the server with
|
||||
`JULIA_DEBUG=FileServer` and the only setting under which they
|
||||
are emitted at all.
|
||||
"""
|
||||
function with_logger_named(f, which::Symbol, path::AbstractString)
|
||||
if which === :null
|
||||
return with_logger(f, NullLogger())
|
||||
elseif which === :format
|
||||
return with_logger(f, ConsoleLogger(devnull))
|
||||
elseif which === :flush || which === :debug
|
||||
level = which === :debug ? Logging.Debug : Logging.Info
|
||||
return open(path, "w") do io
|
||||
with_logger(f, FS.FlushLogger(ConsoleLogger(io, level)))
|
||||
end
|
||||
end
|
||||
error("unknown logger: $which")
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------ components
|
||||
|
||||
"""
|
||||
component_rows(cfg, clf, jobs, opts) -> Vector
|
||||
|
||||
Time each piece of stage 1 on its own. Non-consuming pieces (`filesize`,
|
||||
`read_features`, `Lux.apply`, `classify`, the log lines) run `reps` times over
|
||||
the corpus; consuming pieces (`move_to`, the full handler) run once per corpus
|
||||
file with an untimed reset between passes.
|
||||
"""
|
||||
function component_rows(cfg::FS.Config, clf::FS.Classifier, jobs::Vector{FS.Job}, opts)
|
||||
reps, trials = opts["reps"], opts["trials"]
|
||||
nfiles = length(jobs)
|
||||
paths = [j.path for j in jobs]
|
||||
rows = []
|
||||
|
||||
# A feature vector already in memory, so the inference row measures the net
|
||||
# and not the disk read in front of it.
|
||||
feats = FS.read_features(paths[1])
|
||||
x = reshape(feats, FS.FEATURE_DIM, 1)
|
||||
|
||||
logger = ConsoleLogger(devnull) # components other than the log rows: quiet
|
||||
|
||||
# --- filesize: the stat() read_features does before touching the bytes
|
||||
push!(rows, (; name = "filesize (stat)", part = "classify",
|
||||
ns = best_of(noop; trials) do
|
||||
@inbounds for i in 1:reps
|
||||
SINK[] = filesize(paths[(i - 1) % nfiles + 1])
|
||||
end
|
||||
reps
|
||||
end))
|
||||
|
||||
# --- read_features: open + head read + seek + tail read + scale
|
||||
push!(rows, (; name = "read_features", part = "classify",
|
||||
ns = best_of(noop; trials) do
|
||||
@inbounds for i in 1:reps
|
||||
SINK[] = FS.read_features(paths[(i - 1) % nfiles + 1])
|
||||
end
|
||||
reps
|
||||
end))
|
||||
|
||||
# --- Lux.apply: the network alone, features already in memory
|
||||
push!(rows, (; name = "Lux.apply (1x32)", part = "classify",
|
||||
ns = best_of(noop; trials) do
|
||||
for _ in 1:reps
|
||||
SINK[] = Lux.apply(clf.model, x, clf.ps, clf.st)
|
||||
end
|
||||
reps
|
||||
end))
|
||||
|
||||
# --- classify: read_features + apply + argmax, what the handler calls
|
||||
push!(rows, (; name = "classify (total)", part = "classify",
|
||||
ns = best_of(noop; trials) do
|
||||
@inbounds for i in 1:reps
|
||||
SINK[] = FS.classify(clf, paths[(i - 1) % nfiles + 1])
|
||||
end
|
||||
reps
|
||||
end))
|
||||
|
||||
# --- move_to: the rename out of spool/. Consuming: reset before each pass.
|
||||
push!(rows, (; name = "move_to (rename)", part = "route",
|
||||
ns = best_of(() -> respool!(cfg, jobs); trials) do
|
||||
@inbounds for job in jobs
|
||||
SINK[] = FS.move_to(cfg.unknown_dir, job)
|
||||
end
|
||||
nfiles
|
||||
end))
|
||||
|
||||
# --- enqueue: lock, push, notify on an uncontended, non-full queue
|
||||
q = FS.ChannelQueue(nfiles + 1)
|
||||
stats = FS.StageStats()
|
||||
push!(rows, (; name = "enqueue_blocking!", part = "route",
|
||||
ns = best_of(() -> drain!(q); trials) do
|
||||
@inbounds for job in jobs
|
||||
SINK[] = FS.enqueue_blocking!(q, job, stats;
|
||||
retry_seconds = FS.ROUTE_ENQUEUE_RETRY_SECONDS)
|
||||
end
|
||||
nfiles
|
||||
end))
|
||||
drain!(q)
|
||||
|
||||
# --- the two @info lines, under each of the three loggers
|
||||
job1 = jobs[1]
|
||||
logfile = joinpath(cfg.spool_dir, "..", "bench_stage1.log")
|
||||
for (which, label) in ((:null, "logging (NullLogger)"),
|
||||
(:format, "logging (format only)"),
|
||||
(:flush, "logging (flush→file)"))
|
||||
ns = with_logger_named(which, logfile) do
|
||||
best_of(noop; trials) do
|
||||
for _ in 1:reps
|
||||
@info "classified file" worker=1 id=job1.id name=job1.original_name size=job1.size classification=:unknown
|
||||
@info "routed to content triage" worker=1 id=job1.id dest=job1.path
|
||||
end
|
||||
reps
|
||||
end
|
||||
end
|
||||
push!(rows, (; name = label, part = "log", ns))
|
||||
end
|
||||
rm(logfile; force = true)
|
||||
|
||||
return rows, logger
|
||||
end
|
||||
|
||||
"""
|
||||
handler_rows(cfg, jobs, opts) -> Vector
|
||||
|
||||
Time the real `handle_classify_job` end to end under each logger. The difference
|
||||
between the rows is the cost logging adds to a file; the `:flush` row is what the
|
||||
running server actually pays.
|
||||
"""
|
||||
function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
|
||||
trials = opts["trials"]
|
||||
nfiles = length(jobs)
|
||||
logfile = joinpath(dirname(cfg.spool_dir), "bench_stage1.log")
|
||||
known = FS.ChannelQueue(nfiles + 1)
|
||||
unknown = FS.ChannelQueue(nfiles + 1)
|
||||
stats = FS.StageStats()
|
||||
rows = []
|
||||
for (which, label) in ((:null, "handle_classify_job (NullLogger)"),
|
||||
(:format, "handle_classify_job (format only)"),
|
||||
(:flush, "handle_classify_job (flush→file)"),
|
||||
(:debug, "handle_classify_job (JULIA_DEBUG)"))
|
||||
ns = with_logger_named(which, logfile) do
|
||||
best_of(() -> (respool!(cfg, jobs); drain!(known); drain!(unknown)); trials) do
|
||||
@inbounds for job in jobs
|
||||
FS.handle_classify_job(job, cfg, 1, known, unknown, stats)
|
||||
end
|
||||
nfiles
|
||||
end
|
||||
end
|
||||
push!(rows, (; name = label, part = "total", ns))
|
||||
end
|
||||
respool!(cfg, jobs); drain!(known); drain!(unknown)
|
||||
rm(logfile; force = true)
|
||||
return rows
|
||||
end
|
||||
|
||||
"""
|
||||
thread_rows(cfg, jobs, opts) -> Vector
|
||||
|
||||
Run the full handler across worker counts, under the server's real logger. A
|
||||
component that owns a lock — the queue's condition variable, the logger's
|
||||
stream — stops scaling here even though it looked cheap single-threaded, so this
|
||||
is where the single-thread ranking gets checked against the deployed one.
|
||||
"""
|
||||
function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
|
||||
trials = opts["trials"]
|
||||
nfiles = length(jobs)
|
||||
counts = opts["threads"] === nothing ?
|
||||
unique([1; 2; 4; 8; Threads.nthreads()]) :
|
||||
[parse(Int, s) for s in split(String(opts["threads"]), ",")]
|
||||
counts = sort(unique(filter(k -> 1 <= k <= Threads.nthreads(), counts)))
|
||||
|
||||
logfile = joinpath(dirname(cfg.spool_dir), "bench_stage1.log")
|
||||
known = FS.ChannelQueue(nfiles + 1)
|
||||
unknown = FS.ChannelQueue(nfiles + 1)
|
||||
stats = FS.StageStats()
|
||||
rows = []
|
||||
base = 0.0
|
||||
for k in counts
|
||||
ns = with_logger_named(:flush, logfile) do
|
||||
best_of(() -> (respool!(cfg, jobs); drain!(known); drain!(unknown)); trials) do
|
||||
# Static split: each task takes a contiguous slice, so the only
|
||||
# sharing between workers is the state the server also shares —
|
||||
# the classifier, the queues, the logger, the filesystem.
|
||||
chunk = cld(nfiles, k)
|
||||
@sync for t in 1:k
|
||||
lo = (t - 1) * chunk + 1
|
||||
hi = min(t * chunk, nfiles)
|
||||
lo > hi && continue
|
||||
Threads.@spawn begin
|
||||
@inbounds for i in lo:hi
|
||||
FS.handle_classify_job(jobs[i], cfg, t, known, unknown, stats)
|
||||
end
|
||||
end
|
||||
end
|
||||
nfiles
|
||||
end
|
||||
end
|
||||
r = rate(ns) # files/sec aggregate (ns is already per file, wall-clock)
|
||||
k == counts[1] && (base = r)
|
||||
push!(rows, (; workers = k, ns, files_per_sec = r, speedup = r / base))
|
||||
end
|
||||
respool!(cfg, jobs); drain!(known); drain!(unknown)
|
||||
rm(logfile; force = true)
|
||||
return rows
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- reporting
|
||||
|
||||
function print_components(rows, total_ns)
|
||||
@printf("%-34s %-9s %12s %10s %9s\n", "component", "part", "per file", "rate", "% total")
|
||||
rule()
|
||||
for r in rows
|
||||
@printf("%-34s %-9s %12s %10s %8.1f%%\n", r.name, r.part, human_time(r.ns),
|
||||
human_rate(rate(r.ns)), 100 * r.ns / total_ns)
|
||||
end
|
||||
end
|
||||
|
||||
function print_threads(rows)
|
||||
@printf("%-9s %12s %12s %9s\n", "workers", "per file", "throughput", "speedup")
|
||||
rule()
|
||||
for r in rows
|
||||
@printf("%-9d %12s %12s %8.2fx\n", r.workers, human_time(r.ns),
|
||||
human_rate(r.files_per_sec), r.speedup)
|
||||
end
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- main
|
||||
|
||||
function main(argv)
|
||||
opts = parse_args(argv)
|
||||
modelpath = String(opts["model"])
|
||||
isfile(modelpath) || (println(stderr, "model artifact not found: $modelpath"); return 1)
|
||||
|
||||
size = parse_size(String(opts["size"]))
|
||||
nfiles, trials = opts["files"], opts["trials"]
|
||||
|
||||
root = opts["dir"] === nothing ?
|
||||
mktempdir(pwd(); prefix = "bench_stage1_") : String(opts["dir"])
|
||||
owned = opts["dir"] === nothing
|
||||
cfg = FS.Config(
|
||||
spool_dir = joinpath(root, "spool"),
|
||||
known_dir = joinpath(root, "known"),
|
||||
unknown_dir = joinpath(root, "unknown"),
|
||||
failed_dir = joinpath(root, "failed"),
|
||||
model_path = modelpath,
|
||||
)
|
||||
for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.failed_dir)
|
||||
mkpath(d)
|
||||
end
|
||||
|
||||
clf = FS.load_classifier(modelpath)
|
||||
FS.CLASSIFIER[] = clf # handle_classify_job reads the global, as in the server
|
||||
|
||||
println("stage-1 component benchmark")
|
||||
rule()
|
||||
@printf("%-22s %s\n", "julia threads", Threads.nthreads())
|
||||
@printf("%-22s %s\n", "model", modelpath)
|
||||
@printf("%-22s %s\n", "corpus", "$(nfiles) files x $(size) B in $(root)")
|
||||
@printf("%-22s %s\n", "reps / trials", "$(opts["reps"]) / $(trials)")
|
||||
|
||||
rng = MersenneTwister(0x5741524d)
|
||||
jobs = make_corpus(cfg, nfiles, size, rng)
|
||||
|
||||
try
|
||||
comps, _ = component_rows(cfg, clf, jobs, opts)
|
||||
handlers = handler_rows(cfg, jobs, opts)
|
||||
|
||||
# The denominator is the handler as the server actually runs it: the real
|
||||
# flushing logger at Info level, one worker. Percentages are shares of
|
||||
# that, so they are directly comparable and the parts can be checked
|
||||
# against the whole. The JULIA_DEBUG row is deliberately *not* the
|
||||
# baseline — it is the opt-in configuration, and letting it set the scale
|
||||
# would make every other component look free.
|
||||
total = only(r.ns for r in handlers if r.name == "handle_classify_job (flush→file)")
|
||||
|
||||
header("Components (single worker)")
|
||||
print_components(comps, total)
|
||||
|
||||
header("Whole handler, by logger")
|
||||
print_components(handlers, total)
|
||||
|
||||
# Stage 1's own log lines are `@debug`, so what the deployed handler pays
|
||||
# for them is the disabled-macro cost, not a formatted line.
|
||||
accounted = sum(r.ns for r in comps if r.name in
|
||||
("classify (total)", "move_to (rename)", "enqueue_blocking!", "logging (NullLogger)"))
|
||||
println()
|
||||
@printf("accounted: %s of %s (%.0f%%); unaccounted overhead %s\n",
|
||||
human_time(accounted), human_time(total), 100 * accounted / total,
|
||||
human_time(max(total - accounted, 0)))
|
||||
|
||||
threads = nothing
|
||||
if !opts["no-threads"] && Threads.nthreads() > 1
|
||||
threads = thread_rows(cfg, jobs, opts)
|
||||
header("Full handler across workers (server logger)")
|
||||
print_threads(threads)
|
||||
end
|
||||
|
||||
if opts["json"] !== nothing
|
||||
open(String(opts["json"]), "w") do io
|
||||
JSON3.write(io, (;
|
||||
julia_threads = Threads.nthreads(),
|
||||
file_size = size, files = nfiles, reps = opts["reps"], trials,
|
||||
components = comps, handlers, threads,
|
||||
))
|
||||
end
|
||||
println("\nwrote ", opts["json"])
|
||||
end
|
||||
finally
|
||||
owned && rm(root; recursive = true, force = true)
|
||||
end
|
||||
return 0
|
||||
end
|
||||
|
||||
exit(main(ARGS))
|
||||
699
bin/bench_stage2.jl
Normal file
699
bin/bench_stage2.jl
Normal file
@@ -0,0 +1,699 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# bench_stage2.jl — take stage 2 apart and find the slowest component.
|
||||
#
|
||||
# bin/bench.jl reports stage 2 as a single number (its throughput and worker
|
||||
# utilization under whole-pipeline contention). It doesn't say *which part* of
|
||||
# the stage costs the most, and stage 2 is the one stage whose cost is dominated
|
||||
# by something outside Julia entirely: it forks `exiftool`, a Perl program, once
|
||||
# per file. Per file the stage does:
|
||||
#
|
||||
# build_metadata
|
||||
# run_exiftool fork/exec exiftool -json -G -n, capture stdout
|
||||
# run_with_timeout the watchdog wrapper around the subprocess
|
||||
# JSON3.read parse the dump
|
||||
# normalize_metadata coalesce ~20 tag names into the sidecar schema
|
||||
# finalize_known! (commit_enriched!)
|
||||
# JSON3.write serialize the sidecar payload
|
||||
# write + fsync durably persist the sidecar bytes to a temp name
|
||||
# mv + fsync_dir commit the sidecar, then persist the rename itself
|
||||
# move_to rename known/<f> -> done/<f>, the commit point
|
||||
# logging one @info line ("enriched")
|
||||
#
|
||||
# This script times each of those in isolation, then times the real
|
||||
# `handle_known_job` end to end so the parts can be checked against the whole.
|
||||
#
|
||||
# Three things here that the stage-1 benchmark has no equivalent of:
|
||||
#
|
||||
# * The corpus must be real files. exiftool's cost depends on what it finds;
|
||||
# random bytes exit early and would understate the stage by a lot. The
|
||||
# default corpus is `data/done` — files that already went through stage 2 on
|
||||
# this machine — copied back into a scratch known/ dir.
|
||||
# * Two rows price the *alternatives* to one-fork-per-file, because if the
|
||||
# fork dominates then the only fixes are to stop paying it per file:
|
||||
# `exiftool (batched Nx)` runs the whole corpus through one process, and
|
||||
# `exiftool (-stay_open)` keeps a single process alive and feeds it one file
|
||||
# at a time over a pipe — the shape a streaming pipeline could actually use.
|
||||
# Both are measured, not assumed.
|
||||
# * `run_with_timeout` gets its own row *next to* a bare `Base.run` of the same
|
||||
# command. The difference is what the watchdog costs, and its polling loop
|
||||
# (`sleep(0.1)`) is a suspicious enough design to want measured rather than
|
||||
# reasoned about.
|
||||
#
|
||||
# The `--threads` sweep runs the full handler across worker counts: subprocess
|
||||
# spawning contends on things (the kernel's fork path, page cache, the logger's
|
||||
# stream) that a single-threaded ranking can't reveal.
|
||||
#
|
||||
# Usage:
|
||||
# julia --project=. -t auto bin/bench_stage2.jl [options]
|
||||
#
|
||||
# --files N corpus files per timed pass (default: 48). The concurrency
|
||||
# sweep wants more than the component rows do — with a single
|
||||
# 2 s file in the corpus, 48 files can't show more than ~4x no
|
||||
# matter how many workers run, so pass --files 150 when the
|
||||
# question is scaling.
|
||||
# --reps N calls per timed pass for cheap, non-consuming benchmarks (default: 2000)
|
||||
# --trials N timed passes; the minimum is reported (default: 3)
|
||||
# --corpus PATH directory of real files to draw the corpus from (default: data/done)
|
||||
# --dir PATH working directory for the corpus (default: a temp dir under data/)
|
||||
# --timeout SEC exiftool timeout, as Config.exiftool_timeout (default: 30)
|
||||
# --threads LIST worker counts for the concurrency sweep (default: 1,2,4,8,nthreads)
|
||||
# --no-threads skip the concurrency sweep
|
||||
# --no-stay-open skip the persistent-exiftool probe
|
||||
# --json PATH also write the results as JSON
|
||||
#
|
||||
# Reported times are the *minimum* over trials: the floor is the signal and
|
||||
# everything above it is scheduler, page-cache and GC noise.
|
||||
|
||||
using FileServer
|
||||
using JSON3
|
||||
using Logging
|
||||
using Printf
|
||||
using Random
|
||||
|
||||
const FS = FileServer
|
||||
|
||||
# ---------------------------------------------------------------- option parsing
|
||||
|
||||
const DEFAULTS = Dict{String,Any}(
|
||||
"files" => 48,
|
||||
"reps" => 2000,
|
||||
"trials" => 3,
|
||||
"corpus" => "data/done",
|
||||
"dir" => nothing,
|
||||
"timeout" => 30,
|
||||
"threads" => nothing,
|
||||
"no-threads" => false,
|
||||
"no-stay-open" => false,
|
||||
"json" => nothing,
|
||||
)
|
||||
|
||||
const FLAGS = ("no-threads", "no-stay-open")
|
||||
const INTS = ("files", "reps", "trials", "timeout")
|
||||
|
||||
function parse_args(argv)
|
||||
opts = copy(DEFAULTS)
|
||||
i = 1
|
||||
while i <= length(argv)
|
||||
a = argv[i]
|
||||
startswith(a, "--") || error("unexpected argument: $a")
|
||||
key = a[3:end]
|
||||
haskey(opts, key) || error("unknown option: $a")
|
||||
if key in FLAGS
|
||||
opts[key] = true; i += 1; continue
|
||||
end
|
||||
i + 1 <= length(argv) || error("option --$key needs a value")
|
||||
opts[key] = key in INTS ? parse(Int, argv[i+1]) : argv[i+1]
|
||||
i += 2
|
||||
end
|
||||
return opts
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- measurement
|
||||
|
||||
# Every timed loop stores its result here. Without a visible side effect the
|
||||
# compiler is free to hoist a pure call out of the loop and we would be timing an
|
||||
# empty `for`.
|
||||
const SINK = Ref{Any}(nothing)
|
||||
|
||||
"""
|
||||
best_of(pass, prepare; trials) -> ns_per_op
|
||||
|
||||
Run `pass()` `trials` times and report the fastest, in nanoseconds per operation
|
||||
(`pass` returns the number of operations it performed). `prepare()` runs before
|
||||
each pass and is *not* timed — that is where a consuming benchmark puts the file
|
||||
back where it started. `pass` comes first so callers can pass it as a `do` block.
|
||||
|
||||
The first pass is thrown away: it pays Julia's JIT compilation, which on calls
|
||||
this small is orders of magnitude more than the thing being measured.
|
||||
"""
|
||||
function best_of(pass, prepare; trials::Int)
|
||||
best = Inf
|
||||
for t in 0:trials
|
||||
prepare()
|
||||
GC.gc()
|
||||
t0 = time_ns()
|
||||
n = pass()
|
||||
dt = Float64(time_ns() - t0)
|
||||
t == 0 && continue # warm-up: compiled, not measured
|
||||
best = min(best, dt / n)
|
||||
end
|
||||
return best
|
||||
end
|
||||
|
||||
noop() = nothing
|
||||
|
||||
# ------------------------------------------------------------------- formatting
|
||||
|
||||
function human_time(ns::Real)
|
||||
ns < 1_000 && return @sprintf("%.0f ns", ns)
|
||||
ns < 1_000_000 && return @sprintf("%.2f µs", ns / 1e3)
|
||||
ns < 1e9 && return @sprintf("%.2f ms", ns / 1e6)
|
||||
return @sprintf("%.2f s", ns / 1e9)
|
||||
end
|
||||
|
||||
function human_rate(r::Real)
|
||||
r >= 1e6 && return @sprintf("%.2fM/s", r / 1e6)
|
||||
r >= 1e3 && return @sprintf("%.1fk/s", r / 1e3)
|
||||
return @sprintf("%.0f/s", r)
|
||||
end
|
||||
|
||||
rate(ns::Real) = 1e9 / max(ns, 1e-9)
|
||||
|
||||
rule(n = 84) = println("-" ^ n)
|
||||
|
||||
function header(title)
|
||||
println()
|
||||
println(title)
|
||||
rule()
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- corpus
|
||||
|
||||
"""
|
||||
make_corpus(cfg, corpus_dir, n) -> Vector{Job}
|
||||
|
||||
Copy up to `n` real files from `corpus_dir` into `known/` and build the `Job`
|
||||
references a stage-2 worker would dequeue for them — the exact input
|
||||
`handle_known_job` sees.
|
||||
|
||||
Real files, not generated ones: exiftool's cost is a function of what it can
|
||||
parse, and a file of random bytes bails out early enough to understate the stage
|
||||
by an order of magnitude. `.meta.json` sidecars are skipped — they are stage-2
|
||||
*output*, and enriching them would measure the wrong population.
|
||||
"""
|
||||
function make_corpus(cfg::FS.Config, corpus_dir::AbstractString, n::Int)
|
||||
isdir(corpus_dir) || error("corpus dir not found: $corpus_dir")
|
||||
names = filter(readdir(corpus_dir)) do f
|
||||
!endswith(f, ".meta.json") && isfile(joinpath(corpus_dir, f))
|
||||
end
|
||||
isempty(names) && error("no usable files in corpus dir: $corpus_dir")
|
||||
sort!(names) # deterministic selection across runs
|
||||
length(names) > n && (names = names[1:n])
|
||||
|
||||
jobs = FS.Job[]
|
||||
for (i, name) in enumerate(names)
|
||||
src = joinpath(corpus_dir, name)
|
||||
# Give it a fresh id/spool-style filename so nothing collides with the
|
||||
# corpus the file came from.
|
||||
id, spooled = FS.spool_path(cfg, @sprintf("s2-%04d-%s", i, basename(name)))
|
||||
cp(src, spooled; force = true)
|
||||
dest = joinpath(cfg.known_dir, basename(spooled))
|
||||
mv(spooled, dest; force = true)
|
||||
push!(jobs, FS.Job(id, basename(name), dest, filesize(dest), time()))
|
||||
end
|
||||
return jobs
|
||||
end
|
||||
|
||||
"""
|
||||
reknown!(cfg, jobs)
|
||||
|
||||
Put every corpus file back in `known/`, wherever the last pass left it (done/ or
|
||||
already home), and delete any sidecar it produced. This is the untimed `prepare`
|
||||
step for benchmarks that consume their input by committing it.
|
||||
"""
|
||||
function reknown!(cfg::FS.Config, jobs::Vector{FS.Job})
|
||||
for job in jobs
|
||||
base = basename(job.path)
|
||||
for dir in (cfg.done_dir, cfg.failed_dir)
|
||||
sidecar = joinpath(dir, string(base, ".meta.json"))
|
||||
rm(sidecar; force = true)
|
||||
rm(string(sidecar, ".tmp"); force = true)
|
||||
end
|
||||
isfile(job.path) && continue
|
||||
for dir in (cfg.done_dir, cfg.failed_dir)
|
||||
candidate = joinpath(dir, base)
|
||||
if isfile(candidate)
|
||||
mv(candidate, job.path; force = true)
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# --------------------------------------------------------------------- loggers
|
||||
|
||||
"""
|
||||
with_logger_named(f, which, path)
|
||||
|
||||
Run `f` under one of the loggers the cost of logging is bracketed by:
|
||||
|
||||
* `:null` — `NullLogger`: the `@info` macro's own overhead, nothing else.
|
||||
* `:format` — `ConsoleLogger` to `devnull`: message formatting and key/value
|
||||
interpolation, but no I/O.
|
||||
* `:flush` — `FlushLogger(ConsoleLogger(io))` to a real file: what
|
||||
`FileServer.run` installs, under the redirect it was written
|
||||
for. Stage 2's per-file line is `@info`, not `@debug`, so this
|
||||
row is what the deployed server actually pays.
|
||||
"""
|
||||
function with_logger_named(f, which::Symbol, path::AbstractString)
|
||||
if which === :null
|
||||
return with_logger(f, NullLogger())
|
||||
elseif which === :format
|
||||
return with_logger(f, ConsoleLogger(devnull))
|
||||
elseif which === :flush
|
||||
return open(path, "w") do io
|
||||
with_logger(f, FS.FlushLogger(ConsoleLogger(io, Logging.Info)))
|
||||
end
|
||||
end
|
||||
error("unknown logger: $which")
|
||||
end
|
||||
|
||||
# --------------------------------------------------- exiftool spawn alternatives
|
||||
|
||||
"""
|
||||
capture(cmd) -> Vector{UInt8}
|
||||
|
||||
Run `cmd` and return its stdout, tolerating a non-zero exit the way
|
||||
`run_with_timeout` does. `read(cmd, String)` would throw instead, and a real
|
||||
corpus makes that a question of when, not whether: exiftool exits 1 on a file
|
||||
whose type it can't recognize, which in this pipeline is a routine outcome (it
|
||||
yields a degraded sidecar, not a failure). This is `run_with_timeout` minus the
|
||||
watchdog, so the gap between the two rows prices the watchdog exactly.
|
||||
"""
|
||||
function capture(cmd::Cmd)
|
||||
out = IOBuffer()
|
||||
proc = Base.run(pipeline(cmd; stdout = out, stderr = devnull); wait = false)
|
||||
wait(proc)
|
||||
return take!(out)
|
||||
end
|
||||
|
||||
"""
|
||||
batched_ns(paths, trials) -> ns_per_file
|
||||
|
||||
Run the whole corpus through *one* `exiftool` process and divide by the file
|
||||
count. This is the floor for "what does exiftool cost if you stop paying the
|
||||
interpreter startup per file" — the fork, the Perl boot and the module loads are
|
||||
paid once for the batch instead of once per file.
|
||||
"""
|
||||
function batched_ns(paths::Vector{String}, trials::Int)
|
||||
return best_of(noop; trials) do
|
||||
SINK[] = capture(`exiftool -json -G -n $paths`)
|
||||
length(paths)
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
stay_open_ns(paths, trials) -> ns_per_file (or nothing if unsupported)
|
||||
|
||||
Feed files one at a time to a single long-lived `exiftool -stay_open True -@ -`
|
||||
process over a pipe, reading its `{ready}` sentinel after each. Unlike the
|
||||
batched row this preserves the pipeline's actual shape — one file in, one result
|
||||
out, arriving whenever it arrives — so it prices the realistic fix rather than
|
||||
an unrealistic one.
|
||||
"""
|
||||
function stay_open_ns(paths::Vector{String}, trials::Int)
|
||||
inp, outp = Pipe(), Pipe()
|
||||
proc = Base.run(pipeline(`exiftool -stay_open True -@ -`;
|
||||
stdin = inp, stdout = outp, stderr = devnull); wait = false)
|
||||
close(inp.out); close(outp.in)
|
||||
|
||||
ask(path) = begin
|
||||
write(inp, "-json\n-G\n-n\n", path, "\n-execute\n")
|
||||
flush(inp)
|
||||
readuntil(outp, "{ready}")
|
||||
end
|
||||
try
|
||||
ask(paths[1]) # pay the one-time process startup untimed
|
||||
return best_of(noop; trials) do
|
||||
for p in paths
|
||||
SINK[] = ask(p)
|
||||
end
|
||||
length(paths)
|
||||
end
|
||||
finally
|
||||
try
|
||||
write(inp, "-stay_open\nFalse\n"); flush(inp); close(inp)
|
||||
wait(proc)
|
||||
catch
|
||||
kill(proc, Base.SIGKILL)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------ components
|
||||
|
||||
"""
|
||||
component_rows(cfg, jobs, opts) -> Vector
|
||||
|
||||
Time each piece of stage 2 on its own. The subprocess rows run once per corpus
|
||||
file (they cost milliseconds and don't need repetition); the in-memory and
|
||||
filesystem rows run `reps` times; the committing rows run once per corpus file
|
||||
with an untimed reset between passes.
|
||||
"""
|
||||
function component_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
|
||||
reps, trials = opts["reps"], opts["trials"]
|
||||
timeout = opts["timeout"]
|
||||
nfiles = length(jobs)
|
||||
paths = String[j.path for j in jobs]
|
||||
rows = []
|
||||
add!(name, part, ns) = push!(rows, (; name, part, ns))
|
||||
|
||||
# --- the bare interpreter: fork/exec + Perl boot, reading no file at all.
|
||||
# Everything the real call does beyond this is actual work.
|
||||
add!("exiftool -ver (spawn)", "extract", best_of(noop; trials) do
|
||||
for _ in 1:nfiles
|
||||
SINK[] = capture(`exiftool -ver`)
|
||||
end
|
||||
nfiles
|
||||
end)
|
||||
|
||||
# --- the real command, run directly: spawn + parse the file, no watchdog.
|
||||
add!("exiftool -json (raw run)", "extract", best_of(noop; trials) do
|
||||
@inbounds for p in paths
|
||||
SINK[] = capture(`exiftool -json -G -n $p`)
|
||||
end
|
||||
nfiles
|
||||
end)
|
||||
|
||||
# --- the same command through the watchdog wrapper the stage actually uses.
|
||||
# The gap to the row above is what the timeout costs.
|
||||
add!("run_with_timeout", "extract", best_of(noop; trials) do
|
||||
@inbounds for p in paths
|
||||
SINK[] = FS.run_with_timeout(`exiftool -json -G -n $p`, timeout)
|
||||
end
|
||||
nfiles
|
||||
end)
|
||||
|
||||
# --- run_exiftool: the wrapper plus JSON3.read plus the group-stripped Dict.
|
||||
add!("run_exiftool (total)", "extract", best_of(noop; trials) do
|
||||
@inbounds for p in paths
|
||||
SINK[] = FS.run_exiftool(p, timeout)
|
||||
end
|
||||
nfiles
|
||||
end)
|
||||
|
||||
# --- what a fork-free exiftool would cost, two ways (see the docstrings).
|
||||
add!("exiftool (batched $(nfiles)x)", "alt", batched_ns(paths, trials))
|
||||
if !opts["no-stay-open"]
|
||||
try
|
||||
add!("exiftool (-stay_open)", "alt", stay_open_ns(paths, trials))
|
||||
catch e
|
||||
@warn "persistent-exiftool probe failed; skipping" exception = e
|
||||
end
|
||||
end
|
||||
|
||||
# --- parsing alone, from bytes already captured: isolates JSON3 from the fork.
|
||||
raw = [FS.run_with_timeout(`exiftool -json -G -n $p`, timeout) for p in paths]
|
||||
valid = [b for b in raw if b !== nothing]
|
||||
if !isempty(valid)
|
||||
add!("JSON3.read (parse dump)", "extract", best_of(noop; trials) do
|
||||
@inbounds for i in 1:reps
|
||||
SINK[] = JSON3.read(String(copy(valid[(i - 1) % length(valid) + 1])))
|
||||
end
|
||||
reps
|
||||
end)
|
||||
end
|
||||
|
||||
# --- normalize_metadata: the ~20 tag coalesces, on a tag map already in memory.
|
||||
bytags = [FS.run_exiftool(p, timeout) for p in paths]
|
||||
good = [(j, b) for (j, b) in zip(jobs, bytags) if b !== nothing]
|
||||
isempty(good) && error("exiftool produced no parseable output for any corpus file")
|
||||
add!("normalize_metadata", "extract", best_of(noop; trials) do
|
||||
@inbounds for i in 1:reps
|
||||
j, b = good[(i - 1) % length(good) + 1]
|
||||
SINK[] = FS.normalize_metadata(j, b)
|
||||
end
|
||||
reps
|
||||
end)
|
||||
|
||||
# The sidecar payloads, built once, untimed: the commit rows below measure
|
||||
# committing, not extracting.
|
||||
metas = [FS.normalize_metadata(j, b) for (j, b) in good]
|
||||
|
||||
# --- serializing the sidecar (the raw dump makes this bigger than it looks).
|
||||
add!("JSON3.write (sidecar)", "commit", best_of(noop; trials) do
|
||||
@inbounds for i in 1:reps
|
||||
SINK[] = JSON3.write(metas[(i - 1) % length(metas) + 1])
|
||||
end
|
||||
reps
|
||||
end)
|
||||
|
||||
# --- sidecar bytes: open + write + flush + fsync, to a temp name.
|
||||
# Non-consuming: same path rewritten each rep, as commit_enriched! does.
|
||||
tmp = joinpath(cfg.done_dir, "bench_stage2_sidecar.tmp")
|
||||
blobs = [JSON3.write(m) for m in metas]
|
||||
# One pass over the real sidecar population, not `reps` of them. Two reasons,
|
||||
# and the first is a correctness trap: thousands of back-to-back fsyncs
|
||||
# saturate the device's write cache and each one starts waiting on the
|
||||
# queue, which reported this row at 16 ms/file — eight times the whole
|
||||
# `commit_enriched!` that contains it. The real stage fsyncs once per file
|
||||
# with ~160 ms of exiftool between, and never queues that way. Second, real
|
||||
# sidecars vary hugely in size (a zip's raw dump dwarfs a jpeg's), so the
|
||||
# honest per-file number is one pass over all of them, not a cycle.
|
||||
nio = length(blobs)
|
||||
add!("write + fsync (sidecar)", "commit", best_of(noop; trials) do
|
||||
@inbounds for i in 1:nio
|
||||
open(tmp, "w") do io
|
||||
write(io, blobs[(i - 1) % length(blobs) + 1])
|
||||
flush(io)
|
||||
FS.fsync_fd(fd(io))
|
||||
end
|
||||
end
|
||||
nio
|
||||
end)
|
||||
rm(tmp; force = true)
|
||||
|
||||
# --- fsync_dir: persisting the rename itself, once per file in the real path.
|
||||
add!("fsync_dir (done/)", "commit", best_of(noop; trials) do
|
||||
for _ in 1:nio
|
||||
SINK[] = FS.fsync_dir(cfg.done_dir)
|
||||
end
|
||||
nio
|
||||
end)
|
||||
|
||||
# --- move_to: the rename known/<f> -> done/<f>. Consuming: reset each pass.
|
||||
add!("move_to (rename)", "commit", best_of(() -> reknown!(cfg, jobs); trials) do
|
||||
@inbounds for job in jobs
|
||||
SINK[] = FS.move_to(cfg.done_dir, job)
|
||||
end
|
||||
nfiles
|
||||
end)
|
||||
|
||||
# --- commit_enriched!: the whole sidecar-first commit, extraction excluded.
|
||||
committable = [j for (j, _) in good]
|
||||
add!("commit_enriched! (total)", "commit",
|
||||
best_of(() -> reknown!(cfg, jobs); trials) do
|
||||
@inbounds for (k, job) in enumerate(committable)
|
||||
SINK[] = FS.commit_enriched!(cfg.done_dir, job, metas[k])
|
||||
end
|
||||
length(committable)
|
||||
end)
|
||||
reknown!(cfg, jobs)
|
||||
|
||||
# --- the one @info line, under each logger.
|
||||
job1, meta1 = good[1][1], metas[1]
|
||||
logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
|
||||
for (which, label) in ((:null, "logging (NullLogger)"),
|
||||
(:format, "logging (format only)"),
|
||||
(:flush, "logging (flush→file)"))
|
||||
ns = with_logger_named(which, logfile) do
|
||||
best_of(noop; trials) do
|
||||
for _ in 1:reps
|
||||
@info "enriched" worker=1 id=job1.id dest=job1.path sidecar="x.meta.json" file_type=meta1.file_type created_by=meta1.created_by degraded=(meta1.error !== nothing)
|
||||
end
|
||||
reps
|
||||
end
|
||||
end
|
||||
add!(label, "log", ns)
|
||||
end
|
||||
rm(logfile; force = true)
|
||||
|
||||
return rows
|
||||
end
|
||||
|
||||
"""
|
||||
handler_rows(cfg, jobs, opts) -> Vector
|
||||
|
||||
Time the real `handle_known_job` end to end under each logger. The difference
|
||||
between the rows is the cost logging adds to a file; the `:flush` row is what the
|
||||
running server actually pays.
|
||||
"""
|
||||
function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
|
||||
trials = opts["trials"]
|
||||
nfiles = length(jobs)
|
||||
logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
|
||||
rows = []
|
||||
for (which, label) in ((:null, "handle_known_job (NullLogger)"),
|
||||
(:format, "handle_known_job (format only)"),
|
||||
(:flush, "handle_known_job (flush→file)"))
|
||||
ns = with_logger_named(which, logfile) do
|
||||
best_of(() -> reknown!(cfg, jobs); trials) do
|
||||
@inbounds for job in jobs
|
||||
FS.handle_known_job(job, cfg, 1)
|
||||
end
|
||||
nfiles
|
||||
end
|
||||
end
|
||||
push!(rows, (; name = label, part = "total", ns))
|
||||
end
|
||||
reknown!(cfg, jobs)
|
||||
rm(logfile; force = true)
|
||||
return rows
|
||||
end
|
||||
|
||||
"""
|
||||
thread_rows(cfg, jobs, opts) -> Vector
|
||||
|
||||
Run the full handler across worker counts, under the server's real logger. Stage
|
||||
2 spends most of its time in a child process, so this is the sweep that matters
|
||||
most: whether the stage scales is a question about the kernel's fork path and the
|
||||
machine's cores, not about Julia.
|
||||
|
||||
Workers pull from a shared atomic counter rather than taking a contiguous slice.
|
||||
That matches the server (its pool pulls from one queue), and it matters here in a
|
||||
way it doesn't for stage 1: per-file exiftool time spans two orders of magnitude
|
||||
on a real corpus — a single 2 s archive among 48 files — so a static split leaves
|
||||
whichever worker drew it running alone while the rest idle, and the sweep would
|
||||
report a scaling ceiling that is really just load imbalance.
|
||||
"""
|
||||
function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
|
||||
trials = opts["trials"]
|
||||
nfiles = length(jobs)
|
||||
counts = opts["threads"] === nothing ?
|
||||
unique([1; 2; 4; 8; Threads.nthreads()]) :
|
||||
[parse(Int, s) for s in split(String(opts["threads"]), ",")]
|
||||
counts = sort(unique(filter(k -> 1 <= k <= Threads.nthreads(), counts)))
|
||||
|
||||
logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
|
||||
rows = []
|
||||
base = 0.0
|
||||
for k in counts
|
||||
ns = with_logger_named(:flush, logfile) do
|
||||
next = Threads.Atomic{Int}(1)
|
||||
best_of(() -> (reknown!(cfg, jobs); next[] = 1); trials) do
|
||||
# Shared counter, not a contiguous slice: every worker takes the
|
||||
# next unclaimed file the moment it frees up, exactly as the
|
||||
# server's pool takes the next job off the known queue.
|
||||
@sync for t in 1:k
|
||||
Threads.@spawn begin
|
||||
while true
|
||||
i = Threads.atomic_add!(next, 1)
|
||||
i > nfiles && break
|
||||
@inbounds FS.handle_known_job(jobs[i], cfg, t)
|
||||
end
|
||||
end
|
||||
end
|
||||
nfiles
|
||||
end
|
||||
end
|
||||
r = rate(ns) # files/sec aggregate (ns is already per file, wall-clock)
|
||||
k == counts[1] && (base = r)
|
||||
push!(rows, (; workers = k, ns, files_per_sec = r, speedup = r / base))
|
||||
end
|
||||
reknown!(cfg, jobs)
|
||||
rm(logfile; force = true)
|
||||
return rows
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- reporting
|
||||
|
||||
function print_components(rows, total_ns)
|
||||
@printf("%-34s %-9s %12s %10s %9s\n", "component", "part", "per file", "rate", "% total")
|
||||
rule()
|
||||
for r in rows
|
||||
@printf("%-34s %-9s %12s %10s %8.1f%%\n", r.name, r.part, human_time(r.ns),
|
||||
human_rate(rate(r.ns)), 100 * r.ns / total_ns)
|
||||
end
|
||||
end
|
||||
|
||||
function print_threads(rows)
|
||||
@printf("%-9s %12s %12s %9s\n", "workers", "per file", "throughput", "speedup")
|
||||
rule()
|
||||
for r in rows
|
||||
@printf("%-9d %12s %12s %8.2fx\n", r.workers, human_time(r.ns),
|
||||
human_rate(r.files_per_sec), r.speedup)
|
||||
end
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------------- main
|
||||
|
||||
function main(argv)
|
||||
opts = parse_args(argv)
|
||||
try
|
||||
FS.assert_exiftool()
|
||||
catch e
|
||||
println(stderr, sprint(showerror, e)); return 1
|
||||
end
|
||||
|
||||
root = opts["dir"] === nothing ?
|
||||
mktempdir(pwd(); prefix = "bench_stage2_") : String(opts["dir"])
|
||||
owned = opts["dir"] === nothing
|
||||
cfg = FS.Config(
|
||||
spool_dir = joinpath(root, "spool"),
|
||||
known_dir = joinpath(root, "known"),
|
||||
done_dir = joinpath(root, "done"),
|
||||
failed_dir = joinpath(root, "failed"),
|
||||
exiftool_timeout = opts["timeout"],
|
||||
)
|
||||
for d in (cfg.spool_dir, cfg.known_dir, cfg.done_dir, cfg.failed_dir)
|
||||
mkpath(d)
|
||||
end
|
||||
|
||||
jobs = try
|
||||
make_corpus(cfg, String(opts["corpus"]), opts["files"])
|
||||
catch e
|
||||
owned && rm(root; recursive = true, force = true)
|
||||
println(stderr, sprint(showerror, e)); return 1
|
||||
end
|
||||
bytes = sum(j.size for j in jobs)
|
||||
|
||||
println("stage-2 component benchmark")
|
||||
rule()
|
||||
@printf("%-22s %s\n", "julia threads", Threads.nthreads())
|
||||
@printf("%-22s %s\n", "exiftool", strip(read(`exiftool -ver`, String)))
|
||||
@printf("%-22s %s\n", "corpus", "$(length(jobs)) files ($(round(bytes / 1024^2; digits=1)) MiB) from $(opts["corpus"])")
|
||||
@printf("%-22s %s\n", "scratch", root)
|
||||
@printf("%-22s %s\n", "reps / trials", "$(opts["reps"]) / $(opts["trials"])")
|
||||
@printf("%-22s %s\n", "exiftool timeout", "$(opts["timeout"]) s")
|
||||
|
||||
try
|
||||
comps = component_rows(cfg, jobs, opts)
|
||||
handlers = handler_rows(cfg, jobs, opts)
|
||||
|
||||
# The denominator is the handler as the server actually runs it: the real
|
||||
# flushing logger, one worker. Percentages are shares of that, so they are
|
||||
# directly comparable and the parts can be checked against the whole.
|
||||
total = only(r.ns for r in handlers if r.name == "handle_known_job (flush→file)")
|
||||
|
||||
header("Components (single worker)")
|
||||
print_components(comps, total)
|
||||
|
||||
header("Whole handler, by logger")
|
||||
print_components(handlers, total)
|
||||
|
||||
pick(name) = only(r.ns for r in comps if r.name == name)
|
||||
accounted = pick("run_exiftool (total)") + pick("commit_enriched! (total)") +
|
||||
pick("logging (flush→file)")
|
||||
println()
|
||||
@printf("accounted: %s of %s (%.0f%%); unaccounted overhead %s\n",
|
||||
human_time(accounted), human_time(total), 100 * accounted / total,
|
||||
human_time(max(total - accounted, 0)))
|
||||
|
||||
threads = nothing
|
||||
if !opts["no-threads"] && Threads.nthreads() > 1
|
||||
threads = thread_rows(cfg, jobs, opts)
|
||||
header("Full handler across workers (server logger)")
|
||||
print_threads(threads)
|
||||
end
|
||||
|
||||
if opts["json"] !== nothing
|
||||
open(String(opts["json"]), "w") do io
|
||||
JSON3.write(io, (;
|
||||
julia_threads = Threads.nthreads(),
|
||||
files = length(jobs), corpus_bytes = bytes,
|
||||
reps = opts["reps"], trials = opts["trials"],
|
||||
exiftool_timeout = opts["timeout"],
|
||||
components = comps, handlers, threads,
|
||||
))
|
||||
end
|
||||
println("\nwrote ", opts["json"])
|
||||
end
|
||||
finally
|
||||
owned && rm(root; recursive = true, force = true)
|
||||
end
|
||||
return 0
|
||||
end
|
||||
|
||||
exit(main(ARGS))
|
||||
217
bin/cluster_calibrate.jl
Normal file
217
bin/cluster_calibrate.jl
Normal file
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# Phase-A calibration for stage-5 header clustering (model/DESIGN_clustering.md
|
||||
# §7). Runs labeled known files through the exact clustering pipeline, scores the
|
||||
# recovered partition against magic-collapsed ground truth with ARI / V-measure,
|
||||
# grid-tunes (α, β, bg_mass, n), and cross-checks the winning config against a
|
||||
# model-free NCD (gzip) baseline (§8). The settings printed here are the ones the
|
||||
# machine rediscovers known formats at — copy the winner into config.jl.
|
||||
#
|
||||
# julia --project=. bin/cluster_calibrate.jl [training_set_dir]
|
||||
#
|
||||
# Defaults to ../training_set. Prints a report; writes nothing.
|
||||
|
||||
using Random
|
||||
using Printf
|
||||
|
||||
include(joinpath(@__DIR__, "..", "src", "cluster.jl"))
|
||||
|
||||
# --- ground truth: magic-collapsed classes, NOT extensions (DESIGN §7.2) -----
|
||||
|
||||
"""
|
||||
truth_label(path) -> String
|
||||
|
||||
The magic-collapsed format class of a file, read from its actual bytes (so
|
||||
docx≡zip and the whole ELF family merge, exactly the answer we want the
|
||||
clustering to reproduce). `tar` is detected by the `ustar` magic at offset 257 —
|
||||
outside the model's front window, so tars are the accepted blind spot that
|
||||
scatters to background.
|
||||
"""
|
||||
function truth_label(path::AbstractString)
|
||||
b = zeros(UInt8, 262)
|
||||
open(path) do io
|
||||
chunk = read(io, 262)
|
||||
copyto!(b, 1, chunk, 1, length(chunk))
|
||||
end
|
||||
b[1] == 0x1f && b[2] == 0x8b && return "gzip"
|
||||
b[1] == 0x50 && b[2] == 0x4b && return "pkzip"
|
||||
b[1] == 0x25 && b[2] == 0x50 && b[3] == 0x44 && b[4] == 0x46 && return "pdf"
|
||||
b[1] == 0xff && b[2] == 0xd8 && b[3] == 0xff && return "jpeg"
|
||||
b[1] == 0x7f && b[2] == 0x45 && b[3] == 0x4c && b[4] == 0x46 && return "elf"
|
||||
(b[258] == 0x75 && b[259] == 0x73 && b[260] == 0x74 && b[261] == 0x61 && b[262] == 0x72) && return "tar"
|
||||
return "other"
|
||||
end
|
||||
|
||||
# --- NCD (Normalized Compression Distance) baseline, model-free (DESIGN §8) ---
|
||||
|
||||
"gzip-compressed size of a byte buffer, via the gzip CLI (no CodecZlib dep)."
|
||||
function gz_size(bytes::Vector{UInt8})
|
||||
out = IOBuffer()
|
||||
open(pipeline(`gzip -c`; stdout=out); write=true) do io
|
||||
write(io, bytes)
|
||||
end
|
||||
return length(take!(out))
|
||||
end
|
||||
|
||||
"NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)) — 0 = identical, ~1 = unrelated."
|
||||
function ncd(xb, yb, cx, cy)
|
||||
cxy = gz_size(vcat(xb, yb))
|
||||
return (cxy - min(cx, cy)) / max(cx, cy)
|
||||
end
|
||||
|
||||
"""
|
||||
ncd_1nn_purity(paths, truth; head_bytes) -> Float64
|
||||
|
||||
Fraction of files whose NCD-nearest neighbour shares its true label — a cheap,
|
||||
O(N²) sanity read on how well raw gzip-similarity alone separates formats on the
|
||||
same input. The Bayesian clusters should broadly agree; a big gap is a red flag
|
||||
(DESIGN §10.3). Uses each file's first `head_bytes` so the giant files don't
|
||||
dominate compression time.
|
||||
"""
|
||||
function ncd_1nn_purity(paths::Vector{String}, truth::Vector{String}; head_bytes::Int=4096)
|
||||
bufs = map(paths) do p
|
||||
open(io -> read(io, head_bytes), p)
|
||||
end
|
||||
csz = gz_size.(bufs)
|
||||
N = length(paths)
|
||||
correct = 0
|
||||
for i in 1:N
|
||||
best_j = 0; best_d = Inf
|
||||
for j in 1:N
|
||||
i == j && continue
|
||||
d = ncd(bufs[i], bufs[j], csz[i], csz[j])
|
||||
if d < best_d
|
||||
best_d = d; best_j = j
|
||||
end
|
||||
end
|
||||
best_j != 0 && truth[best_j] == truth[i] && (correct += 1)
|
||||
end
|
||||
return correct / N
|
||||
end
|
||||
|
||||
"1-NN label purity of a *cluster* assignment vs truth (same yardstick as NCD's)."
|
||||
function cluster_1nn_purity(pred::Vector{Int}, truth::Vector{String})
|
||||
# For each file, its 'nearest neighbour' is any other file in the same
|
||||
# cluster; purity = P(a random same-cluster neighbour shares the true label).
|
||||
groups = Dict{Int,Vector{Int}}()
|
||||
for (i, k) in enumerate(pred)
|
||||
push!(get!(groups, k, Int[]), i)
|
||||
end
|
||||
correct = 0; total = 0
|
||||
for (_, idxs) in groups
|
||||
length(idxs) < 2 && continue
|
||||
for i in idxs
|
||||
same = count(j -> j != i && truth[j] == truth[i], idxs)
|
||||
total += 1
|
||||
same > 0 && (correct += 1)
|
||||
end
|
||||
end
|
||||
return total == 0 ? 0.0 : correct / total
|
||||
end
|
||||
|
||||
# --- data ---------------------------------------------------------------------
|
||||
|
||||
function load_corpus(dir::AbstractString)
|
||||
paths = String[]
|
||||
for name in readdir(dir; join=true)
|
||||
isfile(name) && push!(paths, name)
|
||||
end
|
||||
truth = truth_label.(paths)
|
||||
return paths, truth
|
||||
end
|
||||
|
||||
# --- grid search --------------------------------------------------------------
|
||||
|
||||
function evaluate(X, truth; α, β, bg_mass, sweeps, restarts, seed)
|
||||
r = gibbs_cluster(X; α=α, β=β, bg_mass=bg_mass, sweeps=sweeps,
|
||||
restarts=restarts, rng=MersenneTwister(seed))
|
||||
pred = r.assignments
|
||||
ari = adjusted_rand_index(truth, pred)
|
||||
keep = truth .!= "tar"
|
||||
ari_notar = adjusted_rand_index(truth[keep], pred[keep])
|
||||
v, h, comp = v_measure(truth, pred)
|
||||
return (; ari, ari_notar, v, h, comp, k=length(r.clusters),
|
||||
bg=count(==(0), pred), result=r)
|
||||
end
|
||||
|
||||
function main()
|
||||
dir = length(ARGS) >= 1 ? ARGS[1] : joinpath(@__DIR__, "..", "..", "training_set")
|
||||
isdir(dir) || error("training set dir not found: $dir")
|
||||
paths, truth = load_corpus(dir)
|
||||
classes = sort(unique(truth))
|
||||
counts = [(c, count(==(c), truth)) for c in classes]
|
||||
@printf("corpus: %d files from %s\n", length(paths), dir)
|
||||
println("magic-collapsed truth classes: ", join(["$c=$n" for (c, n) in counts], " "))
|
||||
println()
|
||||
|
||||
sweeps = 150
|
||||
restarts = 6
|
||||
seed = 20260703
|
||||
|
||||
# Grid. n is expensive to re-featurize, so loop it outermost. Ranges are
|
||||
# centred where the coarse sweep found the optimum: small β (peaked
|
||||
# per-position priors) is what separates formats whose headers differ in only
|
||||
# a few magic bytes; large β over-merges. bg_mass barely moves the result
|
||||
# here (almost nothing lands in background on this corpus), so it is fixed.
|
||||
αs = [1.0, 2.0]
|
||||
βs = [0.05, 0.08, 0.1, 0.15, 0.2]
|
||||
bgs = [5.0]
|
||||
ns = [32, 64]
|
||||
|
||||
println("grid search (sweeps=$sweeps, restarts=$restarts):")
|
||||
@printf(" %-4s %-5s %-5s %-6s | %-6s %-8s %-6s %-6s %-6s %-4s %-4s\n",
|
||||
"n", "alpha", "beta", "bgmss", "ARI", "ARI-tar", "V", "homog", "compl", "k", "bg")
|
||||
results = Vector{Any}()
|
||||
for n in ns
|
||||
X = header_matrix(paths; n=n)
|
||||
for α in αs, β in βs, bg in bgs
|
||||
e = evaluate(X, truth; α=α, β=β, bg_mass=bg, sweeps=sweeps, restarts=restarts, seed=seed)
|
||||
push!(results, (; n, α, β, bg, e))
|
||||
@printf(" %-4d %-5.1f %-5.2f %-6.1f | %-6.3f %-8.3f %-6.3f %-6.3f %-6.3f %-4d %-4d\n",
|
||||
n, α, β, bg, e.ari, e.ari_notar, e.v, e.h, e.comp, e.k, e.bg)
|
||||
end
|
||||
end
|
||||
|
||||
# Rank by ARI-excluding-tar (tar is the accepted blind spot; scoring it would
|
||||
# penalise the correct answer of scattering tars to background — DESIGN §7.2).
|
||||
sort!(results; by=r -> r.e.ari_notar, rev=true)
|
||||
best = results[1]
|
||||
println()
|
||||
@printf("BEST (by ARI excl. tar): n=%d α=%.1f β=%.2f bg_mass=%.1f\n",
|
||||
best.n, best.α, best.β, best.bg)
|
||||
@printf(" ARI=%.3f ARI(excl tar)=%.3f V=%.3f homogeneity=%.3f completeness=%.3f clusters=%d background=%d\n",
|
||||
best.e.ari, best.e.ari_notar, best.e.v, best.e.h, best.e.comp, best.e.k, best.e.bg)
|
||||
|
||||
# Per-cluster composition of the winning partition, and promotion nominations.
|
||||
pred = best.e.result.assignments
|
||||
println("\nwinning partition — cluster composition (truth breakdown):")
|
||||
for (id, c) in sort(collect(best.e.result.clusters); by=x -> -x[2].members)
|
||||
members = [truth[i] for i in eachindex(pred) if pred[i] == id]
|
||||
comp = sort([(l, count(==(l), members)) for l in unique(members)]; by=x -> -x[2])
|
||||
sig = signature(c)
|
||||
promo = is_promotable(c, sig; min_members=20, min_magic=3) ? " ✓NOMINATE" : ""
|
||||
@printf(" cluster %-4d n=%-3d magic=%-2d %s%s\n",
|
||||
id, c.members, magic_positions(sig),
|
||||
join(["$l:$k" for (l, k) in comp], " "), promo)
|
||||
end
|
||||
nbg = count(==(0), pred)
|
||||
bg_truth = [truth[i] for i in eachindex(pred) if pred[i] == 0]
|
||||
bgc = sort([(l, count(==(l), bg_truth)) for l in unique(bg_truth)]; by=x -> -x[2])
|
||||
@printf(" background n=%-3d %s\n", nbg, join(["$l:$k" for (l, k) in bgc], " "))
|
||||
|
||||
# NCD baseline cross-check on a subsample (O(N²), so keep it small).
|
||||
println("\nNCD (gzip) baseline cross-check:")
|
||||
subn = min(150, length(paths))
|
||||
sub = shuffle(MersenneTwister(seed), collect(1:length(paths)))[1:subn]
|
||||
subpaths = paths[sub]; subtruth = truth[sub]
|
||||
ncd_pur = ncd_1nn_purity(subpaths, subtruth)
|
||||
Xsub = header_matrix(subpaths; n=best.n)
|
||||
rsub = gibbs_cluster(Xsub; α=best.α, β=best.β, bg_mass=best.bg,
|
||||
sweeps=sweeps, restarts=restarts, rng=MersenneTwister(seed))
|
||||
bay_pur = cluster_1nn_purity(rsub.assignments, subtruth)
|
||||
@printf(" subsample=%d NCD 1-NN label purity=%.3f Bayesian same-cluster purity=%.3f\n",
|
||||
subn, ncd_pur, bay_pur)
|
||||
println(" (both high ⇒ header-byte signal agrees with model-free gzip similarity — DESIGN §10.3)")
|
||||
end
|
||||
|
||||
main()
|
||||
19
bin/cluster_sweep.jl
Normal file
19
bin/cluster_sweep.jl
Normal file
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# Stage-5 phase-B runner (model/DESIGN_clustering.md §9): the single-owner,
|
||||
# periodic/cron process that sweeps `binary/`, folds new files into the durable
|
||||
# format catalog by sequential CRP-predictive assignment, and (re)writes
|
||||
# promotion nominations. Run it single-threaded on a schedule — it is the ONLY
|
||||
# writer of the catalog, so no locking is needed.
|
||||
#
|
||||
# julia --project=. bin/cluster_sweep.jl # incremental live sweep
|
||||
# julia --project=. bin/cluster_sweep.jl --compact # offline Gibbs (seed / recompact)
|
||||
#
|
||||
# On a fresh catalog (nothing processed yet) the incremental sweep would send
|
||||
# every file to background — there are no clusters to match — so the first run
|
||||
# auto-promotes to a compaction pass to seed the catalog. Configure via the
|
||||
# FS_CLUSTER_* / FS_NOMINATED_DIR env vars (see src/config.jl).
|
||||
|
||||
using FileServer
|
||||
|
||||
FileServer.cluster_sweep_cli(ARGS)
|
||||
314
model/DESIGN_clustering.md
Normal file
314
model/DESIGN_clustering.md
Normal file
@@ -0,0 +1,314 @@
|
||||
# Stage-5: Unknown-format discovery by Bayesian header clustering
|
||||
|
||||
Status: **phases A and B implemented and calibrated** (`src/cluster.jl` +
|
||||
`src/catalog.jl`, `bin/cluster_calibrate.jl` + `bin/cluster_sweep.jl`, tests in
|
||||
`test/runtests.jl`). Phase A (offline Gibbs) is calibrated; phase B's durable
|
||||
single-owner catalog, incremental sweep, and nomination writer are now built on
|
||||
top of the `assign_file` scoring core. Product
|
||||
of a design interview; captures the decisions and — as important — the
|
||||
assumptions we *rejected* so they don't get silently reintroduced. §11 records
|
||||
what building it actually taught us, including three assumptions in this document
|
||||
that the data corrected.
|
||||
|
||||
## 1. Goal
|
||||
|
||||
Discover **recurring new file formats** hiding in the `binary/` bucket (the
|
||||
`:unknown` sink from `classify.jl` → stage-3 triage). A genuinely novel format is
|
||||
a plausible proxy for a genuinely novel producing application, but we do **not**
|
||||
try to identify producers directly (see §3). The output is a **growing catalog of
|
||||
discovered formats**, each with a magic-byte signature that can be promoted into
|
||||
the classifier's fast path.
|
||||
|
||||
Task shape (settled): **unsupervised clustering with an unknown number of
|
||||
clusters.** Not pairwise "same producer" scoring, not classification against a
|
||||
fixed label set.
|
||||
|
||||
## 2. Two phases — build (A) then run (B)
|
||||
|
||||
**(A) Batch, offline — the science.** Cluster the accumulated pile from scratch.
|
||||
Its job is *not* to be the catalog; it is to (i) prove the header-byte signal
|
||||
actually separates formats, cross-checked against an NCD baseline (§8), and
|
||||
(ii) **calibrate hyperparameters** against known formats (§7). Ship this first —
|
||||
it de-risks (B). If (A)'s clusters are garbage, (B)'s machinery is wasted.
|
||||
|
||||
**(B) Online, live — the catalog.** The target deliverable. A persistent catalog
|
||||
where each discovered format has a **durable, frozen ID** and stored sufficient
|
||||
statistics. New unknown files are scored against existing clusters; only
|
||||
genuinely novel ones spawn a new entry. Clusters that accumulate enough evidence
|
||||
are **nominated for promotion** into the classifier (§6).
|
||||
|
||||
## 3. What we are and are NOT clustering
|
||||
|
||||
We cluster by **file format**, not by producer. The first-*n* header bytes are
|
||||
format-mandated and producer-invariant: every valid PNG shares the same magic
|
||||
regardless of which program wrote it; a PDF's producer string lives deep inside
|
||||
the file, not in the header. Producer identity, where recoverable at all, is
|
||||
`exiftool`'s job (stage 2), not this stage's.
|
||||
|
||||
Corollary already visible in `../training_set`: extension labels are **not**
|
||||
header-format labels. `docx` *is* a PK zip; `so`/`o`/`elf`/`out` are all ELF.
|
||||
Merging those is **correct**, not error (see §7).
|
||||
|
||||
## 4. Model: DP mixture of per-position categoricals
|
||||
|
||||
A cluster is a **product of independent per-position categorical distributions**
|
||||
over the first *n* header bytes. Position *i* carries a distribution `θᵢ` over a
|
||||
**257-symbol alphabet**: byte values `0–255`, plus symbol `256 = "past EOF"`.
|
||||
|
||||
- Invariant positions (magic bytes) learn a spiked `θᵢ`; variable positions
|
||||
(lengths, timestamps) learn a flat one. A cluster's signature = the vector of
|
||||
modal symbols + per-position peakedness. That signature **is a magic-number
|
||||
template** — this is the entire reason for the categorical choice.
|
||||
- `257` alphabet handles short files honestly: a format that is always 20 bytes
|
||||
produces a spiked "past-EOF" at positions 20–31, which is real, discriminative
|
||||
signal. No zero-padding (would collide `0x00` padding with real `0x00` bytes).
|
||||
|
||||
**Priors:** Dirichlet on each `θᵢ` (conjugate to Categorical); **Dirichlet
|
||||
process (CRP)** over cluster assignments → unknown *k* falls out natively.
|
||||
|
||||
**Why categorical, not Euclidean.** Bytes are categorical, not ordinal: `0x89`
|
||||
and `0x88` are not "close," `0x00` and `0xFF` are not "far." k-means / Gaussian
|
||||
mixtures over scaled bytes assert a metric that does not exist in header space.
|
||||
**Do not reuse `model.jl`'s `[0,1]` byte scaling here** — that scaling is correct
|
||||
for the Lux net and wrong for this model. We need the raw `0–255` byte as a
|
||||
categorical index.
|
||||
|
||||
### 4a. Background component (high-entropy handling)
|
||||
|
||||
Add a fixed, **non-adaptive uniform component** (each position uniform over 257)
|
||||
as the "junk drawer." Compressed/encrypted/structureless blobs are ~uniform after
|
||||
any magic and would otherwise either (i) mint a singleton per file or (ii)
|
||||
collapse into one flat cluster that then matches everything. The background
|
||||
absorbs them cleanly.
|
||||
|
||||
Two populations, to be precise:
|
||||
- **Structured prefix + random tail** (gzip `1f 8b`, PK zip, zstd, most encrypted
|
||||
*containers*): peaked at positions 0–3, flat after. These form **real clusters
|
||||
for free** — genuine discoveries, no special handling.
|
||||
- **Uniform from byte 0** (raw encrypted streams, key material): nothing in the
|
||||
header to cluster on → absorbed by background.
|
||||
|
||||
The background is **never promotable**. But it is **not a silent sink**: its
|
||||
size / growth / entropy histogram is surfaced as a first-class signal ("12% of
|
||||
this week's unknowns are structureless"). If sub-clustering the structureless
|
||||
residue ever matters, that needs a *different* feature (byte histogram / entropy),
|
||||
a separate v3 model — header bytes genuinely cannot do it.
|
||||
|
||||
### 4b. Feature window
|
||||
|
||||
**Front-only, `n = 32`** (config knob; try 64 if under-resolved). Magic lives at
|
||||
offset 0. Tail window **deferred to v2** — a minority of formats have trailers
|
||||
(ZIP EOCD, ID3v1, PDF `%%EOF`); add as an independent *second block* of positions
|
||||
only if real trailer-formats show up in the residue.
|
||||
|
||||
**Known blind spot: tar.** `ustar` magic is at **offset 257**, outside the
|
||||
window, so all 100 training tars scatter to background. Accepted for v1 — tar is
|
||||
already a *known* format, so discovery doesn't need it. General lesson: a minority
|
||||
of formats put magic at a fixed deeper offset; the fix (if ever needed) is a
|
||||
**sparse probe window** at that offset (e.g. bytes 257–262 as a third block), not
|
||||
densely modeling 257 front bytes — that would 8× every cluster's `n×257`
|
||||
sufficient-stat table to catch one format.
|
||||
|
||||
## 5. Inference: different mode per phase (resolves the Bayesian-vs-catalog tension)
|
||||
|
||||
A sampler yields a *posterior over partitions*; a catalog needs *one partition
|
||||
with durable IDs*. Two MCMC gotchas: **label switching** (cluster #3 is not a
|
||||
stable identity across iterations/runs) and **distribution-not-answer** (1000
|
||||
partitions, not one). We sidestep both by using two inference modes:
|
||||
|
||||
- **Phase (A), offline:** full **collapsed Gibbs** sampler over the
|
||||
Dirichlet-Categorical (conjugacy → ~100 lines, no continuous approximation,
|
||||
unknown *k* native). Used to validate signal, tune `α` + Dirichlet strength,
|
||||
and seed the initial catalog (summarize to a point partition **once**, via a
|
||||
VI/Binder loss over the posterior similarity matrix — tolerated because it is
|
||||
offline, never in the hot path).
|
||||
- **Phase (B), live:** **deterministic sequential CRP-predictive assignment.**
|
||||
Each catalog cluster stores per-position 257-count vectors (sufficient stats).
|
||||
A new file's CRP predictive probability of joining each existing cluster vs.
|
||||
the background vs. spawning a new cluster is computed; assign to the argmax.
|
||||
A new cluster is minted only if the new-cluster evidence beats the background
|
||||
by a margin. **IDs are frozen at birth → no label switching.** This is exactly
|
||||
the Gibbs predictive rule with existing assignments held fixed — same math, not
|
||||
an ad-hoc hack.
|
||||
- **Periodic compaction, offline:** re-run Gibbs seeded from the current catalog
|
||||
to merge drifted clusters / split bloated ones.
|
||||
|
||||
## 6. Promotion (closing the loop to the classifier)
|
||||
|
||||
**Layered known-check at ingest** becomes:
|
||||
1. Match against **promoted signatures** (exact, fast) — runs *before* the net.
|
||||
2. Else the Lux `:known` / `:unknown` classifier.
|
||||
3. Else route to `binary/` for this stage.
|
||||
|
||||
**Promotion = append a magic-byte signature to a registry.** A cluster's spiked
|
||||
positions (posterior max-prob `> ~0.9`) become required bytes; flat positions
|
||||
become wildcards — a libmagic-style signature. This is a **data change, not a
|
||||
retrain**; interpretable, auditable, reversible. Retraining the Lux net is a
|
||||
separate, *optional periodic* activity using accumulated signature-labeled files,
|
||||
never the promotion mechanism itself.
|
||||
|
||||
**Nominate automatically, activate by hand.** A cluster crossing thresholds —
|
||||
`≥ N` members (start `N ≈ 20–50`, loose dial since a human is the backstop) **and**
|
||||
`≥ ~3` magic positions **and** not the background — is written to a `nominated/`
|
||||
registry with its signature, member count, and example files. A human glance
|
||||
promotes it into the active set. Human gate guards the one hard-to-reverse action
|
||||
(redefining "known"); everything upstream stays automatic.
|
||||
|
||||
## 7. Calibration: recover known formats, then trust on unknowns
|
||||
|
||||
Do not pick priors blind. We have ground truth: `../training_set` (100 each of
|
||||
tgz/tar/pdf/docx, 98 zip, 93 jpg, ELF family) and the `data/done` corpus.
|
||||
|
||||
1. Run **labeled known files** through the exact clustering pipeline.
|
||||
2. Ground truth = **magic-collapsed classes**, *not* extensions:
|
||||
`{gzip (tgz), PKzip (docx≡zip), ELF (so/o/elf/out/x86_64), JPEG, PDF, tar}`.
|
||||
Merging docx+zip and the ELF family is the **correct** answer — scoring
|
||||
against raw extensions would penalize correctness and mistune `α`.
|
||||
3. Measure recovered-vs-truth agreement with **Adjusted Rand Index / V-measure**.
|
||||
4. **Grid-tune `α` and the Dirichlet pseudocount to maximize agreement** — the
|
||||
settings at which the machine rediscovers formats we already know.
|
||||
5. Freeze, deploy on the `:unknown` pile.
|
||||
|
||||
Splitting docx from zip is a **later tier**: the discriminating info
|
||||
(central-directory filenames like `word/document.xml`) sits at a *variable
|
||||
offset*, not a fixed position — a different feature problem, deferred.
|
||||
|
||||
## 8. Julia package surface
|
||||
|
||||
- **Hand-rolled collapsed Gibbs** over Dirichlet-Categorical — recommended. The
|
||||
conjugacy makes it short/fast; we own the online + promotion logic; no library
|
||||
impedance. `Distributions.jl` for `Dirichlet`/`Categorical` primitives.
|
||||
- **`CodecZlib`** for the **NCD (Normalized Compression Distance)** baseline —
|
||||
model-free gzip-similarity clustering. Excellent at format grouping and a great
|
||||
step-(A) sanity check, but O(N²), non-generative (no signature, no online
|
||||
scoring, no promotion) → **baseline only, cannot be the catalog.**
|
||||
- **`Clustering.jl`** — `randindex` / `vmeasure` for the §7 calibration metric,
|
||||
plus a throwaway k-modes-ish baseline. **Not** the real model (its k-means is
|
||||
the Euclidean trap of §4).
|
||||
- **`Turing.jl`** — considered and rejected: discrete assignment latents + DP are
|
||||
awkward, and we'd still hand-roll the online path. Overkill.
|
||||
|
||||
## 9. Architecture: single-owner batch stage, NOT inline inference
|
||||
|
||||
The classifier is stateless, immutable, shared read-only across worker threads
|
||||
(see `classify.jl`). **The catalog is the opposite: mutable, learned, shared** —
|
||||
every assigned file updates a cluster's counts. It therefore must **not** copy the
|
||||
classifier's concurrency model (concurrent workers → lock contention, torn reads
|
||||
of sufficient stats, CRP assignment against stale mass).
|
||||
|
||||
Because **promotion is human-gated, nothing here is latency-sensitive.** So:
|
||||
|
||||
- Workers stay stateless — they deposit `:unknown` files into `binary/` exactly as
|
||||
today. **No catalog access on the hot path.**
|
||||
- A **separate stage-5 process** (periodic / cron, single-threaded) owns the
|
||||
catalog **exclusively**: sweeps newly-arrived `binary/` files, runs sequential
|
||||
CRP-predictive assignment, updates sufficient stats, writes nominations.
|
||||
**One writer, zero locks, no cross-thread shared mutable state.**
|
||||
- The catalog is a **durable file** mutated by one process — reuse the stage-2
|
||||
**sidecar-first durable-commit** discipline (`commit_enriched!`: temp write →
|
||||
fsync bytes → rename → fsync dir) so a crash can't corrupt it or lose a rename.
|
||||
|
||||
This slots in as a batch stage, matching how stages 2/3/4 already work. New
|
||||
`Config` knobs follow the existing `FS_*` env-override convention (e.g.
|
||||
`FS_CLUSTER_DIR`, `FS_CLUSTER_N`, `FS_CLUSTER_ALPHA`, `FS_CLUSTER_PSEUDOCOUNT`,
|
||||
`FS_PROMOTE_MIN_MEMBERS`).
|
||||
|
||||
## 10. Concrete test assertions (write these first)
|
||||
|
||||
1. **Discovers nothing from noise.** Current `data/binary` = 20 small random
|
||||
blobs + 1 giant PDF. Correct output: PDF is a singleton that **never promotes**
|
||||
(N=1), 20 blobs absorbed by background, **zero promoted clusters.** Any
|
||||
promoted cluster from this pile = broken.
|
||||
2. **Recovers known formats.** On a `../training_set` sample, calibrated settings
|
||||
cluster into the ~6 magic-collapsed classes with high ARI (docx+zip merged,
|
||||
ELF family merged, tar in background as the accepted blind spot).
|
||||
3. **NCD agreement.** Step-(A) Bayesian clusters broadly agree with the NCD
|
||||
baseline on the same input; large disagreement is a red flag to investigate
|
||||
before trusting the generative model.
|
||||
|
||||
## 11. Implementation status & calibration results (v1)
|
||||
|
||||
**Shipped.** `src/cluster.jl` — feature extraction (`header_symbols`, 257-symbol
|
||||
alphabet), collapsed Gibbs (`gibbs_cluster`, phase A), the sequential
|
||||
CRP-predictive rule (`assign_file`, phase B core), signatures/promotion
|
||||
(`signature`, `is_promotable`), and calibration metrics (`adjusted_rand_index`,
|
||||
`v_measure`). All base-Julia — a base-only Lanczos `loggamma` keeps the
|
||||
Dirichlet-multinomial marginal dependency-free (no Manifest churn). Config knobs
|
||||
`FS_CLUSTER_*` (§9) added. `bin/cluster_calibrate.jl` runs the §7 grid and the §8
|
||||
NCD baseline. Concrete §10 assertions are in the test suite (hermetic synthetic
|
||||
corpora, so they need neither `../training_set` nor gzip).
|
||||
|
||||
**Calibrated defaults** (grid over the 700-file `training_set`, ranked by ARI
|
||||
excluding tar): **n=32, α=1.0, β=0.1, bg_mass=5.0** → ARI **0.77** (0.885 excl.
|
||||
tar), V-measure 0.83, homogeneity 0.87. Clusters are clean and promotable:
|
||||
`pkzip:197` (docx+zip correctly merged, §7.2 ✓), `gzip:100`, `jpeg`, and several
|
||||
`pdf` clusters all self-nominate. The **NCD baseline agrees** (§10.3): on a
|
||||
150-file subsample, NCD 1-NN label purity 0.90 vs. the model's same-cluster
|
||||
purity 0.987 — the generative header model separates formats at least as well as
|
||||
model-free gzip similarity.
|
||||
|
||||
### Three assumptions the data corrected
|
||||
|
||||
1. **Tar is not in the background here; it merges into ELF.** §4b/§7 assumed
|
||||
tar's `ustar`-at-257 magic is out of window so tars scatter to background. But
|
||||
98/100 tars in the corpus are Hex/Elixir package tarballs whose *first
|
||||
archived file is named `VERSION`* → a constant, strongly-peaked `VERSION\0`
|
||||
prefix at offset 0. They do form a peaked cluster — but it **merges with ELF**,
|
||||
because ELF's ident padding and tar's name-field zero-padding give the two a
|
||||
long shared run of `0x00` in bytes 5–31; they differ in only ~3 magic bytes,
|
||||
and 32 equally-weighted positions let ~20 shared zeros outvote 3 real ones. No
|
||||
β both separates ELF/tar and keeps the other formats whole. The honest v1
|
||||
position: this is the *same* "tar is hard" reality §4b flagged, just wearing a
|
||||
different mask. **Fix (v2):** weight positions by inverse entropy so a
|
||||
low-information shared-zero run stops dominating a few high-information magic
|
||||
bytes — this generalizes beyond tar and is the highest-value next lever.
|
||||
|
||||
2. **You cannot cold-start every point in the background.** A natural reading of
|
||||
§4a/§5 is "everything starts in the junk drawer, real clusters condense out."
|
||||
That **deadlocks**: at a format's first file a fresh cluster and the background
|
||||
are *both* uniform, so with the `bg_mass ≥ α` that §4a needs for absorption,
|
||||
the background always wins and no cluster is ever seeded. Fix: **initialize
|
||||
every file in its own singleton**; same-format singletons merge and snowball,
|
||||
while a lone random-blob singleton dissolves on resample and is reclaimed by
|
||||
the (stickier) background. Absorption still works — just not as the *initial*
|
||||
state.
|
||||
|
||||
3. **Two pieces of math that look optional but aren't.** (a) Signature peakedness
|
||||
is a **Bernoulli** question ("is this position fixed to byte v?"), so it uses a
|
||||
2-way posterior `(count+β)/(members+2β)`, **not** the 257-way mixture
|
||||
predictive — the alphabet-wide denominator drags even a unanimous position
|
||||
below 0.9 once β<1, which would make promotion *impossible*. (b) Ranking Gibbs
|
||||
restarts needs the **collapsed Dirichlet-multinomial marginal** (with its
|
||||
`loggamma` normalizer / Occam penalty); a plain product-of-predictives score
|
||||
omits the penalty and actively **rewards merging** everything into one blob
|
||||
(observed, then fixed).
|
||||
|
||||
### Known v1 limitations (accepted)
|
||||
|
||||
- **β=0.1 over-splits** PDF and JPEG into several *pure* sub-clusters (e.g. PDF by
|
||||
version byte). This costs completeness/ARI but not the mission: each sub-cluster
|
||||
still carries valid magic and promotes independently, and a human dedupes
|
||||
overlapping `%PDF-1.x` nominations at the gate.
|
||||
- The point partition is the **best of N Gibbs restarts by marginal likelihood**,
|
||||
a MAP-style stand-in for the VI/Binder posterior summary §5 defers — adequate
|
||||
because the formats are strongly separated; revisit if compaction (§5) needs it.
|
||||
- Phase B's **live single-owner batch process** (§9) and the durable catalog file
|
||||
are implemented in `src/catalog.jl` (the `Catalog` durable state, the
|
||||
incremental `catalog_sweep!`, the offline `compact!` seed/recompaction, and
|
||||
`write_nominations!`), driven by `bin/cluster_sweep.jl` (cron/periodic; the
|
||||
first run auto-compacts to seed, subsequent runs sweep incrementally). The
|
||||
catalog is persisted with the stage-2 sidecar-first temp→fsync→rename→fsync-dir
|
||||
discipline. As §5B predicts, under the calibrated `bg_mass > α` the live sweep
|
||||
never mints single-file clusters — new formats are discovered by the offline
|
||||
`compact!` re-clustering the background residue, not by the live path.
|
||||
|
||||
## Open items (deferred, intentionally)
|
||||
|
||||
- **v2, now top priority: inverse-entropy position weighting** (unblocks ELF/tar
|
||||
and any format pair that shares a long constant run — see §11).
|
||||
|
||||
- v2: tail-window block; sparse deep-offset probe (tar-class).
|
||||
- v3: sub-clustering structureless high-entropy residue (needs entropy/histogram
|
||||
feature, not header bytes).
|
||||
- Later tier: docx-vs-zip split via variable-offset central-directory names.
|
||||
- Periodic Lux retrain from accumulated signature-labeled files.
|
||||
@@ -1,27 +1,40 @@
|
||||
module FileServer
|
||||
|
||||
using Logging
|
||||
using Random
|
||||
using UUIDs
|
||||
using HTTP
|
||||
using JSON3
|
||||
using Oxygen
|
||||
using Lux
|
||||
using JLD2
|
||||
using Languages
|
||||
|
||||
include("multipart.jl") # streaming multipart reader (defines UPLOAD_CHUNK_BYTES, used by config.jl)
|
||||
include("config.jl")
|
||||
include("job.jl")
|
||||
include("queue.jl")
|
||||
include("stats.jl") # per-stage counters behind GET /stats (needs Config/Job/JobQueue)
|
||||
include("spool.jl")
|
||||
include("model.jl") # build_model() + read_features(); shared with bin/train.jl
|
||||
include("classify.jl") # Classifier + load_classifier/classify (needs model.jl)
|
||||
include("metadata.jl") # exiftool extraction + sidecar enrichment (stage 2)
|
||||
include("content.jl") # binary-vs-text triage for unknown files (stage 3)
|
||||
include("language.jl") # natural + programming language enrichment for text (stage 4)
|
||||
include("cluster.jl") # unknown-format discovery by header clustering (stage 5, science)
|
||||
include("catalog.jl") # durable single-owner format catalog (stage 5, phase B; needs cluster.jl + metadata.jl fsync)
|
||||
include("worker.jl")
|
||||
|
||||
# Globals the HTTP handlers read at request time. Set once in `run`, before the
|
||||
# server starts accepting connections. Declared after the includes above so the
|
||||
# `Config`/`ChannelQueue` types exist.
|
||||
const CONFIG = Ref{Config}()
|
||||
const QUEUE = Ref{ChannelQueue}()
|
||||
const QUEUE = Ref{ChannelQueue}() # stage-1 (classification) queue; HTTP intake enqueues here
|
||||
const KNOWN_QUEUE = Ref{ChannelQueue}() # stage-2 (enrichment) queue; stage-1 workers enqueue here
|
||||
const UNKNOWN_QUEUE = Ref{ChannelQueue}() # stage-3 (content triage) queue; stage-1 workers enqueue here
|
||||
const TEXT_QUEUE = Ref{ChannelQueue}() # stage-4 (language enrichment) queue; stage-3 workers enqueue here
|
||||
const CLASSIFIER = Ref{Classifier}() # loaded once at startup, shared read-only across workers
|
||||
const DETECTOR = Ref{LanguageDetector}() # natural-language detector; built once at startup, shared read-only
|
||||
|
||||
include("server.jl") # registers routes (references CONFIG/QUEUE at call time)
|
||||
|
||||
@@ -68,26 +81,82 @@ function run(; overrides...)
|
||||
cfg = config_from_env(; overrides...)
|
||||
ensure_dirs(cfg)
|
||||
|
||||
if cfg.worker_count > Threads.nthreads()
|
||||
@warn "worker_count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" worker_count=cfg.worker_count nthreads=Threads.nthreads()
|
||||
# All pools draw from the same OS threads. Warn on the *combined* size (still
|
||||
# allowed): oversubscription just means tasks share threads, not a failure.
|
||||
total_workers = cfg.worker_count + cfg.known_worker_count + cfg.unknown_worker_count + cfg.text_worker_count
|
||||
if total_workers > Threads.nthreads()
|
||||
@warn "combined worker count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" classify_workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count total=total_workers nthreads=Threads.nthreads()
|
||||
end
|
||||
|
||||
queue = ChannelQueue(cfg.queue_capacity)
|
||||
CONFIG[] = cfg
|
||||
QUEUE[] = queue
|
||||
# exiftool is a hard prerequisite for stage-2 enrichment. Fail fast at
|
||||
# startup rather than discover it missing on the first known file.
|
||||
assert_exiftool()
|
||||
|
||||
# github-linguist powers stage-4 *programming*-language detection, but it's
|
||||
# best-effort (natural-language enrichment stands on its own), so a missing
|
||||
# binary is a warning, not a fatal error — per-file lookups degrade to none.
|
||||
linguist_available() || @warn "github-linguist not found on PATH; stage-4 text files will have no programming language (install it to enable)"
|
||||
|
||||
queue = ChannelQueue(cfg.queue_capacity)
|
||||
known_queue = ChannelQueue(cfg.known_queue_capacity)
|
||||
unknown_queue = ChannelQueue(cfg.unknown_queue_capacity)
|
||||
text_queue = ChannelQueue(cfg.text_queue_capacity)
|
||||
CONFIG[] = cfg
|
||||
QUEUE[] = queue
|
||||
KNOWN_QUEUE[] = known_queue
|
||||
UNKNOWN_QUEUE[] = unknown_queue
|
||||
TEXT_QUEUE[] = text_queue
|
||||
|
||||
# Load the classifier before serving. Fail fast: a server that silently
|
||||
# doesn't classify is a worse surprise than a clear startup error.
|
||||
CLASSIFIER[] = load_classifier(cfg.model_path)
|
||||
@info "loaded classifier" path=cfg.model_path
|
||||
|
||||
recovered = recover_spool!(cfg, queue)
|
||||
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count capacity=cfg.queue_capacity recovered=recovered
|
||||
# Build the natural-language detector once (it loads the whatlang n-gram
|
||||
# model) and share it read-only across the stage-4 pool, like the classifier.
|
||||
DETECTOR[] = LanguageDetector()
|
||||
@info "loaded language detector"
|
||||
|
||||
workers = [Threads.@spawn worker_loop(i, cfg, queue) for i in 1:cfg.worker_count]
|
||||
# Stage-aware recovery: re-drive each stage's leftovers onto its own queue so
|
||||
# files resume where they were, not from scratch. spool/ → stage-1,
|
||||
# known/ → stage-2, unknown/ → stage-3.
|
||||
recovered = recover_dir!(cfg.spool_dir, queue)
|
||||
recovered_known = recover_dir!(cfg.known_dir, known_queue)
|
||||
recovered_unknown = recover_dir!(cfg.unknown_dir, unknown_queue)
|
||||
recovered_text = recover_dir!(cfg.text_dir, text_queue)
|
||||
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity recovered=recovered recovered_known=recovered_known recovered_unknown=recovered_unknown recovered_text=recovered_text
|
||||
|
||||
# Zero the counters here, not at module load: `since` should mean "serving
|
||||
# since", so a scrape's totals cover the run, not the minutes spent loading
|
||||
# the classifier. Nothing has been processed yet — recovery only enqueues.
|
||||
reset_metrics!()
|
||||
|
||||
# Each pool gets its stage's counters (src/stats.jl); `worker_loop` records
|
||||
# into them, `GET /stats` reads them out. Stage 1 and 3 also hand theirs to
|
||||
# their handler, which charges time parked on a full downstream queue to
|
||||
# `blocked_ns` so it isn't mistaken for work.
|
||||
st = METRICS.stages
|
||||
workers = [Threads.@spawn worker_loop(i, cfg, queue,
|
||||
(job, c, wid) -> handle_classify_job(job, c, wid, known_queue, unknown_queue, st.classify),
|
||||
st.classify)
|
||||
for i in 1:cfg.worker_count]
|
||||
known_workers = [Threads.@spawn worker_loop(i, cfg, known_queue, handle_known_job, st.enrich)
|
||||
for i in 1:cfg.known_worker_count]
|
||||
unknown_workers = [Threads.@spawn worker_loop(i, cfg, unknown_queue,
|
||||
(job, c, wid) -> handle_unknown_job(job, c, wid, text_queue, st.triage),
|
||||
st.triage)
|
||||
for i in 1:cfg.unknown_worker_count]
|
||||
text_workers = [Threads.@spawn worker_loop(i, cfg, text_queue,
|
||||
(job, c, wid) -> handle_text_job(job, c, wid, DETECTOR[]),
|
||||
st.language)
|
||||
for i in 1:cfg.text_worker_count]
|
||||
|
||||
register_routes()
|
||||
serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false)
|
||||
# `handler` replaces Oxygen's root stream handler so POST /upload can read its
|
||||
# body incrementally instead of having it buffered into memory first; every
|
||||
# other route still goes through Oxygen (see `root_stream_handler`).
|
||||
serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false,
|
||||
handler = root_stream_handler)
|
||||
|
||||
# Idempotent graceful drain: stop accepting uploads, let workers finish the
|
||||
# buffered jobs, then exit. Called from two places:
|
||||
@@ -99,10 +168,17 @@ function run(; overrides...)
|
||||
drained = Threads.Atomic{Bool}(false)
|
||||
function drain()
|
||||
Threads.atomic_xchg!(drained, true) && return # run at most once
|
||||
@info "draining queue and stopping workers"
|
||||
terminate() # stop accepting new HTTP requests
|
||||
close!(queue) # let workers drain buffered jobs, then exit
|
||||
foreach(wait, workers)
|
||||
@info "draining queues and stopping workers"
|
||||
terminate() # 1. stop accepting new HTTP requests
|
||||
close!(queue) # 2. no new classify jobs; stage-1 drains buffered
|
||||
foreach(wait, workers) # 3. wait out stage-1 — the ONLY producer of BOTH the
|
||||
# known and unknown queues — so nothing else enqueues
|
||||
close!(known_queue) # 4. now safe to close the queues stage-1 fed
|
||||
close!(unknown_queue)
|
||||
foreach(wait, known_workers) # 5. wait out stage-2 (terminal) and stage-3 — stage-3 is
|
||||
foreach(wait, unknown_workers) # the ONLY producer of the text queue
|
||||
close!(text_queue) # 6. now safe to close the queue stage-3 fed
|
||||
foreach(wait, text_workers) # 7. wait out stage-4
|
||||
@info "shutdown complete"
|
||||
end
|
||||
atexit(drain)
|
||||
|
||||
403
src/catalog.jl
Normal file
403
src/catalog.jl
Normal file
@@ -0,0 +1,403 @@
|
||||
# Stage-5 phase B: the live, single-owner format catalog (DESIGN §5B/§9).
|
||||
#
|
||||
# `cluster.jl` is the *science* — feature extraction, the collapsed Gibbs sampler
|
||||
# (phase A, offline), and `assign_file` (the phase-B scoring core). This file is
|
||||
# the *plumbing* that turns that core into a durable, growing catalog:
|
||||
#
|
||||
# * a `Catalog` = the surviving clusters' sufficient statistics + a record of
|
||||
# which files have already been folded in + a few example filenames each;
|
||||
# * `catalog_sweep!` = the phase-B loop — for every *new* file in `binary/`,
|
||||
# run the deterministic CRP-predictive `assign_file` and fold it into the
|
||||
# chosen cluster's stats (DESIGN §5B);
|
||||
# * `compact!` = the offline Gibbs pass that *seeds* the catalog on first run
|
||||
# and periodically re-clusters the pile (DESIGN §5, "seed the initial
|
||||
# catalog" / "periodic compaction");
|
||||
# * `write_nominations!` = surface promotable clusters to a human (DESIGN §6).
|
||||
#
|
||||
# Concurrency model is the deliberate opposite of the stateless classify workers
|
||||
# (DESIGN §9): exactly ONE process owns the catalog, so there are no locks and no
|
||||
# torn reads of sufficient stats. The catalog is a single durable file mutated by
|
||||
# that one process; it is committed with the same sidecar-first temp→fsync→rename
|
||||
# →fsync-dir discipline as the stage-2 sidecars (`commit_enriched!`), so a crash
|
||||
# mid-write can neither corrupt it nor lose the rename. This file lives in the
|
||||
# module (not dependency-flat like cluster.jl) because it needs JSON3 + the fsync
|
||||
# helpers from metadata.jl.
|
||||
|
||||
"How many example filenames to retain per cluster (for the human promotion gate,
|
||||
DESIGN §6). A handful is plenty to eyeball; the count/signature carry the weight."
|
||||
const CATALOG_EXAMPLE_CAP = 8
|
||||
|
||||
"""
|
||||
Catalog
|
||||
|
||||
The mutable phase-B state owned by the single sweep process:
|
||||
|
||||
* `n` — header window these clusters were built at (must match the
|
||||
feature window used to score new files; frozen once seeded).
|
||||
* `clusters` — id → `ClusterStats` (per-position 257-counts + member count).
|
||||
Ids are **frozen at birth** — never renumbered — so there is no
|
||||
label switching across sweeps (DESIGN §5).
|
||||
* `examples` — id → up to `CATALOG_EXAMPLE_CAP` member filenames, for the
|
||||
human nomination glance.
|
||||
* `next_id` — the next fresh cluster id to hand out (monotone; retired ids are
|
||||
never reused, keeping ids globally unique over the catalog's life).
|
||||
* `processed` — basenames of every `binary/` file already folded in, so a sweep
|
||||
is incremental: it touches only files it has not seen. This set
|
||||
grows with the `binary/` pile it mirrors — the same population,
|
||||
no faster — which is acceptable for v1 (DESIGN §9).
|
||||
"""
|
||||
mutable struct Catalog
|
||||
n::Int
|
||||
clusters::Dict{Int,ClusterStats}
|
||||
examples::Dict{Int,Vector{String}}
|
||||
next_id::Int
|
||||
processed::Set{String}
|
||||
end
|
||||
|
||||
"An empty catalog at header window `n` (no clusters seen yet)."
|
||||
Catalog(n::Integer) = Catalog(Int(n), Dict{Int,ClusterStats}(),
|
||||
Dict{Int,Vector{String}}(), 1, Set{String}())
|
||||
|
||||
"Record `name` as an example of cluster `id`, capped at `CATALOG_EXAMPLE_CAP`."
|
||||
function record_example!(cat::Catalog, id::Integer, name::AbstractString)
|
||||
ex = get!(cat.examples, id, String[])
|
||||
length(ex) < CATALOG_EXAMPLE_CAP && push!(ex, String(name))
|
||||
return nothing
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Durable persistence (reuse the stage-2 sidecar-first commit discipline)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Count tables are stored sparsely — only the non-zero `(position, symbol, count)`
|
||||
# triples — because a cluster's n×257 table is overwhelmingly zero (a peaked
|
||||
# magic byte touches one of 257 symbols per position). Sparse keeps the catalog
|
||||
# file small and its load O(non-zeros), not O(n·257·K).
|
||||
function _sparse_counts(counts::Matrix{Int})
|
||||
n, A = size(counts)
|
||||
triples = Vector{Vector{Int}}()
|
||||
@inbounds for i in 1:n, v in 1:A
|
||||
c = counts[i, v]
|
||||
c != 0 && push!(triples, [i, v, c])
|
||||
end
|
||||
return triples
|
||||
end
|
||||
|
||||
function _dense_counts(triples, n::Integer)
|
||||
counts = zeros(Int, n, ALPHABET)
|
||||
for t in triples
|
||||
counts[Int(t[1]), Int(t[2])] = Int(t[3])
|
||||
end
|
||||
return counts
|
||||
end
|
||||
|
||||
"Serialize a `Catalog` to a plain `NamedTuple` ready for `JSON3.write`."
|
||||
function catalog_payload(cat::Catalog)
|
||||
clusters = [(
|
||||
id = id,
|
||||
members = c.members,
|
||||
counts = _sparse_counts(c.counts),
|
||||
examples = get(cat.examples, id, String[]),
|
||||
) for (id, c) in sort(collect(cat.clusters); by=first)]
|
||||
return (
|
||||
n = cat.n,
|
||||
next_id = cat.next_id,
|
||||
clusters = clusters,
|
||||
processed = sort(collect(cat.processed)),
|
||||
)
|
||||
end
|
||||
|
||||
"""
|
||||
save_catalog!(path, cat)
|
||||
|
||||
Durably write `cat` to `path` with the sidecar-first ordering (DESIGN §9): write
|
||||
to a temp name, fsync the bytes, atomically rename into place, then fsync the
|
||||
containing directory so the rename itself survives power loss. A crash can leave
|
||||
at most a stale `.tmp`, never a torn catalog.
|
||||
"""
|
||||
function save_catalog!(path::AbstractString, cat::Catalog)
|
||||
dir = dirname(path)
|
||||
isempty(dir) || mkpath(dir)
|
||||
tmp = string(path, ".tmp")
|
||||
open(tmp, "w") do io
|
||||
write(io, JSON3.write(catalog_payload(cat)))
|
||||
flush(io)
|
||||
fsync_fd(fd(io)) # persist bytes before the rename
|
||||
end
|
||||
mv(tmp, path; force=true) # atomic replace
|
||||
fsync_dir(isempty(dir) ? "." : dir) # persist the rename itself
|
||||
return path
|
||||
end
|
||||
|
||||
"""
|
||||
load_catalog(path; n) -> Catalog
|
||||
|
||||
Load the durable catalog from `path`, or return a fresh empty `Catalog(n)` if it
|
||||
does not exist yet (first run). `n` is the configured header window used only for
|
||||
the empty case; a loaded catalog keeps its own frozen `n`.
|
||||
"""
|
||||
function load_catalog(path::AbstractString; n::Integer=HEADER_N)
|
||||
isfile(path) || return Catalog(n)
|
||||
doc = JSON3.read(read(path, String))
|
||||
cn = Int(doc.n)
|
||||
cat = Catalog(cn)
|
||||
cat.next_id = Int(doc.next_id)
|
||||
for entry in doc.clusters
|
||||
id = Int(entry.id)
|
||||
c = ClusterStats(_dense_counts(entry.counts, cn), Int(entry.members))
|
||||
cat.clusters[id] = c
|
||||
cat.examples[id] = String[String(e) for e in entry.examples]
|
||||
end
|
||||
for name in doc.processed
|
||||
push!(cat.processed, String(name))
|
||||
end
|
||||
return cat
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Listing the input pile
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
binary_files(dir) -> Vector{String}
|
||||
|
||||
Sorted absolute paths of the regular files in `dir` to be swept, skipping
|
||||
`.meta.json` sidecars and any `.tmp` scratch. Sorted so a sweep's sequential
|
||||
CRP-predictive assignment (which is order-dependent) is deterministic run to run.
|
||||
"""
|
||||
function binary_files(dir::AbstractString)
|
||||
isdir(dir) || return String[]
|
||||
paths = String[]
|
||||
for name in readdir(dir; join=true)
|
||||
isfile(name) || continue
|
||||
(endswith(name, ".meta.json") || endswith(name, ".tmp")) && continue
|
||||
push!(paths, name)
|
||||
end
|
||||
sort!(paths)
|
||||
return paths
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase B: the incremental sweep (the deliverable)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
catalog_sweep!(cat, cfg) -> NamedTuple
|
||||
|
||||
Fold every *new* file in `cfg.cluster_dir` into `cat` using the deterministic
|
||||
CRP-predictive rule (`assign_file`, DESIGN §5B), updating the chosen cluster's
|
||||
sufficient statistics in place. Files already in `cat.processed` are skipped, so
|
||||
repeated sweeps are incremental and idempotent over the pile.
|
||||
|
||||
Per file, `assign_file` returns the argmax component:
|
||||
* an existing cluster id → the file joins it (`add!`);
|
||||
* `0` (background) → counted, not clustered — a novel-but-unmatched file parks
|
||||
here by design; genuinely new formats are discovered by the offline
|
||||
`compact!` re-clustering the residue, not by single-file minting;
|
||||
* `-1` (mint) → a fresh cluster is seeded with a frozen `next_id`. Under the
|
||||
calibrated `bg_mass > α` this never fires on the live path (fresh and
|
||||
background share the one-file likelihood, so background always wins) — the
|
||||
branch exists for correctness, not as a routine outcome (DESIGN §5B).
|
||||
|
||||
Mutates `cat` but does NOT persist it — the caller commits once, after the sweep.
|
||||
Returns a summary of what happened this sweep.
|
||||
"""
|
||||
function catalog_sweep!(cat::Catalog, cfg::Config)
|
||||
n_seen = 0; n_bg = 0; n_joined = 0; n_minted = 0
|
||||
for path in binary_files(cfg.cluster_dir)
|
||||
base = basename(path)
|
||||
base in cat.processed && continue
|
||||
x = header_symbols(path; n=cat.n)
|
||||
ids = sort!(collect(keys(cat.clusters)))
|
||||
k = assign_file(x, cat.clusters, ids;
|
||||
α=cfg.cluster_alpha, β=cfg.cluster_pseudocount,
|
||||
bg_mass=cfg.cluster_bg_mass)
|
||||
if k == -1
|
||||
id = cat.next_id
|
||||
cat.next_id += 1
|
||||
c = ClusterStats(cat.n)
|
||||
add!(c, x)
|
||||
cat.clusters[id] = c
|
||||
record_example!(cat, id, base)
|
||||
n_minted += 1
|
||||
elseif k == 0
|
||||
n_bg += 1
|
||||
else
|
||||
add!(cat.clusters[k], x)
|
||||
record_example!(cat, k, base)
|
||||
n_joined += 1
|
||||
end
|
||||
push!(cat.processed, base)
|
||||
n_seen += 1
|
||||
end
|
||||
return (; n_seen, n_joined, n_bg, n_minted,
|
||||
n_clusters=length(cat.clusters), n_processed=length(cat.processed))
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Offline seed / compaction (wraps the phase-A Gibbs sampler)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
compact!(cat, cfg; sweeps, restarts, rng) -> NamedTuple
|
||||
|
||||
Re-cluster the *entire* `binary/` pile with the offline collapsed Gibbs sampler
|
||||
and adopt the winning partition as the catalog's clusters (DESIGN §5). This is
|
||||
both the **seed** on first run (an empty catalog has no clusters, so the live
|
||||
sweep alone would send everything to background) and the **periodic compaction**
|
||||
that merges drifted clusters / splits bloated ones later.
|
||||
|
||||
Ids are taken from the Gibbs partition and frozen; because compaction re-derives
|
||||
the whole partition, this is a wholesale replace of `clusters`/`examples`, and
|
||||
every file in the pile is marked processed. Callers run this on an explicit
|
||||
schedule (e.g. `--compact`), never on the latency path.
|
||||
"""
|
||||
function compact!(cat::Catalog, cfg::Config;
|
||||
sweeps::Integer=150, restarts::Integer=6,
|
||||
rng::AbstractRNG=Random.default_rng())
|
||||
paths = binary_files(cfg.cluster_dir)
|
||||
if isempty(paths)
|
||||
return (; n_files=0, n_clusters=length(cat.clusters), n_bg=0)
|
||||
end
|
||||
X = header_matrix(paths; n=cat.n)
|
||||
result = gibbs_cluster(X; α=cfg.cluster_alpha, β=cfg.cluster_pseudocount,
|
||||
bg_mass=cfg.cluster_bg_mass, sweeps=sweeps,
|
||||
restarts=restarts, rng=rng)
|
||||
empty!(cat.clusters)
|
||||
empty!(cat.examples)
|
||||
empty!(cat.processed)
|
||||
for (id, c) in result.clusters
|
||||
cat.clusters[id] = c
|
||||
end
|
||||
cat.next_id = (isempty(result.clusters) ? 0 : maximum(keys(result.clusters))) + 1
|
||||
for (j, path) in enumerate(paths)
|
||||
base = basename(path)
|
||||
push!(cat.processed, base)
|
||||
z = result.assignments[j]
|
||||
z > 0 && record_example!(cat, z, base)
|
||||
end
|
||||
n_bg = count(==(0), result.assignments)
|
||||
return (; n_files=length(paths), n_clusters=length(cat.clusters), n_bg)
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nominations (closing the loop to the classifier — DESIGN §6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"Render a signature (from `signature`) into a human-readable hex template:
|
||||
two hex digits for a required byte, `EOF` for a required past-EOF, `??` for a
|
||||
wildcard. This is what a human eyeballs at the promotion gate."
|
||||
function signature_hex(sig::AbstractVector)
|
||||
parts = map(sig) do s
|
||||
s === nothing ? "??" :
|
||||
s == PAST_EOF ? "EOF" :
|
||||
string(s; base=16, pad=2)
|
||||
end
|
||||
return join(parts, " ")
|
||||
end
|
||||
|
||||
"The required (non-wildcard) positions of a signature as `(position, byte)`
|
||||
records; `byte` is the raw 0–255 value, or the string `\"past_eof\"`."
|
||||
function signature_magic(sig::AbstractVector)
|
||||
magic = Vector{NamedTuple{(:position, :byte),Tuple{Int,Any}}}()
|
||||
for (i, s) in enumerate(sig)
|
||||
s === nothing && continue
|
||||
push!(magic, (position=i, byte=(s == PAST_EOF ? "past_eof" : s)))
|
||||
end
|
||||
return magic
|
||||
end
|
||||
|
||||
"""
|
||||
write_nominations!(cat, cfg) -> Vector{String}
|
||||
|
||||
Write one JSON nomination per promotable cluster (`is_promotable`, DESIGN §6)
|
||||
into `cfg.nominated_dir`, each carrying the cluster's signature (hex template +
|
||||
required magic positions), member count, and example filenames — everything a
|
||||
human needs to glance and promote. The background (id 0) is never a cluster here,
|
||||
so it can never be nominated, by construction.
|
||||
|
||||
Nominations are rewritten every sweep (membership only grows), so each file is
|
||||
durably replaced via the same temp→fsync→rename→fsync-dir commit as the catalog.
|
||||
Returns the paths written. Non-promotable clusters are left alone — a cluster
|
||||
that *was* nominated and later fell below threshold cannot happen (members only
|
||||
grow), so there is nothing to retract.
|
||||
"""
|
||||
function write_nominations!(cat::Catalog, cfg::Config)
|
||||
mkpath(cfg.nominated_dir)
|
||||
written = String[]
|
||||
for (id, c) in sort(collect(cat.clusters); by=first)
|
||||
sig = signature(c) # default β — signature β is
|
||||
# decoupled from clustering β (DESIGN §11.3a)
|
||||
is_promotable(c, sig; min_members=cfg.promote_min_members,
|
||||
min_magic=cfg.promote_min_magic) || continue
|
||||
payload = (
|
||||
cluster_id = id,
|
||||
members = c.members,
|
||||
magic_length = magic_positions(sig),
|
||||
signature_hex = signature_hex(sig),
|
||||
magic = signature_magic(sig),
|
||||
examples = get(cat.examples, id, String[]),
|
||||
)
|
||||
dest = joinpath(cfg.nominated_dir, "cluster-$(id).json")
|
||||
tmp = string(dest, ".tmp")
|
||||
open(tmp, "w") do io
|
||||
write(io, JSON3.write(payload))
|
||||
flush(io)
|
||||
fsync_fd(fd(io))
|
||||
end
|
||||
mv(tmp, dest; force=true)
|
||||
push!(written, dest)
|
||||
end
|
||||
fsync_dir(cfg.nominated_dir)
|
||||
return written
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Orchestration + CLI (the periodic single-owner process — DESIGN §9)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
run_cluster_sweep(cfg; compact=false, rng) -> NamedTuple
|
||||
|
||||
One end-to-end pass of the single-owner stage-5 process: load the durable
|
||||
catalog, either `compact!` (offline Gibbs — used to seed on first run or to
|
||||
recompact) or `catalog_sweep!` (the incremental live path), then durably persist
|
||||
the catalog and (re)write nominations. This is the whole job the cron/periodic
|
||||
runner performs; `bin/cluster_sweep.jl` is a thin shell around it.
|
||||
|
||||
`compact` is forced when the catalog is empty (no clusters AND nothing processed
|
||||
yet): a first live sweep against no clusters would send every file to background,
|
||||
so the catalog must be seeded by an offline Gibbs pass before it can assign.
|
||||
"""
|
||||
function run_cluster_sweep(cfg::Config; compact::Bool=false,
|
||||
rng::AbstractRNG=Random.default_rng())
|
||||
cat = load_catalog(cfg.cluster_catalog_path; n=cfg.cluster_n)
|
||||
if cat.n != cfg.cluster_n
|
||||
@warn "configured cluster_n differs from the catalog's frozen window; using the catalog's" catalog_n=cat.n configured_n=cfg.cluster_n
|
||||
end
|
||||
is_empty = isempty(cat.clusters) && isempty(cat.processed)
|
||||
mode = (compact || is_empty) ? :compact : :sweep
|
||||
summary = mode === :compact ? compact!(cat, cfg; rng=rng) : catalog_sweep!(cat, cfg)
|
||||
save_catalog!(cfg.cluster_catalog_path, cat)
|
||||
nominated = write_nominations!(cat, cfg)
|
||||
return (; mode, summary, n_nominated=length(nominated), nominated,
|
||||
n_clusters=length(cat.clusters), n_processed=length(cat.processed))
|
||||
end
|
||||
|
||||
"""
|
||||
cluster_sweep_cli(args)
|
||||
|
||||
Entry point for `bin/cluster_sweep.jl`. Builds a `Config` from the environment,
|
||||
runs one `run_cluster_sweep`, and logs a one-line summary. `--compact` forces the
|
||||
offline Gibbs re-cluster (seed / periodic compaction) instead of the incremental
|
||||
live sweep.
|
||||
"""
|
||||
function cluster_sweep_cli(args::AbstractVector{<:AbstractString}=String[])
|
||||
compact = "--compact" in args
|
||||
cfg = config_from_env()
|
||||
ensure_dirs(cfg)
|
||||
@info "stage-5 sweep starting" catalog=cfg.cluster_catalog_path input=cfg.cluster_dir compact=compact
|
||||
r = run_cluster_sweep(cfg; compact=compact)
|
||||
@info "stage-5 sweep complete" mode=r.mode clusters=r.n_clusters processed=r.n_processed nominated=r.n_nominated summary=r.summary
|
||||
return r
|
||||
end
|
||||
518
src/cluster.jl
Normal file
518
src/cluster.jl
Normal file
@@ -0,0 +1,518 @@
|
||||
# Stage-5: unknown-format discovery by Bayesian header clustering.
|
||||
#
|
||||
# See model/DESIGN_clustering.md for the full rationale. In brief: files that
|
||||
# stage-3 sorted into `binary/` are the `:unknown` sink — genuinely
|
||||
# unrecognized bytes. This stage clusters them by *file format* (not producer)
|
||||
# using the first `HEADER_N` header bytes, modeled as a Dirichlet-process
|
||||
# mixture of per-position categoricals over a 257-symbol alphabet
|
||||
# (byte 0–255, plus symbol 257 = "past EOF"). Each cluster's signature is a
|
||||
# magic-number template that can be promoted into the classifier's fast path.
|
||||
#
|
||||
# This file is deliberately dependency-light: everything below is base Julia
|
||||
# (only `log`, no `SpecialFunctions`), so it drops into the existing module and
|
||||
# the offline calibration script alike without touching the Manifest. The model
|
||||
# is categorical on purpose — do NOT reuse model.jl's [0,1] byte scaling here
|
||||
# (that metric is meaningful for the Lux net and meaningless for header bytes,
|
||||
# where 0x89 and 0x88 are not "close"; see DESIGN §4).
|
||||
|
||||
"Number of leading header bytes modeled per file (the feature window). DESIGN §4b."
|
||||
const HEADER_N = 32
|
||||
|
||||
"Alphabet size: byte values 0–255 plus one extra symbol for 'past end of file'."
|
||||
const ALPHABET = 257
|
||||
|
||||
"The 'past EOF' symbol (1-based index `ALPHABET`). A file shorter than a given
|
||||
position emits this here — real, discriminative signal for fixed-length formats,
|
||||
and it avoids colliding zero-padding with genuine 0x00 header bytes (DESIGN §4)."
|
||||
const PAST_EOF = ALPHABET
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
header_symbols(path; n=HEADER_N) -> Vector{Int}
|
||||
|
||||
Read the first `n` bytes of the file at `path` and map them to a length-`n`
|
||||
vector of 1-based categorical symbols: byte value `b` → `b + 1` (so `1..256`),
|
||||
and every position at or beyond end-of-file → `PAST_EOF` (`257`). Never reads
|
||||
more than `n` bytes, so memory stays flat regardless of file size.
|
||||
"""
|
||||
function header_symbols(path::AbstractString; n::Integer=HEADER_N)
|
||||
syms = fill(PAST_EOF, n)
|
||||
open(path, "r") do io
|
||||
bytes = read(io, n)
|
||||
@inbounds for i in eachindex(bytes)
|
||||
syms[i] = Int(bytes[i]) + 1
|
||||
end
|
||||
end
|
||||
return syms
|
||||
end
|
||||
|
||||
"""
|
||||
header_matrix(paths; n=HEADER_N) -> Matrix{Int}
|
||||
|
||||
Stack `header_symbols` for every path into an `n × length(paths)` matrix (one
|
||||
column per file), the input layout the Gibbs sampler and predictive scorer both
|
||||
consume.
|
||||
"""
|
||||
function header_matrix(paths::AbstractVector{<:AbstractString}; n::Integer=HEADER_N)
|
||||
X = Matrix{Int}(undef, n, length(paths))
|
||||
for (j, p) in enumerate(paths)
|
||||
X[:, j] = header_symbols(p; n=n)
|
||||
end
|
||||
return X
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model: DP mixture of per-position categoricals (Dirichlet-Categorical)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
ClusterStats
|
||||
|
||||
Sufficient statistics for one cluster: a per-position count table `counts`
|
||||
(`n × ALPHABET`; `counts[i, v]` = how many member files show symbol `v` at
|
||||
position `i`) and the member count `members`. These are exactly what phase-B
|
||||
persists per catalog entry, and everything the collapsed predictive needs.
|
||||
A slot with `members == 0` is inactive (reusable) — the Gibbs sweep prunes
|
||||
emptied clusters without renumbering, so surviving cluster ids stay stable.
|
||||
"""
|
||||
mutable struct ClusterStats
|
||||
counts::Matrix{Int} # n × ALPHABET
|
||||
members::Int
|
||||
end
|
||||
|
||||
ClusterStats(n::Integer) = ClusterStats(zeros(Int, n, ALPHABET), 0)
|
||||
|
||||
"Add file `x` (a length-n symbol vector) into cluster `c`'s sufficient stats."
|
||||
function add!(c::ClusterStats, x::AbstractVector{<:Integer})
|
||||
@inbounds for i in eachindex(x)
|
||||
c.counts[i, x[i]] += 1
|
||||
end
|
||||
c.members += 1
|
||||
return c
|
||||
end
|
||||
|
||||
"Remove file `x` from cluster `c`'s sufficient stats (inverse of `add!`)."
|
||||
function remove!(c::ClusterStats, x::AbstractVector{<:Integer})
|
||||
@inbounds for i in eachindex(x)
|
||||
c.counts[i, x[i]] -= 1
|
||||
end
|
||||
c.members -= 1
|
||||
return c
|
||||
end
|
||||
|
||||
"""
|
||||
log_predictive(c, x, β) -> Float64
|
||||
|
||||
Log probability that file `x` was produced by cluster `c` under the collapsed
|
||||
Dirichlet-Categorical predictive, given `c`'s current counts: at each position
|
||||
`i`, `p(x_i | c) = (counts[i, x_i] + β) / (members + ALPHABET·β)`, summed in log
|
||||
space over positions. Call with `c` NOT containing `x` (Gibbs excludes the point
|
||||
being resampled), so an emptied cluster reduces to the uniform prior `1/ALPHABET`
|
||||
per position — identical to a brand-new cluster, as it should be.
|
||||
"""
|
||||
function log_predictive(c::ClusterStats, x::AbstractVector{<:Integer}, β::Float64)
|
||||
denom = log(c.members + ALPHABET * β)
|
||||
s = 0.0
|
||||
@inbounds for i in eachindex(x)
|
||||
s += log(c.counts[i, x[i]] + β) - denom
|
||||
end
|
||||
return s
|
||||
end
|
||||
|
||||
"Log likelihood of `x` under the fixed uniform component (each position uniform
|
||||
over the 257 symbols): `n · log(1/ALPHABET)`. Used for both the never-adaptive
|
||||
background 'junk drawer' and the prior predictive of a fresh cluster (DESIGN §4a)."
|
||||
log_uniform(n::Integer) = -n * log(ALPHABET)
|
||||
|
||||
# Lanczos approximation to log Γ(x) for x > 0, so partition scoring (below) needs
|
||||
# the Dirichlet-multinomial marginal's gamma terms without pulling in
|
||||
# SpecialFunctions — keeping this stage dependency-flat (no Manifest churn).
|
||||
# g = 7, standard coefficients; accurate to ~1e-14 over the range we use.
|
||||
const _LANCZOS_G = 7
|
||||
const _LANCZOS_C = (0.99999999999980993, 676.5203681218851, -1259.1392167224028,
|
||||
771.32342877765313, -176.61502916214059, 12.507343278686905,
|
||||
-0.13857109526572012, 9.9843695780195716e-6, 1.5056327351493116e-7)
|
||||
|
||||
function loggamma(x::Float64)
|
||||
x < 0.5 && return log(π / sin(π * x)) - loggamma(1.0 - x) # reflection
|
||||
x -= 1.0
|
||||
a = _LANCZOS_C[1]
|
||||
t = x + _LANCZOS_G + 0.5
|
||||
@inbounds for i in 1:_LANCZOS_G + 1
|
||||
a += _LANCZOS_C[i + 1] / (x + i)
|
||||
end
|
||||
return 0.5 * log(2π) + (x + 0.5) * log(t) - t + log(a)
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase A: collapsed Gibbs sampler (offline — the science)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
GibbsResult
|
||||
|
||||
Output of `gibbs_cluster`: `assignments` (one per input file; `0` = absorbed by
|
||||
the background junk drawer, positive ints = cluster id), the surviving
|
||||
`clusters` keyed by id, and `score` (the partition's collapsed pseudo-likelihood,
|
||||
used to rank restarts).
|
||||
"""
|
||||
struct GibbsResult
|
||||
assignments::Vector{Int}
|
||||
clusters::Dict{Int,ClusterStats}
|
||||
score::Float64
|
||||
end
|
||||
|
||||
"""
|
||||
gibbs_cluster(X; α, β, bg_mass, sweeps, restarts, rng) -> GibbsResult
|
||||
|
||||
Cluster the columns of `X` (an `n × N` header-symbol matrix) with a collapsed
|
||||
Gibbs sampler over a CRP/Dirichlet-Categorical mixture plus a fixed uniform
|
||||
background (DESIGN §5). Unknown *k* falls out of the CRP natively.
|
||||
|
||||
Per point, per sweep, the point is removed from its cluster and reassigned by
|
||||
sampling from the CRP-predictive weights:
|
||||
|
||||
* existing cluster `k`: `members_k · exp(log_predictive)`
|
||||
* background: `bg_mass · (1/ALPHABET)^n` (never adapts)
|
||||
* a fresh cluster: `α · (1/ALPHABET)^n`
|
||||
|
||||
A uniform/high-entropy blob matches no structured cluster, and background vs.
|
||||
fresh is then decided by `bg_mass` vs. `α`; with `bg_mass ≥ α` such blobs are
|
||||
absorbed rather than minting singletons. `restarts` independent runs are made
|
||||
from different seeds and the highest-scoring partition is returned (a cheap,
|
||||
base-only stand-in for the offline Binder/VI point-summary the design defers).
|
||||
"""
|
||||
function gibbs_cluster(X::AbstractMatrix{<:Integer};
|
||||
α::Float64=1.0, β::Float64=0.5, bg_mass::Float64=5.0,
|
||||
sweeps::Integer=80, restarts::Integer=4,
|
||||
rng::AbstractRNG=Random.default_rng())
|
||||
best = nothing
|
||||
for _ in 1:restarts
|
||||
r = _gibbs_once(X; α=α, β=β, bg_mass=bg_mass, sweeps=sweeps, rng=rng)
|
||||
if best === nothing || r.score > best.score
|
||||
best = r
|
||||
end
|
||||
end
|
||||
return best
|
||||
end
|
||||
|
||||
function _gibbs_once(X::AbstractMatrix{<:Integer};
|
||||
α::Float64, β::Float64, bg_mass::Float64,
|
||||
sweeps::Integer, rng::AbstractRNG)
|
||||
n, N = size(X)
|
||||
# Seed every file in its own singleton (NOT the background). Cold-starting
|
||||
# from the background deadlocks: at a format's first file, a fresh cluster
|
||||
# and the background are equally uniform, so with bg_mass ≥ α the background
|
||||
# always wins and no real cluster is ever seeded. Singleton init sidesteps
|
||||
# this — same-format singletons merge and snowball, while a lone
|
||||
# random-blob singleton dissolves on resample and is reclaimed by the
|
||||
# (stickier) background. See DESIGN §4a.
|
||||
z = collect(1:N)
|
||||
clusters = Dict{Int,ClusterStats}()
|
||||
for j in 1:N
|
||||
c = ClusterStats(n)
|
||||
add!(c, view(X, :, j))
|
||||
clusters[j] = c
|
||||
end
|
||||
next_id = N + 1
|
||||
log_u = log_uniform(n)
|
||||
log_bg = log(bg_mass) + log_u
|
||||
log_new = log(α) + log_u
|
||||
|
||||
# Reused across every point-visit so the sampler's hot loop allocates nothing
|
||||
# per step (2M+ visits per run): `idbuf[t]` is the cluster id whose weight is
|
||||
# `logw[t+1]` (logw[1] = background, logw[end] = fresh). Rebuilding these with
|
||||
# fresh `Vector`/`collect(keys(...))` each step was both slow and enough GC
|
||||
# churn to trip a Julia GC segfault on long grid runs.
|
||||
idbuf = Int[]
|
||||
logw = Float64[]
|
||||
for _ in 1:sweeps
|
||||
for j in 1:N
|
||||
x = view(X, :, j)
|
||||
|
||||
# Remove point j from its current component.
|
||||
zj = z[j]
|
||||
if zj > 0
|
||||
c = clusters[zj]
|
||||
remove!(c, x)
|
||||
if c.members == 0
|
||||
delete!(clusters, zj) # prune emptied cluster; id retired
|
||||
end
|
||||
end
|
||||
|
||||
# Candidate log-weights: background, each live cluster, fresh.
|
||||
empty!(idbuf); empty!(logw)
|
||||
push!(logw, log_bg)
|
||||
for (k, c) in clusters
|
||||
push!(idbuf, k)
|
||||
push!(logw, log(c.members) + log_predictive(c, x, β))
|
||||
end
|
||||
push!(logw, log_new)
|
||||
|
||||
# Gumbel-max sample from the categorical over components.
|
||||
pick = _gumbel_argmax(logw, rng)
|
||||
|
||||
if pick == 1
|
||||
z[j] = 0 # background
|
||||
elseif pick == length(logw)
|
||||
id = next_id; next_id += 1 # fresh cluster
|
||||
c = ClusterStats(n)
|
||||
add!(c, x)
|
||||
clusters[id] = c
|
||||
z[j] = id
|
||||
else
|
||||
id = idbuf[pick - 1]
|
||||
add!(clusters[id], x)
|
||||
z[j] = id
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return GibbsResult(z, clusters, partition_logmarginal(X, z, clusters, α, β))
|
||||
end
|
||||
|
||||
"Argmax of `logw .+ Gumbel noise` — an exact draw from softmax(logw) without
|
||||
normalizing (numerically safe for the tiny header-likelihood magnitudes)."
|
||||
function _gumbel_argmax(logw::AbstractVector{Float64}, rng::AbstractRNG)
|
||||
best_i = 1
|
||||
best_v = -Inf
|
||||
@inbounds for i in eachindex(logw)
|
||||
g = logw[i] - log(-log(rand(rng)))
|
||||
if g > best_v
|
||||
best_v = g
|
||||
best_i = i
|
||||
end
|
||||
end
|
||||
return best_i
|
||||
end
|
||||
|
||||
"""
|
||||
partition_logmarginal(X, z, clusters, α, β) -> Float64
|
||||
|
||||
The joint log-evidence `log p(z, X)` of a partition under the CRP prior and the
|
||||
collapsed Dirichlet-Categorical likelihood — the principled score for ranking
|
||||
Gibbs restarts (higher = better). It is the sum of:
|
||||
|
||||
* the Dirichlet-multinomial **marginal** of each cluster's per-position counts,
|
||||
`lΓ(Aβ) − lΓ(mₖ+Aβ) + Σ_v [lΓ(c_v+β) − lΓ(β)]`, whose normalizer supplies the
|
||||
Occam penalty that a plain product-of-predictives lacks — it is what makes a
|
||||
*merged, heterogeneous* cluster score **worse** than two clean ones (an
|
||||
earlier pseudo-likelihood scorer omitted this and wrongly rewarded merging);
|
||||
* the CRP prior over the clustered points, `K·log α + Σₖ lΓ(mₖ) + lΓ(α) −
|
||||
lΓ(α+N_clustered)`, penalizing gratuitous extra clusters; and
|
||||
* the fixed uniform term for background-assigned files.
|
||||
"""
|
||||
function partition_logmarginal(X::AbstractMatrix{<:Integer}, z::AbstractVector{<:Integer},
|
||||
clusters::Dict{Int,ClusterStats}, α::Float64, β::Float64)
|
||||
n, N = size(X)
|
||||
Aβ = ALPHABET * β
|
||||
lg_Aβ = loggamma(Aβ)
|
||||
lg_β = loggamma(β)
|
||||
s = 0.0
|
||||
# Dirichlet-Categorical marginal likelihood, per cluster × position.
|
||||
for (_, c) in clusters
|
||||
lg_denom = loggamma(c.members + Aβ)
|
||||
@inbounds for i in 1:n, v in 1:ALPHABET
|
||||
cv = c.counts[i, v]
|
||||
cv > 0 && (s += loggamma(cv + β) - lg_β)
|
||||
end
|
||||
s += n * (lg_Aβ - lg_denom)
|
||||
end
|
||||
# CRP prior over the partition of the clustered points.
|
||||
n_bg = count(==(0), z)
|
||||
n_clustered = N - n_bg
|
||||
K = length(clusters)
|
||||
s += K * log(α) + loggamma(α) - loggamma(α + n_clustered)
|
||||
for (_, c) in clusters
|
||||
s += loggamma(float(c.members))
|
||||
end
|
||||
# Background files: independent, uniform.
|
||||
s += n_bg * log_uniform(n)
|
||||
return s
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase B: sequential CRP-predictive assignment (online — the catalog)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
assign_file(x, clusters, ids; α, β, bg_mass) -> Int
|
||||
|
||||
Deterministically assign a single file `x` against an existing catalog: the
|
||||
same CRP-predictive rule as Gibbs but at the **argmax** (no sampling) with the
|
||||
current assignments held fixed (DESIGN §5B). Returns the id of the chosen
|
||||
cluster, `0` for the background, or `-1` to signal "mint a new cluster". `ids`
|
||||
is the caller's stable ordering of `keys(clusters)`.
|
||||
|
||||
A new cluster is minted (`-1`) only when the fresh-cluster weight strictly wins.
|
||||
Fresh and background share the same `(1/ALPHABET)^n` likelihood (one file, however
|
||||
structured, is indistinguishable from a uniform blob until a *second* like file
|
||||
appears), so this reduces to `α > bg_mass`. Under the calibrated `bg_mass > α`,
|
||||
minting is therefore effectively off on the live path **by design**: a novel file
|
||||
that matches nothing parks in the background, and genuinely new formats are
|
||||
discovered by the periodic **offline Gibbs compaction** re-clustering that
|
||||
residue (DESIGN §5), not by single-file minting. Because ids are frozen at birth
|
||||
by the caller, there is no label switching.
|
||||
"""
|
||||
function assign_file(x::AbstractVector{<:Integer}, clusters::Dict{Int,ClusterStats},
|
||||
ids::AbstractVector{<:Integer};
|
||||
α::Float64=1.0, β::Float64=0.5, bg_mass::Float64=5.0)
|
||||
n = length(x)
|
||||
log_u = log_uniform(n)
|
||||
best_kind = :bg # :bg, :existing, :new
|
||||
best_id = 0
|
||||
best = log(bg_mass) + log_u
|
||||
new_w = log(α) + log_u
|
||||
if new_w > best
|
||||
best = new_w; best_kind = :new
|
||||
end
|
||||
for k in ids
|
||||
c = clusters[k]
|
||||
w = log(c.members) + log_predictive(c, x, β)
|
||||
if w > best
|
||||
best = w; best_kind = :existing; best_id = k
|
||||
end
|
||||
end
|
||||
return best_kind === :bg ? 0 : best_kind === :new ? -1 : best_id
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Signatures and promotion (closing the loop to the classifier — DESIGN §6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
signature(c; peak_threshold=0.9, β=0.5) -> Vector{Union{Int,Nothing}}
|
||||
|
||||
Turn a cluster's counts into a libmagic-style template: at each position, if the
|
||||
modal symbol's posterior probability exceeds `peak_threshold`, that byte is
|
||||
*required* (returned as the 0–255 byte value, or `PAST_EOF`); otherwise the
|
||||
position is a wildcard (`nothing`). The vector of required bytes IS the
|
||||
magic-number template — the whole point of the categorical model (DESIGN §4).
|
||||
|
||||
Peakedness is a **Bernoulli** question ("is this position fixed to byte `v`, or
|
||||
not?"), so it uses a 2-way posterior mean `(count + β)/(members + 2β)` — NOT the
|
||||
257-way mixture predictive. The alphabet-wide version would smear the estimate
|
||||
across 257 symbols (`members + 257β` in the denominator), pulling even a
|
||||
unanimous position below any sane threshold once β is small — which would make
|
||||
promotion impossible. This decouples signature detection from the clustering
|
||||
pseudocount and the alphabet size.
|
||||
"""
|
||||
function signature(c::ClusterStats; peak_threshold::Float64=0.9, β::Float64=0.5)
|
||||
n = size(c.counts, 1)
|
||||
sig = Vector{Union{Int,Nothing}}(nothing, n)
|
||||
c.members == 0 && return sig
|
||||
denom = c.members + 2β
|
||||
@inbounds for i in 1:n
|
||||
v = argmax(view(c.counts, i, :))
|
||||
p = (c.counts[i, v] + β) / denom
|
||||
if p > peak_threshold
|
||||
sig[i] = v == PAST_EOF ? PAST_EOF : v - 1 # back to raw byte value
|
||||
end
|
||||
end
|
||||
return sig
|
||||
end
|
||||
|
||||
"Number of fixed (non-wildcard) positions in a signature — its 'magic length'."
|
||||
magic_positions(sig::AbstractVector) = count(!isnothing, sig)
|
||||
|
||||
"""
|
||||
is_promotable(c, sig; min_members=20, min_magic=3) -> Bool
|
||||
|
||||
A cluster qualifies for *nomination* (still human-gated, DESIGN §6) when it has
|
||||
at least `min_members` files AND at least `min_magic` fixed signature positions.
|
||||
The background (id 0) is never passed here — it is never promotable by design.
|
||||
"""
|
||||
function is_promotable(c::ClusterStats, sig::AbstractVector;
|
||||
min_members::Integer=20, min_magic::Integer=3)
|
||||
return c.members >= min_members && magic_positions(sig) >= min_magic
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Calibration metrics (DESIGN §7): agreement of recovered clusters vs. truth
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
"Map a label vector to consecutive integer ids and a group→indices table."
|
||||
function _groups(labels::AbstractVector)
|
||||
g = Dict{Any,Vector{Int}}()
|
||||
for (i, l) in enumerate(labels)
|
||||
push!(get!(g, l, Int[]), i)
|
||||
end
|
||||
return g
|
||||
end
|
||||
|
||||
"""
|
||||
adjusted_rand_index(a, b) -> Float64
|
||||
|
||||
Adjusted Rand Index between two labelings of the same items: 1.0 = identical
|
||||
partitions (up to relabeling), ~0.0 = chance agreement, can go negative. This is
|
||||
the §7 calibration objective — grid-tuning maximizes ARI of recovered-vs-truth
|
||||
(magic-collapsed) labels. Hand-rolled to keep the dependency footprint flat;
|
||||
matches `Clustering.randindex`.
|
||||
"""
|
||||
function adjusted_rand_index(a::AbstractVector, b::AbstractVector)
|
||||
length(a) == length(b) || throw(DimensionMismatch("label vectors differ in length"))
|
||||
n = length(a)
|
||||
n < 2 && return 1.0
|
||||
ga = collect(values(_groups(a)))
|
||||
gb = collect(values(_groups(b)))
|
||||
# Contingency-table sum of C(n_ij, 2).
|
||||
comb2(x) = x * (x - 1) / 2
|
||||
sa = Set.(ga)
|
||||
index = 0.0
|
||||
for A in sa, B in gb
|
||||
nij = count(in(A), B)
|
||||
index += comb2(nij)
|
||||
end
|
||||
sum_a = sum(comb2(length(g)) for g in ga)
|
||||
sum_b = sum(comb2(length(g)) for g in gb)
|
||||
total = comb2(n)
|
||||
expected = sum_a * sum_b / total
|
||||
maxi = (sum_a + sum_b) / 2
|
||||
denom = maxi - expected
|
||||
return denom == 0 ? 1.0 : (index - expected) / denom
|
||||
end
|
||||
|
||||
"""
|
||||
v_measure(truth, pred; β=1.0) -> (v, homogeneity, completeness)
|
||||
|
||||
Entropy-based cluster agreement (Rosenberg & Hirschberg): homogeneity (each
|
||||
predicted cluster holds one true class), completeness (each true class stays in
|
||||
one predicted cluster), and their weighted harmonic mean `v`. Reported alongside
|
||||
ARI in §7 calibration as a second, differently-biased view.
|
||||
"""
|
||||
function v_measure(truth::AbstractVector, pred::AbstractVector; β::Float64=1.0)
|
||||
n = length(truth)
|
||||
n == 0 && return (1.0, 1.0, 1.0)
|
||||
gt = _groups(truth)
|
||||
gp = _groups(pred)
|
||||
entropy(g) = -sum((length(v) / n) * log(length(v) / n) for v in values(g))
|
||||
H_C = entropy(gt)
|
||||
H_K = entropy(gp)
|
||||
# Conditional entropies via the contingency table.
|
||||
H_CK = 0.0 # H(truth | pred)
|
||||
H_KC = 0.0 # H(pred | truth)
|
||||
for (_, P) in gp
|
||||
Ps = Set(P)
|
||||
for (_, C) in gt
|
||||
nij = count(in(Ps), C)
|
||||
nij == 0 && continue
|
||||
H_CK -= (nij / n) * log(nij / length(P))
|
||||
end
|
||||
end
|
||||
for (_, C) in gt
|
||||
Cs = Set(C)
|
||||
for (_, P) in gp
|
||||
nij = count(in(Cs), P)
|
||||
nij == 0 && continue
|
||||
H_KC -= (nij / n) * log(nij / length(C))
|
||||
end
|
||||
end
|
||||
homogeneity = H_C == 0 ? 1.0 : 1 - H_CK / H_C
|
||||
completeness = H_K == 0 ? 1.0 : 1 - H_KC / H_K
|
||||
v = (homogeneity + completeness == 0) ? 0.0 :
|
||||
(1 + β) * homogeneity * completeness / (β * homogeneity + completeness)
|
||||
return (v, homogeneity, completeness)
|
||||
end
|
||||
116
src/config.jl
116
src/config.jl
@@ -7,10 +7,52 @@ Base.@kwdef struct Config
|
||||
port::Int = 8080
|
||||
worker_count::Int = Threads.nthreads()
|
||||
queue_capacity::Int = 1000
|
||||
spool_dir::String = "data/spool" # files land here on intake (pending)
|
||||
done_dir::String = "data/done" # files move here after successful processing
|
||||
# Stage 2 (enrichment) has its own pool + queue: exiftool work is process-spawn
|
||||
# and I/O bound, a different cost profile than the CPU-bound Lux classify, so
|
||||
# the two pools are tuned independently.
|
||||
known_worker_count::Int = Threads.nthreads()
|
||||
known_queue_capacity::Int = 1000
|
||||
# Stage 3 (content triage) also gets its own pool + queue: sorting an
|
||||
# unrecognized file into binary/ vs text/ is cheap I/O, tuned independently
|
||||
# of the classify and enrich pools.
|
||||
unknown_worker_count::Int = Threads.nthreads()
|
||||
unknown_queue_capacity::Int = 1000
|
||||
# Stage 4 (language enrichment) has its own pool + queue too: detecting a text
|
||||
# file's natural language (Languages.jl) and programming language (shelling to
|
||||
# github-linguist) is a mix of CPU and process-spawn work, tuned independently.
|
||||
text_worker_count::Int = Threads.nthreads()
|
||||
text_queue_capacity::Int = 1000
|
||||
spool_dir::String = "data/spool" # files land here on intake (pending classification)
|
||||
known_dir::String = "data/known" # classified-known, awaiting enrichment (stage 2)
|
||||
unknown_dir::String = "data/unknown" # classified-unknown, awaiting content triage (stage 3)
|
||||
binary_dir::String = "data/binary" # stage-3 sink: unknown files that look like binary data
|
||||
text_dir::String = "data/text" # classified-text, awaiting language enrichment (stage 4)
|
||||
done_dir::String = "data/done" # fully enriched known files (+ .meta.json sidecars)
|
||||
text_done_dir::String = "data/text_done" # fully enriched text files (+ .meta.json sidecars)
|
||||
failed_dir::String = "data/failed" # files move here if a worker throws
|
||||
model_path::String = "model/classifier.jld2" # committed classifier artifact, loaded at startup
|
||||
# Intake reads each upload off the socket in chunks of this size and streams
|
||||
# them straight to the spool file, so this — not the file size — is what
|
||||
# bounds intake memory per in-flight upload (see src/multipart.jl).
|
||||
upload_chunk_bytes::Int = UPLOAD_CHUNK_BYTES
|
||||
exiftool_timeout::Int = 30 # seconds before a stuck exiftool is killed → degraded sidecar
|
||||
linguist_timeout::Int = 30 # seconds before a stuck github-linguist is killed → no programming language
|
||||
# Stage 5 (unknown-format discovery). A separate single-owner batch process
|
||||
# sweeps binary/ and clusters headers; these are its knobs (see
|
||||
# model/DESIGN_clustering.md §9). Values are the calibrated defaults from
|
||||
# bin/cluster_calibrate.jl on the training corpus.
|
||||
cluster_dir::String = "data/binary" # stage-5 input: the :unknown/binary sink to sweep
|
||||
cluster_n::Int = 32 # header bytes modeled per file (HEADER_N)
|
||||
cluster_alpha::Float64 = 1.0 # CRP concentration: propensity to spawn new formats
|
||||
cluster_pseudocount::Float64 = 0.1 # Dirichlet pseudocount β; calibrated on the training corpus
|
||||
cluster_bg_mass::Float64 = 5.0 # fixed mass of the uniform background 'junk drawer'
|
||||
promote_min_members::Int = 20 # cluster size threshold for promotion nomination
|
||||
promote_min_magic::Int = 3 # required fixed signature positions for nomination
|
||||
# The stage-5 catalog is a single durable file mutated by the one sweep
|
||||
# process (never a worker), and nominations are written as one file per
|
||||
# promotable cluster for a human to glance at before promoting (DESIGN §9/§6).
|
||||
cluster_catalog_path::String = "data/catalog.json" # durable phase-B catalog (sufficient stats + processed set)
|
||||
nominated_dir::String = "data/nominated" # one JSON per self-nominated cluster, awaiting a human promote
|
||||
end
|
||||
|
||||
"""
|
||||
@@ -22,26 +64,70 @@ and for `FileServer.run(; port=...)`).
|
||||
|
||||
Recognised variables:
|
||||
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
|
||||
FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH
|
||||
FS_KNOWN_WORKERS, FS_KNOWN_QUEUE_CAPACITY,
|
||||
FS_UNKNOWN_WORKERS, FS_UNKNOWN_QUEUE_CAPACITY,
|
||||
FS_TEXT_WORKERS, FS_TEXT_QUEUE_CAPACITY,
|
||||
FS_SPOOL_DIR, FS_KNOWN_DIR, FS_UNKNOWN_DIR, FS_BINARY_DIR, FS_TEXT_DIR,
|
||||
FS_DONE_DIR, FS_TEXT_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH,
|
||||
FS_UPLOAD_CHUNK_BYTES, FS_EXIFTOOL_TIMEOUT, FS_LINGUIST_TIMEOUT,
|
||||
FS_CLUSTER_DIR, FS_CLUSTER_N, FS_CLUSTER_ALPHA, FS_CLUSTER_PSEUDOCOUNT,
|
||||
FS_CLUSTER_BG_MASS, FS_PROMOTE_MIN_MEMBERS, FS_PROMOTE_MIN_MAGIC,
|
||||
FS_CLUSTER_CATALOG, FS_NOMINATED_DIR
|
||||
"""
|
||||
function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
|
||||
queue_capacity=nothing, spool_dir=nothing,
|
||||
done_dir=nothing, failed_dir=nothing, model_path=nothing)
|
||||
queue_capacity=nothing, known_worker_count=nothing,
|
||||
known_queue_capacity=nothing, unknown_worker_count=nothing,
|
||||
unknown_queue_capacity=nothing, text_worker_count=nothing,
|
||||
text_queue_capacity=nothing, spool_dir=nothing,
|
||||
known_dir=nothing, unknown_dir=nothing, binary_dir=nothing,
|
||||
text_dir=nothing, done_dir=nothing, text_done_dir=nothing,
|
||||
failed_dir=nothing, model_path=nothing,
|
||||
upload_chunk_bytes=nothing, exiftool_timeout=nothing,
|
||||
linguist_timeout=nothing, cluster_dir=nothing, cluster_n=nothing,
|
||||
cluster_alpha=nothing, cluster_pseudocount=nothing,
|
||||
cluster_bg_mass=nothing, promote_min_members=nothing,
|
||||
promote_min_magic=nothing, cluster_catalog_path=nothing,
|
||||
nominated_dir=nothing)
|
||||
Config(
|
||||
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
|
||||
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
|
||||
worker_count = something(worker_count, parse(Int, get(ENV, "FS_WORKERS", string(Threads.nthreads())))),
|
||||
queue_capacity = something(queue_capacity, parse(Int, get(ENV, "FS_QUEUE_CAPACITY", "1000"))),
|
||||
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
|
||||
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
|
||||
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
|
||||
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
|
||||
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
|
||||
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
|
||||
worker_count = something(worker_count, parse(Int, get(ENV, "FS_WORKERS", string(Threads.nthreads())))),
|
||||
queue_capacity = something(queue_capacity, parse(Int, get(ENV, "FS_QUEUE_CAPACITY", "1000"))),
|
||||
known_worker_count = something(known_worker_count, parse(Int, get(ENV, "FS_KNOWN_WORKERS", string(Threads.nthreads())))),
|
||||
known_queue_capacity = something(known_queue_capacity, parse(Int, get(ENV, "FS_KNOWN_QUEUE_CAPACITY", "1000"))),
|
||||
unknown_worker_count = something(unknown_worker_count, parse(Int, get(ENV, "FS_UNKNOWN_WORKERS", string(Threads.nthreads())))),
|
||||
unknown_queue_capacity = something(unknown_queue_capacity, parse(Int, get(ENV, "FS_UNKNOWN_QUEUE_CAPACITY", "1000"))),
|
||||
text_worker_count = something(text_worker_count, parse(Int, get(ENV, "FS_TEXT_WORKERS", string(Threads.nthreads())))),
|
||||
text_queue_capacity = something(text_queue_capacity, parse(Int, get(ENV, "FS_TEXT_QUEUE_CAPACITY", "1000"))),
|
||||
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
|
||||
known_dir = something(known_dir, get(ENV, "FS_KNOWN_DIR", "data/known")),
|
||||
unknown_dir = something(unknown_dir, get(ENV, "FS_UNKNOWN_DIR", "data/unknown")),
|
||||
binary_dir = something(binary_dir, get(ENV, "FS_BINARY_DIR", "data/binary")),
|
||||
text_dir = something(text_dir, get(ENV, "FS_TEXT_DIR", "data/text")),
|
||||
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
|
||||
text_done_dir = something(text_done_dir, get(ENV, "FS_TEXT_DONE_DIR", "data/text_done")),
|
||||
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
|
||||
model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")),
|
||||
upload_chunk_bytes = something(upload_chunk_bytes, parse(Int, get(ENV, "FS_UPLOAD_CHUNK_BYTES", string(UPLOAD_CHUNK_BYTES)))),
|
||||
exiftool_timeout = something(exiftool_timeout, parse(Int, get(ENV, "FS_EXIFTOOL_TIMEOUT", "30"))),
|
||||
linguist_timeout = something(linguist_timeout, parse(Int, get(ENV, "FS_LINGUIST_TIMEOUT", "30"))),
|
||||
cluster_dir = something(cluster_dir, get(ENV, "FS_CLUSTER_DIR", "data/binary")),
|
||||
cluster_n = something(cluster_n, parse(Int, get(ENV, "FS_CLUSTER_N", "32"))),
|
||||
cluster_alpha = something(cluster_alpha, parse(Float64, get(ENV, "FS_CLUSTER_ALPHA", "1.0"))),
|
||||
cluster_pseudocount = something(cluster_pseudocount, parse(Float64, get(ENV, "FS_CLUSTER_PSEUDOCOUNT", "0.1"))),
|
||||
cluster_bg_mass = something(cluster_bg_mass, parse(Float64, get(ENV, "FS_CLUSTER_BG_MASS", "5.0"))),
|
||||
promote_min_members = something(promote_min_members, parse(Int, get(ENV, "FS_PROMOTE_MIN_MEMBERS", "20"))),
|
||||
promote_min_magic = something(promote_min_magic, parse(Int, get(ENV, "FS_PROMOTE_MIN_MAGIC", "3"))),
|
||||
cluster_catalog_path = something(cluster_catalog_path, get(ENV, "FS_CLUSTER_CATALOG", "data/catalog.json")),
|
||||
nominated_dir = something(nominated_dir, get(ENV, "FS_NOMINATED_DIR", "data/nominated")),
|
||||
)
|
||||
end
|
||||
|
||||
"Create the spool/done/failed directories if they don't already exist."
|
||||
"Create all the pipeline-stage directories if they don't already exist."
|
||||
function ensure_dirs(cfg::Config)
|
||||
for d in (cfg.spool_dir, cfg.done_dir, cfg.failed_dir)
|
||||
for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.binary_dir,
|
||||
cfg.text_dir, cfg.done_dir, cfg.text_done_dir, cfg.failed_dir,
|
||||
cfg.nominated_dir)
|
||||
mkpath(d)
|
||||
end
|
||||
return nothing
|
||||
|
||||
65
src/content.jl
Normal file
65
src/content.jl
Normal file
@@ -0,0 +1,65 @@
|
||||
# Stage-3 content triage for unknown files.
|
||||
#
|
||||
# A file that stage-1 couldn't recognize is still sorted into one of two coarse
|
||||
# buckets so downstream tooling can treat them differently: `text/` for
|
||||
# human-readable content, `binary/` for everything else. We sniff only the first
|
||||
# `CONTENT_SNIFF_BYTES` (no full read) and ask two questions: does the window
|
||||
# decode as valid UTF-8, and are any of its control bytes ones that don't belong
|
||||
# in text? This is the Unicode-aware successor to the classic "NUL byte" test —
|
||||
# it accepts non-ASCII text (accents, CJK, emoji) instead of misfiling it as
|
||||
# binary, while still rejecting binary formats, which almost never form valid
|
||||
# UTF-8 near their start (and a NUL is never a valid UTF-8 scalar, so it still
|
||||
# reads as binary for free).
|
||||
|
||||
const CONTENT_SNIFF_BYTES = 8000
|
||||
|
||||
# Control bytes (< 0x20) that appear legitimately in text: BS, TAB, LF, VT, FF,
|
||||
# CR, and ESC (ANSI-colored logs). Any *other* control byte is a binary signal.
|
||||
const TEXT_CONTROL_BYTES = (0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1b)
|
||||
|
||||
# Drop a trailing UTF-8 sequence that the sniff window cut in half, so a
|
||||
# multi-byte character straddling the boundary isn't mistaken for invalid bytes.
|
||||
# Continuation bytes are 0x80–0xBF; a lead byte encodes its own sequence length
|
||||
# in its high bits. We walk back over the trailing continuation bytes, and if
|
||||
# the lead byte we land on expects more bytes than the window actually holds,
|
||||
# trim the whole incomplete sequence.
|
||||
function trim_truncated_utf8(chunk::AbstractVector{UInt8})
|
||||
n = length(chunk)
|
||||
n == 0 && return chunk
|
||||
# Find the start of the final byte sequence: skip back over continuations.
|
||||
i = n
|
||||
while i > 0 && (chunk[i] & 0xc0) == 0x80
|
||||
i -= 1
|
||||
end
|
||||
i == 0 && return chunk # all continuations; leave as-is
|
||||
lead = chunk[i]
|
||||
# How many bytes does this lead byte announce?
|
||||
expected = lead < 0x80 ? 1 : # ASCII
|
||||
lead < 0xe0 ? 2 : # 110xxxxx
|
||||
lead < 0xf0 ? 3 : # 1110xxxx
|
||||
4 # 11110xxx
|
||||
have = n - i + 1
|
||||
return have < expected ? view(chunk, 1:i-1) : chunk
|
||||
end
|
||||
|
||||
"""
|
||||
is_binary(path) -> Bool
|
||||
|
||||
Classify a file as binary (`true`) or text (`false`) by sniffing its first
|
||||
`CONTENT_SNIFF_BYTES` bytes. A file is text when that window (minus any
|
||||
multi-byte character truncated by the window edge) is valid UTF-8 and contains
|
||||
no control bytes outside the text-safe set (`TEXT_CONTROL_BYTES`). An empty file
|
||||
is treated as text.
|
||||
"""
|
||||
function is_binary(path::AbstractString)::Bool
|
||||
open(path, "r") do io
|
||||
chunk = read(io, CONTENT_SNIFF_BYTES)
|
||||
isempty(chunk) && return false
|
||||
window = trim_truncated_utf8(chunk)
|
||||
# Malformed UTF-8 → binary.
|
||||
isvalid(String(copy(window))) || return true
|
||||
# Valid UTF-8, but a stray non-text control byte still means binary.
|
||||
# (NUL is a valid UTF-8 scalar, so it's rejected here, not above.)
|
||||
return any(b -> b < 0x20 && !(b in TEXT_CONTROL_BYTES), window)
|
||||
end
|
||||
end
|
||||
155
src/language.jl
Normal file
155
src/language.jl
Normal file
@@ -0,0 +1,155 @@
|
||||
# Stage-4 language enrichment for text files.
|
||||
#
|
||||
# A file that stage-3 sorted into `text/` is human-readable, but we don't yet
|
||||
# know *what* it is. This stage answers two questions and records them in a
|
||||
# `.meta.json` sidecar, exactly like the stage-2 known-file enrichment:
|
||||
#
|
||||
# * natural language — via Languages.jl's `LanguageDetector` (a Julia port of
|
||||
# the `whatlang` n-gram model): English vs. French vs. Japanese, plus a
|
||||
# confidence score. Pure Julia, no subprocess.
|
||||
# * programming language — via the `github-linguist` CLI, which recognizes
|
||||
# source and markup by extension + content heuristics. There is no
|
||||
# comparable native Julia library, so we shell out (mirroring stage-2's
|
||||
# exiftool dependency).
|
||||
#
|
||||
# Neither detector failing quarantines the file: a text file is wanted whether
|
||||
# or not we can name its language, so a failure yields a *degraded* sidecar
|
||||
# (what we know plus an `error` note), just like stage 2.
|
||||
#
|
||||
# The github-linguist quirk that shapes this code: run against a path *inside* a
|
||||
# git repository, linguist reads the file's committed git blob, not the bytes on
|
||||
# disk — and an untracked file (which every file under `data/` is) has no blob,
|
||||
# so it crashes. We sidestep this by copying the file to a fresh temp dir outside
|
||||
# any repo (preserving its name so linguist's extension heuristics still fire)
|
||||
# and pointing linguist there.
|
||||
|
||||
# How much of a text file to feed the natural-language detector. The whatlang
|
||||
# model saturates quickly, so a bounded prefix keeps memory flat on huge logs
|
||||
# while still giving the detector plenty of signal.
|
||||
const LANG_SAMPLE_BYTES = 65_536
|
||||
|
||||
"Return true if the `github-linguist` binary is on PATH."
|
||||
function linguist_available()
|
||||
try
|
||||
Base.run(pipeline(`github-linguist --version`; stdout=devnull, stderr=devnull))
|
||||
return true
|
||||
catch
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
read_text_sample(path) -> String
|
||||
|
||||
Read up to `LANG_SAMPLE_BYTES` of `path` as UTF-8 text, trimming a multi-byte
|
||||
character the window may have cut in half (reusing stage-3's `trim_truncated_utf8`)
|
||||
so the tail isn't misread as garbage.
|
||||
"""
|
||||
function read_text_sample(path::AbstractString)::String
|
||||
open(path, "r") do io
|
||||
chunk = read(io, LANG_SAMPLE_BYTES)
|
||||
return String(copy(trim_truncated_utf8(chunk)))
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
detect_natural_language(detector, text) -> (name, code, confidence)
|
||||
|
||||
Run the `LanguageDetector` on `text`, returning the language's English name
|
||||
(e.g. `"English"`), its ISO 639-3 code (e.g. `"eng"`), and the model's
|
||||
confidence in `[0,1]`. Returns `(nothing, nothing, nothing)` when there is no
|
||||
usable text (empty/whitespace) or the detector errors — the caller records that
|
||||
as a degraded result rather than failing the file.
|
||||
"""
|
||||
function detect_natural_language(detector, text::AbstractString)
|
||||
isempty(strip(text)) && return (nothing, nothing, nothing)
|
||||
try
|
||||
lang, _script, confidence = detector(text)
|
||||
return (Languages.english_name(lang), Languages.isocode(lang), confidence)
|
||||
catch
|
||||
return (nothing, nothing, nothing)
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
run_linguist(path, timeout) -> Union{String,Nothing}
|
||||
|
||||
Ask `github-linguist --json` for the programming/markup language of the file at
|
||||
`path`, returning the language name (e.g. `"Python"`, `"Markdown"`) or `nothing`
|
||||
when linguist can't identify one. Plain prose reports as `"Text"` and
|
||||
unrecognized content as JSON `null`; both collapse to `nothing` here, since only
|
||||
a real programming/markup language is worth recording.
|
||||
|
||||
`path` MUST be outside any git repository — see the module header for why.
|
||||
"""
|
||||
function run_linguist(path::AbstractString, timeout::Integer)
|
||||
bytes = run_with_timeout(`github-linguist --json $path`, timeout)
|
||||
bytes === nothing && return nothing
|
||||
|
||||
parsed = try
|
||||
JSON3.read(String(bytes))
|
||||
catch
|
||||
return nothing
|
||||
end
|
||||
# linguist --json emits a single object keyed by the file path; pull the one
|
||||
# entry rather than depend on the exact key spelling.
|
||||
isempty(parsed) && return nothing
|
||||
entry = first(values(parsed))
|
||||
lang = get(entry, :language, nothing)
|
||||
(lang === nothing || lang == "Text") && return nothing
|
||||
return String(lang)
|
||||
end
|
||||
|
||||
"""
|
||||
detect_programming_language(job, cfg) -> Union{String,Nothing}
|
||||
|
||||
Programming/markup language of a text file, or `nothing`. Copies the file to a
|
||||
throwaway temp dir *outside* the git repo — under its sanitized original name so
|
||||
linguist's extension heuristics still apply — runs linguist there, and cleans up.
|
||||
"""
|
||||
function detect_programming_language(job::Job, cfg::Config)
|
||||
lang = nothing
|
||||
mktempdir() do dir # tempdir() → /tmp, outside the repo
|
||||
safe = sanitize_filename(job.original_name)
|
||||
tmp = joinpath(dir, safe)
|
||||
cp(job.path, tmp; force=true)
|
||||
lang = run_linguist(tmp, cfg.linguist_timeout)
|
||||
end
|
||||
return lang
|
||||
end
|
||||
|
||||
"""
|
||||
build_text_metadata(detector, job, cfg) -> NamedTuple
|
||||
|
||||
Build the stage-4 sidecar payload for a text file: its natural language (name +
|
||||
ISO code + confidence) and programming/markup language, plus the Job's
|
||||
authoritative id/name/size. `error` is set only when natural-language detection
|
||||
produced nothing usable (the file is still enriched and committed); programming
|
||||
language is best-effort and its absence is normal, not an error.
|
||||
"""
|
||||
function build_text_metadata(detector, job::Job, cfg::Config)
|
||||
text = read_text_sample(job.path)
|
||||
name, code, confidence = detect_natural_language(detector, text)
|
||||
programming_language = detect_programming_language(job, cfg)
|
||||
|
||||
return (
|
||||
id = job.id,
|
||||
original_name = job.original_name,
|
||||
file_size = job.size, # authoritative, from intake
|
||||
content_type = "text",
|
||||
language = name,
|
||||
language_code = code,
|
||||
language_confidence = confidence,
|
||||
programming_language = programming_language,
|
||||
error = name === nothing ? "language detection produced no result" : nothing,
|
||||
)
|
||||
end
|
||||
|
||||
"""
|
||||
finalize_text!(cfg, job, meta) -> (file_dest, sidecar_dest)
|
||||
|
||||
Commit an enriched text file (stage 4) to `text_done/` via the shared
|
||||
sidecar-first `commit_enriched!`, giving text files the same crash-safe
|
||||
"file implies sidecar" guarantee as stage-2 known files.
|
||||
"""
|
||||
finalize_text!(cfg::Config, job::Job, meta) = commit_enriched!(cfg.text_done_dir, job, meta)
|
||||
267
src/metadata.jl
Normal file
267
src/metadata.jl
Normal file
@@ -0,0 +1,267 @@
|
||||
# Stage-2 metadata extraction and enrichment.
|
||||
#
|
||||
# Known files are enriched by shelling out to `exiftool -json -G` (the only tool
|
||||
# with broad, multi-format coverage — there is no comparable native Julia
|
||||
# library), then normalizing its output into a small, stable, documented schema
|
||||
# that downstream consumers can rely on, while preserving the full raw dump.
|
||||
#
|
||||
# exiftool being installed is a hard startup prerequisite (see `assert_exiftool`,
|
||||
# called from `run`). A per-file extraction failure or hang does NOT quarantine
|
||||
# the file — it produces a *degraded* sidecar recording what we know plus the
|
||||
# error, because a file that passed classification is wanted regardless of
|
||||
# whether we could read its metadata.
|
||||
|
||||
"Throw at startup if the `exiftool` binary isn't on PATH — fail fast rather than discover it per file."
|
||||
function assert_exiftool()
|
||||
try
|
||||
Base.run(pipeline(`exiftool -ver`; stdout=devnull, stderr=devnull))
|
||||
catch
|
||||
error("exiftool not found on PATH (install libimage-exiftool-perl / exiftool). It is required for stage-2 metadata enrichment.")
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
# Each normalized field is a coalesce over exiftool tag names, tried in order;
|
||||
# the first present, non-empty value wins. exiftool with `-G` prefixes tags by
|
||||
# group (e.g. "EXIF:Software"), so we match on the bare tag name after the last
|
||||
# colon. Extend a field simply by appending tag names here.
|
||||
#
|
||||
# Note the documented `Creator` ambiguity: in PDF it's the authoring app, but
|
||||
# elsewhere it's the person. We take the simple route — `Creator` feeds
|
||||
# `created_by` only, and `author` relies on the person-specific tags.
|
||||
const CREATED_BY_TAGS = ["Producer", "CreatorTool", "Creator", "Software", "Application", "Encoder", "EncodingTool", "HostComputer"]
|
||||
const AUTHOR_TAGS = ["Author", "Artist", "By-line", "Owner", "Artist"]
|
||||
const CREATED_DATE_TAGS = ["DateTimeOriginal", "CreateDate", "MediaCreateDate", "CreationDate"]
|
||||
const MODIFIED_DATE_TAGS = ["ModifyDate", "FileModifyDate"]
|
||||
|
||||
"fsync an open file descriptor, throwing on failure — used to make a write durable before a rename commits it."
|
||||
function fsync_fd(fd)
|
||||
ccall(:fsync, Cint, (Cint,), fd) == 0 || error("fsync failed: $(Base.Libc.strerror())")
|
||||
return nothing
|
||||
end
|
||||
|
||||
"fsync a directory so a rename into it survives a crash (the rename, not just the file bytes, must be persisted)."
|
||||
function fsync_dir(dir::AbstractString)
|
||||
dfd = ccall(:open, Cint, (Cstring, Cint), dir, 0) # O_RDONLY
|
||||
dfd < 0 && error("cannot open dir for fsync: $dir ($(Base.Libc.strerror()))")
|
||||
try
|
||||
fsync_fd(dfd)
|
||||
finally
|
||||
ccall(:close, Cint, (Cint,), dfd)
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"Strip exiftool's `-G` group prefix (`EXIF:Software` → `Software`) so lookups are group-agnostic."
|
||||
strip_group(tag::AbstractString) = String(last(split(tag, ':')))
|
||||
|
||||
"Return the first present, non-empty value among `tags` in the group-stripped map, or `nothing`."
|
||||
function coalesce_tag(bytag::Dict{String,Any}, tags)
|
||||
for t in tags
|
||||
v = get(bytag, t, nothing)
|
||||
v === nothing && continue
|
||||
s = string(v)
|
||||
isempty(strip(s)) && continue
|
||||
return v
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"How long a child gets to honor SIGTERM before `run_with_timeout` escalates to SIGKILL."
|
||||
const KILL_GRACE_SECONDS = 2.0
|
||||
|
||||
"""
|
||||
Send `signum` to the whole process group `pgid` (a negative pid means "the group"
|
||||
to `kill(2)`). Julia's `kill(::Process, sig)` signals only the child itself,
|
||||
which is not enough to enforce a timeout — see `run_with_timeout`.
|
||||
"""
|
||||
function signal_group(pgid::Integer, signum::Integer)
|
||||
ccall(:kill, Cint, (Cint, Cint), -pgid, signum)
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
run_with_timeout(cmd, timeout) -> Union{Vector{UInt8},Nothing}
|
||||
|
||||
Run `cmd`, capturing stdout, and return the captured bytes on clean exit, or
|
||||
`nothing` on non-zero exit or timeout. The subprocess is killed (SIGTERM, then
|
||||
SIGKILL after a grace period) once it overruns `timeout` seconds, so one
|
||||
pathological input can't wedge a worker forever. Shared by the exiftool (stage 2)
|
||||
and github-linguist (stage 4) shells.
|
||||
|
||||
The child runs in its own process group and the timeout signals the *group*, not
|
||||
just the child. This is what makes the timeout enforceable: `wait` below returns
|
||||
only once the captured stdout pipe closes, and any grandchild inherits that pipe,
|
||||
so signalling the child alone leaves a `sh -c "...; sleep 30"`-shaped process
|
||||
tree running to completion with the worker still blocked on it. The cost of the
|
||||
process group is that a hard crash of the server orphans an in-flight child
|
||||
rather than taking it down; these children are short-lived and timeout-bounded,
|
||||
which is the cheaper side of that trade.
|
||||
"""
|
||||
function run_with_timeout(cmd::Cmd, timeout::Integer)
|
||||
out = IOBuffer()
|
||||
# `detach` puts the child in a fresh process group (it becomes the group
|
||||
# leader, so the group id is its pid); see the docstring for why the group,
|
||||
# and not the child, is what the timeout has to signal.
|
||||
proc = Base.run(pipeline(detach(cmd); stdout=out, stderr=devnull); wait=false)
|
||||
pgid = Base.getpid(proc)
|
||||
|
||||
# A one-shot timer, cancelled the moment the child exits, rather than a
|
||||
# polling loop the caller has to join. The polling version charged every
|
||||
# call the remainder of its in-flight `sleep(0.1)` *after* the child had
|
||||
# already exited — ~50 ms on average, and a measured 101 ms on a process
|
||||
# that exits instantly. That is pure latency on the hot path of two stages
|
||||
# (exiftool here, github-linguist in stage 4), and it dwarfed the work on
|
||||
# anything but a slow file. Waiting on the process directly costs nothing
|
||||
# when the child exits normally, which is the overwhelmingly common case.
|
||||
killed = Ref(false)
|
||||
timer = Timer(timeout) do _
|
||||
process_running(proc) || return
|
||||
killed[] = true
|
||||
signal_group(pgid, Base.SIGTERM)
|
||||
# Escalate: a process that ignores/defers SIGTERM would otherwise pin the
|
||||
# worker forever on the wait(proc) below, defeating the timeout. This
|
||||
# runs off the timer's task so the event loop isn't held during the
|
||||
# grace period, and it is not joined — by the time it wakes, `wait(proc)`
|
||||
# has long since returned and `process_running` settles it.
|
||||
Threads.@spawn begin
|
||||
deadline = time() + KILL_GRACE_SECONDS
|
||||
while process_running(proc) && time() < deadline
|
||||
sleep(0.05)
|
||||
end
|
||||
process_running(proc) && signal_group(pgid, Base.SIGKILL)
|
||||
end
|
||||
end
|
||||
|
||||
try
|
||||
wait(proc)
|
||||
finally
|
||||
close(timer) # cancel the pending kill; a no-op if it already fired
|
||||
end
|
||||
|
||||
(killed[] || !success(proc)) && return nothing
|
||||
return take!(out)
|
||||
end
|
||||
|
||||
"""
|
||||
run_exiftool(path, timeout) -> Union{Dict{String,Any},Nothing}
|
||||
|
||||
Run `exiftool -json -G` on `path`, returning the parsed tag object, or `nothing`
|
||||
on non-zero exit, unparseable output, or timeout.
|
||||
"""
|
||||
function run_exiftool(path::AbstractString, timeout::Integer)
|
||||
# -json: machine output; -G: group-prefixed tags; -n: numeric (unformatted)
|
||||
# values so sizes/durations are numbers, not display strings.
|
||||
bytes = run_with_timeout(`exiftool -json -G -n $path`, timeout)
|
||||
bytes === nothing && return nothing
|
||||
|
||||
parsed = try
|
||||
JSON3.read(String(bytes))
|
||||
catch
|
||||
return nothing
|
||||
end
|
||||
# exiftool -json emits a one-element array of objects (one per input file).
|
||||
(parsed isa AbstractVector && !isempty(parsed)) || return nothing
|
||||
return Dict{String,Any}(String(strip_group(String(k))) => v for (k, v) in pairs(parsed[1]))
|
||||
end
|
||||
|
||||
"""
|
||||
build_metadata(job, cfg) -> NamedTuple
|
||||
|
||||
Extract and normalize metadata for a known file. Always returns a sidecar
|
||||
payload: on extraction success, the normalized fields plus the full raw dump; on
|
||||
failure/timeout, a *degraded* payload with what we know from the Job plus an
|
||||
`error` note. `file_size` always comes from the Job (authoritative), never
|
||||
exiftool.
|
||||
"""
|
||||
function build_metadata(job::Job, cfg::Config)
|
||||
bytag = run_exiftool(job.path, cfg.exiftool_timeout)
|
||||
|
||||
if bytag === nothing
|
||||
return (
|
||||
id = job.id,
|
||||
original_name = job.original_name,
|
||||
file_type = nothing,
|
||||
mime_type = nothing,
|
||||
file_size = job.size,
|
||||
created_date = nothing,
|
||||
modified_date = nothing,
|
||||
author = nothing,
|
||||
created_by = nothing,
|
||||
dimensions = nothing,
|
||||
duration = nothing,
|
||||
page_count = nothing,
|
||||
error = "exiftool extraction failed or timed out",
|
||||
raw = nothing,
|
||||
)
|
||||
end
|
||||
|
||||
return normalize_metadata(job, bytag)
|
||||
end
|
||||
|
||||
"Build the normalized sidecar payload from a successful exiftool tag map."
|
||||
function normalize_metadata(job::Job, bytag::Dict{String,Any})
|
||||
w = get(bytag, "ImageWidth", nothing)
|
||||
h = get(bytag, "ImageHeight", nothing)
|
||||
dims = (w !== nothing && h !== nothing) ? (; width = w, height = h) : nothing
|
||||
|
||||
return (
|
||||
id = job.id,
|
||||
original_name = job.original_name,
|
||||
file_type = get(bytag, "FileType", get(bytag, "FileTypeExtension", nothing)),
|
||||
mime_type = get(bytag, "MIMEType", nothing),
|
||||
file_size = job.size, # authoritative, from intake
|
||||
created_date = coalesce_tag(bytag, CREATED_DATE_TAGS),
|
||||
modified_date = coalesce_tag(bytag, MODIFIED_DATE_TAGS),
|
||||
author = coalesce_tag(bytag, AUTHOR_TAGS),
|
||||
created_by = coalesce_tag(bytag, CREATED_BY_TAGS),
|
||||
dimensions = dims,
|
||||
duration = get(bytag, "Duration", get(bytag, "MediaDuration", nothing)),
|
||||
page_count = get(bytag, "PageCount", nothing),
|
||||
error = nothing,
|
||||
raw = bytag,
|
||||
)
|
||||
end
|
||||
|
||||
"""
|
||||
commit_enriched!(dest_dir, job, meta) -> (file_dest, sidecar_dest)
|
||||
|
||||
Commit an enriched file to `dest_dir` with the sidecar-first ordering so the
|
||||
invariant *"a file in dest_dir implies its sidecar is already there"* always
|
||||
holds. Shared by every enrichment stage that emits a `.meta.json` sidecar
|
||||
(stage-2 known files → `done/`, stage-4 text files → `text_done/`).
|
||||
|
||||
Sequence: write `<name>.meta.json` to a temp name, fsync its bytes, rename it
|
||||
into place, fsync `dest_dir` so the rename itself is durable, THEN move the file
|
||||
into `dest_dir`. A crash between the two leaves only a harmless orphan sidecar in
|
||||
`dest_dir` while the file stays in its stage dir, so stage-aware recovery
|
||||
re-drives it and overwrites the sidecar — idempotent. The fsyncs make the
|
||||
ordering hold across power loss, not just process crashes.
|
||||
"""
|
||||
function commit_enriched!(dest_dir::AbstractString, job::Job, meta)
|
||||
base = basename(job.path)
|
||||
sidecar = joinpath(dest_dir, string(base, ".meta.json"))
|
||||
tmp_sidecar = string(sidecar, ".tmp")
|
||||
|
||||
# Write to a temp name then rename, so a reader in dest_dir never sees a
|
||||
# partial sidecar and a crash mid-write can't masquerade as a committed one.
|
||||
open(tmp_sidecar, "w") do io
|
||||
write(io, JSON3.write(meta))
|
||||
flush(io)
|
||||
fsync_fd(fd(io)) # durably persist bytes before the rename
|
||||
end
|
||||
mv(tmp_sidecar, sidecar; force=true) # sidecar committed first
|
||||
fsync_dir(dest_dir) # persist the rename itself, not just the bytes
|
||||
|
||||
file_dest = move_to(dest_dir, job) # file arrival = commit point
|
||||
return (file_dest, sidecar)
|
||||
end
|
||||
|
||||
"""
|
||||
finalize_known!(cfg, job, meta) -> (file_dest, sidecar_dest)
|
||||
|
||||
Commit an enriched known file (stage 2) to `done/` via the shared sidecar-first
|
||||
`commit_enriched!`.
|
||||
"""
|
||||
finalize_known!(cfg::Config, job::Job, meta) = commit_enriched!(cfg.done_dir, job, meta)
|
||||
|
||||
296
src/multipart.jl
Normal file
296
src/multipart.jl
Normal file
@@ -0,0 +1,296 @@
|
||||
# Streaming multipart/form-data reader.
|
||||
#
|
||||
# Why this exists: HTTP.jl's `parse_multipart_form` takes the *complete* request
|
||||
# body as a byte vector, so using it means every file in the request sits in
|
||||
# memory at once — and is then copied again per part. That contradicts the whole
|
||||
# point of this service: file bytes belong on disk, and only a small reference
|
||||
# travels through the queue. So intake needs a parser that never holds a file.
|
||||
#
|
||||
# This reader walks the body incrementally: it pulls fixed-size chunks off the
|
||||
# socket and hands each part's bytes straight to a sink (the spool file). Peak
|
||||
# memory per connection is `chunk_bytes` + the boundary length, regardless of how
|
||||
# large — or how many — the uploaded files are.
|
||||
#
|
||||
# Interface: two calls in a loop, so the caller keeps ordinary control flow
|
||||
# rather than inverting into callbacks.
|
||||
#
|
||||
# r = MultipartReader(io, boundary)
|
||||
# while (part = next_part!(r)) !== nothing
|
||||
# part.filename === nothing ? skip_part_body!(r) : write_part_body!(sink, r)
|
||||
# end
|
||||
#
|
||||
# The grammar it implements (RFC 2046 §5.1, RFC 7578):
|
||||
#
|
||||
# [preamble] "--" boundary CRLF
|
||||
# part-headers CRLF CRLF part-body
|
||||
# CRLF "--" boundary CRLF ... another part ...
|
||||
# CRLF "--" boundary "--" CRLF ... end of form, [epilogue]
|
||||
#
|
||||
# So the delimiter that *closes* a body is CRLF + "--" + boundary, and the two
|
||||
# bytes after it say whether another part follows (CRLF) or the form is over
|
||||
# ("--"). Every read is bounded, and the buffer retains only the last
|
||||
# `length(delimiter)-1` bytes when no delimiter is found — that tail is what
|
||||
# makes a delimiter split across two chunks parse correctly.
|
||||
|
||||
"Default socket read size, and therefore the memory bound per in-flight upload."
|
||||
const UPLOAD_CHUNK_BYTES = 64 * 1024
|
||||
|
||||
"""
|
||||
A part header block bigger than this is abuse, not a filename. Bounding it keeps
|
||||
the one genuinely unbounded-looking read (headers, which must be buffered whole
|
||||
to be parsed) from being a memory hole.
|
||||
"""
|
||||
const MAX_PART_HEADER_BYTES = 16 * 1024
|
||||
|
||||
const CRLF = UInt8[0x0d, 0x0a]
|
||||
const CRLFCRLF = UInt8[0x0d, 0x0a, 0x0d, 0x0a]
|
||||
const DASHDASH = UInt8[0x2d, 0x2d]
|
||||
|
||||
"A malformed (or truncated) multipart body. Callers turn this into a 400."
|
||||
struct MultipartError <: Exception
|
||||
msg::String
|
||||
end
|
||||
|
||||
Base.showerror(io::IO, e::MultipartError) = print(io, "MultipartError: ", e.msg)
|
||||
|
||||
"What a part's headers said about it. `filename === nothing` means a plain form field, not a file."
|
||||
struct MultipartPart
|
||||
name::Union{String,Nothing}
|
||||
filename::Union{String,Nothing}
|
||||
content_type::Union{String,Nothing}
|
||||
end
|
||||
|
||||
"""
|
||||
MultipartReader(io, boundary; chunk_bytes = UPLOAD_CHUNK_BYTES)
|
||||
|
||||
An incremental reader over the multipart body arriving on `io`. `boundary` is the
|
||||
value from the request's `Content-Type` header (see [`multipart_boundary`](@ref)).
|
||||
"""
|
||||
mutable struct MultipartReader{I<:IO}
|
||||
io::I
|
||||
dash_boundary::Vector{UInt8} # "--" boundary: opens the first part
|
||||
delimiter::Vector{UInt8} # CRLF "--" boundary: closes every part
|
||||
buf::Vector{UInt8} # rolling window; bounded by chunk_bytes + delimiter
|
||||
pos::Int # next unconsumed index in buf
|
||||
scratch::Vector{UInt8} # reused socket read target, so chunks don't churn the GC
|
||||
chunk_bytes::Int
|
||||
state::Symbol # :preamble | :at_delimiter | :body | :done
|
||||
end
|
||||
|
||||
function MultipartReader(io::IO, boundary::AbstractString;
|
||||
chunk_bytes::Int = UPLOAD_CHUNK_BYTES)
|
||||
chunk_bytes > 0 || throw(ArgumentError("chunk_bytes must be positive"))
|
||||
isempty(boundary) && throw(MultipartError("empty multipart boundary"))
|
||||
dash_boundary = Vector{UInt8}(codeunits(string("--", boundary)))
|
||||
return MultipartReader(io, dash_boundary, vcat(CRLF, dash_boundary),
|
||||
UInt8[], 1, Vector{UInt8}(undef, chunk_bytes),
|
||||
chunk_bytes, :preamble)
|
||||
end
|
||||
|
||||
"""
|
||||
multipart_boundary(content_type) -> String | nothing
|
||||
|
||||
Pull the boundary out of a `multipart/form-data` Content-Type header. Returns
|
||||
`nothing` if the header is missing, is some other media type, or has no boundary
|
||||
— all of which are the same 400 to a caller.
|
||||
"""
|
||||
function multipart_boundary(content_type::Union{AbstractString,Nothing})
|
||||
content_type === nothing && return nothing
|
||||
occursin(r"^\s*multipart/form-data"i, content_type) || return nothing
|
||||
m = match(r"(?i:\bboundary)=(?:\"([^\"]+)\"|([^\s;]+))", content_type)
|
||||
m === nothing && return nothing
|
||||
return String(something(m[1], m[2]))
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------ buffer plumbing
|
||||
|
||||
"Unconsumed bytes currently buffered."
|
||||
navail(r::MultipartReader) = length(r.buf) - r.pos + 1
|
||||
|
||||
"Drop already-consumed bytes so the buffer stays bounded across a long body."
|
||||
function compact!(r::MultipartReader)
|
||||
r.pos == 1 && return nothing
|
||||
n = navail(r)
|
||||
n > 0 && copyto!(r.buf, 1, r.buf, r.pos, n)
|
||||
resize!(r.buf, max(n, 0))
|
||||
r.pos = 1
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
Pull one more chunk off the wire, returning `false` at end of body.
|
||||
|
||||
`readbytes!` on an `HTTP.Stream` returns at most what remains of the current
|
||||
content-length or chunk, so this is bounded by `chunk_bytes`; `eof` is what
|
||||
advances a chunked-encoded body to its next chunk, hence the guard.
|
||||
"""
|
||||
function fill_more!(r::MultipartReader)
|
||||
eof(r.io) && return false
|
||||
n = readbytes!(r.io, r.scratch, r.chunk_bytes)
|
||||
n == 0 && return false
|
||||
append!(r.buf, view(r.scratch, 1:n))
|
||||
return true
|
||||
end
|
||||
|
||||
"Buffer until `needle` is found, returning its range, or `nothing` at end of body."
|
||||
function seek_needle!(r::MultipartReader, needle::Vector{UInt8}; limit::Int = 0)
|
||||
while true
|
||||
idx = findnext(needle, r.buf, r.pos)
|
||||
idx === nothing || return idx
|
||||
# Only the last length(needle)-1 bytes can still be part of a match, but
|
||||
# the caller may need the skipped bytes (a part body), so trimming is the
|
||||
# caller's job — we only enforce the optional limit.
|
||||
limit > 0 && navail(r) > limit &&
|
||||
throw(MultipartError("no delimiter within $limit bytes"))
|
||||
compact!(r)
|
||||
fill_more!(r) || return nothing
|
||||
end
|
||||
end
|
||||
|
||||
"Ensure at least `n` bytes are buffered; `false` if the body ended first."
|
||||
function ensure!(r::MultipartReader, n::Int)
|
||||
while navail(r) < n
|
||||
compact!(r)
|
||||
fill_more!(r) || return false
|
||||
end
|
||||
return true
|
||||
end
|
||||
|
||||
# Write `r.buf[range]` to `sink`. Goes through `unsafe_write` because
|
||||
# `write(io, ::SubArray{UInt8})` falls back to a byte-at-a-time loop in Base,
|
||||
# which would dominate the cost of a large upload.
|
||||
function emit!(sink::IO, r::MultipartReader, from::Int, to::Int)
|
||||
n = to - from + 1
|
||||
n <= 0 && return 0
|
||||
buf = r.buf # GC.@preserve needs a plain symbol, not a field access
|
||||
GC.@preserve buf unsafe_write(sink, pointer(buf, from), UInt(n))
|
||||
return n
|
||||
end
|
||||
|
||||
# ------------------------------------------------------------------- parts
|
||||
|
||||
"""
|
||||
next_part!(r) -> MultipartPart | nothing
|
||||
|
||||
Advance to the next part and return its headers, or `nothing` at the end of the
|
||||
form. The previous part's body must have been consumed first (with
|
||||
[`write_part_body!`](@ref) or [`skip_part_body!`](@ref)) — the reader cannot skip
|
||||
a body it hasn't been told to, because the body is only bounded by finding the
|
||||
next delimiter.
|
||||
"""
|
||||
function next_part!(r::MultipartReader)
|
||||
r.state === :done && return nothing
|
||||
r.state === :body &&
|
||||
throw(MultipartError("the current part's body must be consumed before the next part"))
|
||||
|
||||
if r.state === :preamble
|
||||
# Discard the preamble (RFC says ignore it) and consume the opening
|
||||
# delimiter. Bounded: real clients send no preamble at all, and an
|
||||
# unbounded scan here would be a way to make us buffer a whole body.
|
||||
idx = seek_needle!(r, r.dash_boundary; limit = MAX_PART_HEADER_BYTES)
|
||||
idx === nothing && throw(MultipartError("no multipart boundary found in body"))
|
||||
r.pos = last(idx) + 1
|
||||
r.state = :at_delimiter
|
||||
end
|
||||
|
||||
# Just after a delimiter: "--" ends the form, CRLF introduces another part.
|
||||
ensure!(r, 2) || throw(MultipartError("truncated body after a boundary delimiter"))
|
||||
if view(r.buf, r.pos:r.pos+1) == DASHDASH
|
||||
r.pos += 2
|
||||
r.state = :done
|
||||
return nothing
|
||||
end
|
||||
skip_linear_whitespace!(r)
|
||||
ensure!(r, 2) || throw(MultipartError("truncated body after a boundary delimiter"))
|
||||
view(r.buf, r.pos:r.pos+1) == CRLF ||
|
||||
throw(MultipartError("boundary delimiter is not followed by a line ending"))
|
||||
r.pos += 2
|
||||
|
||||
part = read_part_headers!(r)
|
||||
r.state = :body
|
||||
return part
|
||||
end
|
||||
|
||||
"RFC 2046 allows spaces/tabs between the delimiter and its line ending."
|
||||
function skip_linear_whitespace!(r::MultipartReader)
|
||||
while ensure!(r, 1) && (r.buf[r.pos] == 0x20 || r.buf[r.pos] == 0x09)
|
||||
r.pos += 1
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
function read_part_headers!(r::MultipartReader)
|
||||
# A part with no headers at all is `CRLF CRLF body`: the empty line comes
|
||||
# immediately, so searching for CRLFCRLF would run past it into the body.
|
||||
if ensure!(r, 2) && view(r.buf, r.pos:r.pos+1) == CRLF
|
||||
r.pos += 2
|
||||
return MultipartPart(nothing, nothing, nothing)
|
||||
end
|
||||
# The `limit` here bounds *buffering* — it only fires when the headers span
|
||||
# chunks. The explicit length check below is the actual policy, so the rule
|
||||
# doesn't depend on how the body happened to be chunked on the wire.
|
||||
idx = seek_needle!(r, CRLFCRLF; limit = MAX_PART_HEADER_BYTES)
|
||||
idx === nothing && throw(MultipartError("truncated body inside a part's headers"))
|
||||
first(idx) - r.pos > MAX_PART_HEADER_BYTES &&
|
||||
throw(MultipartError("part headers exceed $MAX_PART_HEADER_BYTES bytes"))
|
||||
# Copying is fine: the check above bounds this block.
|
||||
block = String(r.buf[r.pos:first(idx)-1])
|
||||
r.pos = last(idx) + 1
|
||||
return parse_part_headers(block)
|
||||
end
|
||||
|
||||
"Unescape the backslash escapes RFC 2045 allows inside a quoted-string."
|
||||
unquote(s::AbstractString) = replace(s, r"\\(.)" => s"\1")
|
||||
|
||||
function parse_part_headers(block::AbstractString)
|
||||
name = filename = content_type = nothing
|
||||
for line in eachsplit(block, "\r\n")
|
||||
colon = findfirst(':', line)
|
||||
colon === nothing && continue
|
||||
key = lowercase(strip(line[1:colon-1]))
|
||||
value = strip(line[colon+1:end])
|
||||
if key == "content-disposition"
|
||||
# `\b` matters: it keeps the `name=` pattern from matching inside `filename=`.
|
||||
m = match(r"(?i:\bname)=(?:\"((?:[^\"\\]|\\.)*)\"|([^\s;]+))", value)
|
||||
m === nothing || (name = unquote(String(something(m[1], m[2]))))
|
||||
m = match(r"(?i:\bfilename)=(?:\"((?:[^\"\\]|\\.)*)\"|([^\s;]+))", value)
|
||||
m === nothing || (filename = unquote(String(something(m[1], m[2]))))
|
||||
elseif key == "content-type"
|
||||
content_type = String(value)
|
||||
end
|
||||
end
|
||||
return MultipartPart(name, filename, content_type)
|
||||
end
|
||||
|
||||
"""
|
||||
write_part_body!(sink, r) -> Int
|
||||
|
||||
Stream the current part's body into `sink`, returning the number of bytes
|
||||
written. Nothing larger than a chunk is ever held in memory.
|
||||
"""
|
||||
function write_part_body!(sink::IO, r::MultipartReader)
|
||||
r.state === :body || throw(MultipartError("no part body is open"))
|
||||
total = 0
|
||||
keep = length(r.delimiter) - 1 # a delimiter may straddle two chunks
|
||||
while true
|
||||
idx = findnext(r.delimiter, r.buf, r.pos)
|
||||
if idx !== nothing
|
||||
total += emit!(sink, r, r.pos, first(idx) - 1)
|
||||
r.pos = last(idx) + 1
|
||||
r.state = :at_delimiter
|
||||
return total
|
||||
end
|
||||
# Emit only what cannot be the start of a straddling delimiter, then
|
||||
# keep that tail and read more.
|
||||
emit_to = length(r.buf) - keep
|
||||
if emit_to >= r.pos
|
||||
total += emit!(sink, r, r.pos, emit_to)
|
||||
r.pos = emit_to + 1
|
||||
end
|
||||
compact!(r)
|
||||
fill_more!(r) || throw(MultipartError("truncated body inside a part"))
|
||||
end
|
||||
end
|
||||
|
||||
"Consume and discard the current part's body (a form field, or a file we can't take)."
|
||||
skip_part_body!(r::MultipartReader) = write_part_body!(devnull, r)
|
||||
10
src/queue.jl
10
src/queue.jl
@@ -81,6 +81,16 @@ function close!(q::ChannelQueue)
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
capacity(q) -> Int
|
||||
|
||||
How many jobs the queue can hold before `enqueue!` starts refusing. Part of the
|
||||
introspection seam alongside `length`: `/stats` reports depth against capacity,
|
||||
because a depth of 900 means nothing without knowing whether the limit is 1000
|
||||
or 1_000_000.
|
||||
"""
|
||||
capacity(q::ChannelQueue) = q.capacity
|
||||
|
||||
"Number of jobs currently buffered (for logging/introspection)."
|
||||
function Base.length(q::ChannelQueue)
|
||||
lock(q.cond)
|
||||
|
||||
255
src/server.jl
255
src/server.jl
@@ -1,13 +1,21 @@
|
||||
# HTTP layer: a single multipart upload endpoint plus a health check.
|
||||
#
|
||||
# The handler's whole job is to get files onto the queue fast and get out of the
|
||||
# way: spool each uploaded file to disk, enqueue a reference, respond 202. It
|
||||
# way: stream each uploaded file to disk, enqueue a reference, respond 202. It
|
||||
# never does real processing — that's the workers' job.
|
||||
#
|
||||
# Intake is *streamed*, not buffered (see src/multipart.jl for the reader). This
|
||||
# is what makes the service's memory story hold end to end: bytes go from the
|
||||
# socket to the spool file a chunk at a time, so a 4 GB upload costs the same
|
||||
# resident memory as a 4 KB one. It is also why /upload is served by its own
|
||||
# stream handler rather than an Oxygen route — see `root_stream_handler`.
|
||||
#
|
||||
# NOTE: routes are registered at runtime via `register_routes()` (called from
|
||||
# `run`), NOT with top-level macros. In a precompiled package, top-level
|
||||
# `@get`/`@post` would execute during precompilation and be lost before serving.
|
||||
|
||||
const UPLOAD_PATH = "/upload"
|
||||
|
||||
jsonresp(status::Int, data) =
|
||||
HTTP.Response(status, ["Content-Type" => "application/json"], JSON3.write(data))
|
||||
|
||||
@@ -15,48 +23,215 @@ function health_handler(_::HTTP.Request)
|
||||
return jsonresp(200, (; status = "ok"))
|
||||
end
|
||||
|
||||
function upload_handler(req::HTTP.Request)
|
||||
cfg = CONFIG[]
|
||||
queue = QUEUE[]
|
||||
"""
|
||||
`GET /stats` — the pipeline's own counters (src/stats.jl), as JSON.
|
||||
|
||||
parts = try
|
||||
HTTP.parse_multipart_form(req)
|
||||
catch
|
||||
nothing
|
||||
end
|
||||
parts === nothing &&
|
||||
return jsonresp(400, (; error = "expected multipart/form-data"))
|
||||
Read-only and cheap: a few atomic loads and one `length` per queue, no pipeline
|
||||
state touched. Two scrapes Δt apart give per-stage throughput and utilization —
|
||||
see bin/bench.jl, which is the intended consumer.
|
||||
|
||||
files = filter(p -> p.filename !== nothing && !isempty(p.filename), parts)
|
||||
isempty(files) &&
|
||||
return jsonresp(400, (; error = "no files found in request"))
|
||||
|
||||
accepted = NamedTuple{(:id, :name),Tuple{String,String}}[]
|
||||
for p in files
|
||||
bytes = read(p.data)
|
||||
|
||||
job = try
|
||||
spool_file(cfg, p.filename, bytes)
|
||||
catch e
|
||||
@error "spool failed" name=p.filename exception=(e, catch_backtrace())
|
||||
return jsonresp(500, (; error = "failed to store file", accepted))
|
||||
end
|
||||
|
||||
if !enqueue!(queue, job)
|
||||
rm(job.path; force = true) # never queued → don't leave it in spool
|
||||
return jsonresp(503, (; error = "queue full, retry later", accepted))
|
||||
end
|
||||
|
||||
@info "accepted" id=job.id name=job.original_name size=job.size
|
||||
push!(accepted, (; id = job.id, name = job.original_name))
|
||||
end
|
||||
|
||||
return jsonresp(202, (; accepted))
|
||||
Unlike `/upload` this is an ordinary Oxygen route: it has no body to stream, and
|
||||
being in Oxygen's middleware chain is a feature here.
|
||||
"""
|
||||
function stats_handler(_::HTTP.Request)
|
||||
queues = (classify = QUEUE[], enrich = KNOWN_QUEUE[],
|
||||
triage = UNKNOWN_QUEUE[], language = TEXT_QUEUE[])
|
||||
return jsonresp(200, stats_snapshot(CONFIG[], queues))
|
||||
end
|
||||
|
||||
"Register HTTP routes on the Oxygen instance. Must run at runtime, before serve."
|
||||
function register_routes()
|
||||
@get("/health", health_handler)
|
||||
@post("/upload", upload_handler)
|
||||
"Write a JSON response onto a raw stream (the streaming handler's `jsonresp`)."
|
||||
function stream_jsonresp(stream::HTTP.Stream, status::Int, data)
|
||||
body = JSON3.write(data)
|
||||
HTTP.setstatus(stream, status)
|
||||
HTTP.setheader(stream, "Content-Type" => "application/json")
|
||||
HTTP.setheader(stream, "Content-Length" => string(sizeof(body)))
|
||||
HTTP.startwrite(stream)
|
||||
write(stream, body)
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
Read and discard whatever is left of the request body.
|
||||
|
||||
HTTP.jl's server calls `closeread` after the handler and *errors* if the body was
|
||||
only partly consumed — a half-read body can't be followed by another request on a
|
||||
keep-alive connection. So every exit path drains first. Discarding is bounded in
|
||||
memory (one chunk) and costs nothing in the normal case, where the body is
|
||||
already fully consumed and this returns immediately.
|
||||
"""
|
||||
function discard_body!(stream::HTTP.Stream, chunk_bytes::Int)
|
||||
scratch = Vector{UInt8}(undef, chunk_bytes)
|
||||
while !eof(stream)
|
||||
readbytes!(stream, scratch, chunk_bytes) == 0 && break
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
Did this exception mean the client went away, rather than something being wrong
|
||||
on our side?
|
||||
|
||||
A client that hangs up mid-upload (user cancels, network drops) is routine, and
|
||||
must not be logged as a server error or reported as a failed write — but it looks
|
||||
like an I/O failure from inside the parser, so the distinction has to be made
|
||||
explicitly. `EOFError` is what `HTTP.Stream` raises when a connection dies with
|
||||
bytes still promised by `Content-Length`.
|
||||
"""
|
||||
is_client_gone(e) =
|
||||
e isa EOFError ||
|
||||
(e isa Base.IOError && e.code in (Base.UV_EPIPE, Base.UV_ECONNRESET, Base.UV_ECONNABORTED))
|
||||
|
||||
"""
|
||||
Stream a multipart upload to disk, one part at a time.
|
||||
|
||||
Each file part is spooled straight from the socket and its reference enqueued.
|
||||
The response reports what was actually queued:
|
||||
|
||||
* `202` — every file in the request was spooled and queued
|
||||
* `400` — not multipart/form-data, no files present, or a malformed body
|
||||
* `503` — the intake queue filled up; `accepted` lists what got in first
|
||||
* `500` — a file could not be written to disk
|
||||
|
||||
Unlike a buffered handler, this one cannot know up front how many files a request
|
||||
holds or whether they will fit. So when the queue fills mid-request it does not
|
||||
abandon the connection: it stops spooling (discarding the remaining parts rather
|
||||
than writing files it can't queue), drains the body, and answers `503` with the
|
||||
`accepted` list. Files already queued stay queued — a client can retry the rest.
|
||||
"""
|
||||
function upload_stream_handler(stream::HTTP.Stream)
|
||||
try
|
||||
return serve_upload!(stream)
|
||||
catch e
|
||||
# The client vanished — while we were reading its body, draining it, or
|
||||
# answering. Nothing is wrong on our side, so log it as the routine event
|
||||
# it is instead of a server error.
|
||||
is_client_gone(e) || rethrow()
|
||||
@info "upload aborted by client"
|
||||
# HTTP.jl insists a handler write *some* response before returning. The
|
||||
# client is probably already gone, so this write is best-effort: attempt
|
||||
# it only if we haven't started a response, and let it fail silently.
|
||||
if isopen(stream) && !iswritable(stream)
|
||||
try
|
||||
stream_jsonresp(stream, 400, (; error = "upload truncated"))
|
||||
catch e2
|
||||
is_client_gone(e2) || rethrow()
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
end
|
||||
|
||||
function serve_upload!(stream::HTTP.Stream)
|
||||
Threads.atomic_add!(METRICS.intake.requests, 1)
|
||||
cfg = CONFIG[]
|
||||
queue = QUEUE[]
|
||||
chunk = cfg.upload_chunk_bytes
|
||||
|
||||
boundary = multipart_boundary(HTTP.header(stream.message, "Content-Type", nothing))
|
||||
if boundary === nothing
|
||||
discard_body!(stream, chunk)
|
||||
return stream_jsonresp(stream, 400, (; error = "expected multipart/form-data"))
|
||||
end
|
||||
|
||||
reader = MultipartReader(stream, boundary; chunk_bytes = chunk)
|
||||
accepted = NamedTuple{(:id, :name),Tuple{String,String}}[]
|
||||
n_files = 0
|
||||
queue_full = false
|
||||
failure = nothing # (status, message) from a fatal error mid-body
|
||||
|
||||
try
|
||||
while (part = next_part!(reader)) !== nothing
|
||||
# A part with no filename is an ordinary form field, not a file.
|
||||
if part.filename === nothing || isempty(part.filename)
|
||||
skip_part_body!(reader)
|
||||
continue
|
||||
end
|
||||
n_files += 1
|
||||
|
||||
# Already backpressured: consume the part, but don't write a file we
|
||||
# know we cannot enqueue.
|
||||
if queue_full
|
||||
Threads.atomic_add!(METRICS.intake.rejected, 1)
|
||||
skip_part_body!(reader)
|
||||
continue
|
||||
end
|
||||
|
||||
job = try
|
||||
spool_stream(cfg, part.filename) do io
|
||||
write_part_body!(io, reader)
|
||||
end
|
||||
catch e
|
||||
(e isa MultipartError || is_client_gone(e)) && rethrow()
|
||||
# A disk error leaves the reader mid-part, so the rest of the body
|
||||
# can no longer be parsed — stop and report.
|
||||
@error "spool failed" name=part.filename exception=(e, catch_backtrace())
|
||||
failure = (500, "failed to store file")
|
||||
break
|
||||
end
|
||||
|
||||
if enqueue!(queue, job)
|
||||
@info "accepted" id=job.id name=job.original_name size=job.size
|
||||
push!(accepted, (; id = job.id, name = job.original_name))
|
||||
# Counted at the point the job becomes stage 1's problem, so
|
||||
# intake totals and stage-1 arrivals refer to the same files.
|
||||
Threads.atomic_add!(METRICS.intake.files, 1)
|
||||
Threads.atomic_add!(METRICS.intake.bytes, job.size)
|
||||
else
|
||||
rm(job.path; force = true) # never queued → don't leave it in spool
|
||||
Threads.atomic_add!(METRICS.intake.rejected, 1)
|
||||
queue_full = true
|
||||
end
|
||||
end
|
||||
catch e
|
||||
e isa MultipartError || rethrow()
|
||||
@warn "malformed multipart upload" reason=e.msg accepted=length(accepted)
|
||||
failure = (400, "malformed multipart body")
|
||||
end
|
||||
|
||||
discard_body!(stream, chunk)
|
||||
|
||||
failure !== nothing &&
|
||||
return stream_jsonresp(stream, failure[1], (; error = failure[2], accepted))
|
||||
n_files == 0 &&
|
||||
return stream_jsonresp(stream, 400, (; error = "no files found in request"))
|
||||
queue_full &&
|
||||
return stream_jsonresp(stream, 503, (; error = "queue full, retry later", accepted))
|
||||
return stream_jsonresp(stream, 202, (; accepted))
|
||||
end
|
||||
|
||||
"""
|
||||
root_stream_handler(middleware) -> (stream -> nothing)
|
||||
|
||||
Oxygen's root handler wraps `HTTP.streamhandler`, which does
|
||||
`request.body = read(stream)` — the entire body into memory — *before* any route
|
||||
is dispatched. That happens even for an Oxygen `@stream` route, so no route can
|
||||
stream an upload. We therefore intercept `POST /upload` at the stream level and
|
||||
hand everything else to Oxygen unchanged.
|
||||
|
||||
Trade-off: `/upload` bypasses Oxygen's middleware chain, so it is absent from
|
||||
Oxygen's built-in metrics and docs. Deliberate — flat intake memory is the point
|
||||
of this service, and intake is covered by our own counters (`/stats`) and `@info`
|
||||
records anyway.
|
||||
"""
|
||||
function root_stream_handler(middleware::Function)
|
||||
oxygen_handler = Oxygen.Core.stream_handler(middleware)
|
||||
return function (stream::HTTP.Stream)
|
||||
req = stream.message
|
||||
if req.method == "POST" && HTTP.URI(req.target).path == UPLOAD_PATH
|
||||
return upload_stream_handler(stream)
|
||||
end
|
||||
return oxygen_handler(stream)
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
Register HTTP routes on the Oxygen instance. Must run at runtime, before serve.
|
||||
|
||||
`POST /upload` is deliberately absent: it is served by `upload_stream_handler`
|
||||
via `root_stream_handler`, ahead of Oxygen's router.
|
||||
"""
|
||||
function register_routes()
|
||||
@get("/health", health_handler)
|
||||
@get("/stats", stats_handler)
|
||||
return nothing
|
||||
end
|
||||
|
||||
53
src/spool.jl
53
src/spool.jl
@@ -19,15 +19,33 @@ function sanitize_filename(name::AbstractString)::String
|
||||
return first(base, MAX_NAME_LEN)
|
||||
end
|
||||
|
||||
"Write `bytes` to the spool dir under `<uuid>-<sanitized>` and return the Job."
|
||||
function spool_file(cfg::Config, original_name::AbstractString, bytes::Vector{UInt8})::Job
|
||||
id = string(uuid4())
|
||||
safe = sanitize_filename(original_name)
|
||||
path = joinpath(cfg.spool_dir, string(id, "-", safe))
|
||||
open(path, "w") do io
|
||||
write(io, bytes)
|
||||
"Build the spool path for a client-supplied name: `<uuid>-<sanitized>`."
|
||||
function spool_path(cfg::Config, original_name::AbstractString)
|
||||
id = string(uuid4())
|
||||
return id, joinpath(cfg.spool_dir, string(id, "-", sanitize_filename(original_name)))
|
||||
end
|
||||
|
||||
"""
|
||||
spool_stream(write_body!, cfg, original_name) -> Job
|
||||
|
||||
Create the spool file for `original_name`, hand the open `IO` to `write_body!`,
|
||||
and build the `Job` from however many bytes it reports writing.
|
||||
|
||||
This is the streaming counterpart to `spool_file`: the caller pumps bytes in from
|
||||
the network as they arrive, so a file never exists in memory in one piece. A
|
||||
partial file left by a failed or abandoned write is removed — intake either
|
||||
produces a complete spooled file or nothing at all, so recovery on restart never
|
||||
picks up a truncated upload.
|
||||
"""
|
||||
function spool_stream(write_body!, cfg::Config, original_name::AbstractString)::Job
|
||||
id, path = spool_path(cfg, original_name)
|
||||
nbytes = try
|
||||
open(write_body!, path, "w")
|
||||
catch
|
||||
rm(path; force = true)
|
||||
rethrow()
|
||||
end
|
||||
return Job(id, String(original_name), path, length(bytes), time())
|
||||
return Job(id, String(original_name), path, nbytes, time())
|
||||
end
|
||||
|
||||
"Move a spooled file into `dir` (done/ or failed/), returning the destination."
|
||||
@@ -42,17 +60,22 @@ end
|
||||
const UUID_LEN = 36
|
||||
|
||||
"""
|
||||
recover_spool!(cfg, queue) -> Int
|
||||
recover_dir!(dir, queue) -> Int
|
||||
|
||||
Re-enqueue any files already sitting in the spool directory (left by a crash,
|
||||
a hard shutdown, or an intake that never got processed). This is the payoff of
|
||||
spooling to disk: a restart resumes work instead of stranding it. Returns the
|
||||
number of files recovered.
|
||||
Re-enqueue any files sitting in `dir` (left by a crash, hard shutdown, or an
|
||||
intake that never finished) onto `queue`. This is the payoff of spooling to
|
||||
disk: a restart resumes work instead of stranding it. Stage-aware recovery uses
|
||||
one call per stage — `spool/` → stage-1 queue, `known/` → known queue — so each
|
||||
file re-enters at the correct stage rather than being reclassified from scratch.
|
||||
Returns the number of files recovered.
|
||||
|
||||
Skips `.meta.json` sidecars: those are stage-2 output, not work to redo.
|
||||
"""
|
||||
function recover_spool!(cfg::Config, queue::JobQueue)::Int
|
||||
function recover_dir!(dir::AbstractString, queue::JobQueue)::Int
|
||||
n = 0
|
||||
for path in sort(readdir(cfg.spool_dir; join=true))
|
||||
for path in sort(readdir(dir; join=true))
|
||||
isfile(path) || continue
|
||||
endswith(path, ".meta.json") && continue # sidecar, not a work item
|
||||
fname = basename(path)
|
||||
if length(fname) > UUID_LEN + 1
|
||||
id = fname[1:UUID_LEN]
|
||||
|
||||
211
src/stats.jl
Normal file
211
src/stats.jl
Normal file
@@ -0,0 +1,211 @@
|
||||
# Per-stage pipeline metrics.
|
||||
#
|
||||
# End-to-end throughput says how fast the pipeline is; it does not say which
|
||||
# stage is the reason. The external harness can't answer that either: `known/`,
|
||||
# `unknown/` and `text/` are *transient* — a file can pass through one between
|
||||
# two directory polls — so sampling directories from outside undercounts exactly
|
||||
# the stages we most want to measure. Only the pipeline itself sees every job.
|
||||
#
|
||||
# So each stage keeps four counters, all incremented in `worker_loop` (the one
|
||||
# place every stage's work passes through) and read by `GET /stats`:
|
||||
#
|
||||
# completed / failed jobs finished, jobs quarantined → throughput
|
||||
# bytes job bytes processed → per-stage MiB/s
|
||||
# busy_ns summed handler wall time → service time
|
||||
# blocked_ns of that, time parked on a full downstream queue
|
||||
# in_flight handlers running right now → saturation
|
||||
#
|
||||
# Rates fall out of a pair of scrapes taken Δt apart:
|
||||
#
|
||||
# throughput = Δcompleted / Δt
|
||||
# utilization = (Δbusy_ns - Δblocked_ns) / (Δt * workers) ~1.0 ⇒ bottleneck
|
||||
#
|
||||
# Utilization is the useful one. Throughput alone can't distinguish a stage that
|
||||
# is saturated from one that is merely starved by the stage ahead of it — both
|
||||
# show the same files/s — whereas utilization separates them: the saturated stage
|
||||
# sits near 1.0 with its queue backing up, the starved stage sits near 0.
|
||||
#
|
||||
# `blocked_ns` is what keeps that true downstream. Stages 1 and 3 apply blocking
|
||||
# backpressure: when the next queue is full the handler parks and retries rather
|
||||
# than dropping the file (see ROUTE_ENQUEUE_RETRY_SECONDS). That parked time is
|
||||
# inside the handler, so counting it as busy would show stage 1 pinned at 1.0
|
||||
# whenever stage 2 is the real bottleneck — every stage upstream of the jam would
|
||||
# look like the jam. Subtracting it leaves utilization meaning "doing its own
|
||||
# work", and the blocked share becomes its own signal: a stage blocked 90% of the
|
||||
# time is naming its successor as the bottleneck.
|
||||
#
|
||||
# Everything here is monotonic since process start (or since `reset_metrics!`),
|
||||
# in the Prometheus style: counters, never rates. Rates are the reader's job, so
|
||||
# a scrape is stateless and two readers can't disturb each other.
|
||||
#
|
||||
# Cost is a handful of atomic adds per file, against handlers that spawn
|
||||
# exiftool — unmeasurable in practice, and the counters are never read on the
|
||||
# hot path.
|
||||
|
||||
# Stage identity lives here, in declaration order, so the report, the JSON and
|
||||
# the worker wiring can't drift apart: adding stage 5 to the live path means
|
||||
# adding it here and passing the new `StageStats` to its `worker_loop`.
|
||||
const STAGE_KEYS = (:classify, :enrich, :triage, :language)
|
||||
const STAGE_TITLES = (classify = "classify", enrich = "enrich",
|
||||
triage = "triage", language = "language")
|
||||
|
||||
"""
|
||||
Counters for one pipeline stage. All fields are atomic and monotonic: workers
|
||||
only ever add, readers only ever read, so no lock is needed between them.
|
||||
|
||||
`in_flight` is the exception to monotonic — it goes up and down — and is the one
|
||||
counter that is a *level* rather than a total.
|
||||
"""
|
||||
struct StageStats
|
||||
completed::Threads.Atomic{Int}
|
||||
failed::Threads.Atomic{Int}
|
||||
bytes::Threads.Atomic{Int}
|
||||
busy_ns::Threads.Atomic{Int}
|
||||
blocked_ns::Threads.Atomic{Int}
|
||||
in_flight::Threads.Atomic{Int}
|
||||
end
|
||||
|
||||
StageStats() = StageStats(Threads.Atomic{Int}(0), Threads.Atomic{Int}(0),
|
||||
Threads.Atomic{Int}(0), Threads.Atomic{Int}(0),
|
||||
Threads.Atomic{Int}(0), Threads.Atomic{Int}(0))
|
||||
|
||||
"""
|
||||
Counters for HTTP intake, which has no worker loop to hang them off.
|
||||
|
||||
`files` counts what was actually spooled *and* queued — the same thing the 202
|
||||
body reports as `accepted` — so it lines up with stage 1's arrivals. Files
|
||||
dropped because the queue was full are counted separately in `rejected`, since a
|
||||
run where intake outruns the pipeline should be visible as backpressure rather
|
||||
than as slow intake.
|
||||
"""
|
||||
struct IntakeStats
|
||||
requests::Threads.Atomic{Int}
|
||||
files::Threads.Atomic{Int}
|
||||
bytes::Threads.Atomic{Int}
|
||||
rejected::Threads.Atomic{Int}
|
||||
end
|
||||
|
||||
IntakeStats() = IntakeStats(Threads.Atomic{Int}(0), Threads.Atomic{Int}(0),
|
||||
Threads.Atomic{Int}(0), Threads.Atomic{Int}(0))
|
||||
|
||||
"""
|
||||
Every counter in the process, plus the wall-clock origin the totals are measured
|
||||
from.
|
||||
|
||||
`since` matters to a reader computing rates from a single scrape: without it,
|
||||
"1000 files completed" has no denominator. Readers that scrape twice should use
|
||||
their own Δt instead — it excludes the time before they started watching.
|
||||
"""
|
||||
struct Metrics
|
||||
intake::IntakeStats
|
||||
stages::NamedTuple{STAGE_KEYS,NTuple{4,StageStats}}
|
||||
since::Base.RefValue{Float64}
|
||||
end
|
||||
|
||||
Metrics() = Metrics(IntakeStats(),
|
||||
NamedTuple{STAGE_KEYS}(ntuple(_ -> StageStats(), 4)),
|
||||
Ref(time()))
|
||||
|
||||
# Process-global, like the logger: workers on every thread add to it and the
|
||||
# HTTP layer reads it, with no way to thread a handle through both paths that
|
||||
# wouldn't just be this by another name.
|
||||
const METRICS = Metrics()
|
||||
|
||||
"""
|
||||
reset_metrics!()
|
||||
|
||||
Zero every counter and restart the measurement window. For tests, and for a
|
||||
benchmark that wants totals covering only its own run — though a harness that
|
||||
scrapes before and after can subtract instead, which is safer against a server
|
||||
that is also serving someone else.
|
||||
"""
|
||||
function reset_metrics!(m::Metrics = METRICS)
|
||||
for a in (m.intake.requests, m.intake.files, m.intake.bytes, m.intake.rejected)
|
||||
a[] = 0
|
||||
end
|
||||
for s in m.stages
|
||||
s.completed[] = 0
|
||||
s.failed[] = 0
|
||||
s.bytes[] = 0
|
||||
s.busy_ns[] = 0
|
||||
s.blocked_ns[] = 0
|
||||
s.in_flight[] = 0
|
||||
end
|
||||
m.since[] = time()
|
||||
return nothing
|
||||
end
|
||||
|
||||
"Record one finished job: `ok` distinguishes a completion from a quarantine."
|
||||
function record_job!(s::StageStats, ok::Bool, bytes::Int, elapsed_ns::Int)
|
||||
Threads.atomic_add!(ok ? s.completed : s.failed, 1)
|
||||
Threads.atomic_add!(s.bytes, bytes)
|
||||
Threads.atomic_add!(s.busy_ns, elapsed_ns)
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
enqueue_blocking!(queue, job, stats; retry_seconds) -> nothing
|
||||
|
||||
Hand `job` to a downstream queue, parking and retrying until it fits, and charge
|
||||
the parked time to `stats.blocked_ns`.
|
||||
|
||||
This is the routing half of stages 1 and 3: a classified file is never dropped,
|
||||
so a full downstream queue means waiting, not failing. Wrapping the retry loop
|
||||
here — rather than repeating `while !enqueue! sleep end` at each call site — is
|
||||
what makes that wait measurable at all, and keeps the three routing paths from
|
||||
drifting into three different backoff behaviours.
|
||||
"""
|
||||
function enqueue_blocking!(queue::JobQueue, job::Job, stats::StageStats;
|
||||
retry_seconds::Real)
|
||||
enqueue!(queue, job) && return nothing
|
||||
t0 = time_ns()
|
||||
while !enqueue!(queue, job)
|
||||
sleep(retry_seconds)
|
||||
end
|
||||
Threads.atomic_add!(stats.blocked_ns, Int(time_ns() - t0))
|
||||
return nothing
|
||||
end
|
||||
|
||||
# --- snapshot ---------------------------------------------------------------
|
||||
|
||||
"""
|
||||
stats_snapshot(cfg, queues, m = METRICS) -> NamedTuple
|
||||
|
||||
Read every counter into a plain value tree for `GET /stats`.
|
||||
|
||||
`queues` is a NamedTuple of `JobQueue`s keyed by `STAGE_KEYS`, supplying each
|
||||
stage's live depth and capacity — the counters above are about work *done*, and
|
||||
a stage's depth is what says whether work is piling up in front of it.
|
||||
|
||||
The read is not atomic across stages: counters keep moving while we walk them,
|
||||
so a snapshot can show a job counted as complete by stage 1 but not yet arrived
|
||||
at stage 2. Over a benchmark window that skew is a job or two and does not move
|
||||
a rate; a consistent snapshot would mean stopping the pipeline to read it.
|
||||
"""
|
||||
function stats_snapshot(cfg::Config, queues, m::Metrics = METRICS)
|
||||
workers = (classify = cfg.worker_count, enrich = cfg.known_worker_count,
|
||||
triage = cfg.unknown_worker_count, language = cfg.text_worker_count)
|
||||
now = time()
|
||||
stages = map(STAGE_KEYS, ntuple(identity, 4)) do key, i
|
||||
s, q = m.stages[key], queues[key]
|
||||
(; stage = i,
|
||||
name = STAGE_TITLES[key],
|
||||
workers = workers[key],
|
||||
queue_depth = length(q),
|
||||
queue_capacity = capacity(q),
|
||||
completed = s.completed[],
|
||||
failed = s.failed[],
|
||||
bytes = s.bytes[],
|
||||
busy_seconds = s.busy_ns[] / 1e9,
|
||||
blocked_seconds = s.blocked_ns[] / 1e9,
|
||||
in_flight = s.in_flight[])
|
||||
end
|
||||
return (; now,
|
||||
since = m.since[],
|
||||
uptime_seconds = now - m.since[],
|
||||
intake = (; requests = m.intake.requests[],
|
||||
files = m.intake.files[],
|
||||
bytes = m.intake.bytes[],
|
||||
rejected = m.intake.rejected[]),
|
||||
stages = collect(stages))
|
||||
end
|
||||
164
src/worker.jl
164
src/worker.jl
@@ -1,46 +1,172 @@
|
||||
# Worker task: pull jobs off the queue and process them. One of these runs per
|
||||
# configured worker, each as its own `Threads.@spawn`'d task.
|
||||
# Worker tasks: pull jobs off a queue and process them. The loop scaffolding
|
||||
# (dequeue-until-drained, try/catch, quarantine-on-throw) is identical for every
|
||||
# stage, so `worker_loop` is parametrized with a `handler` and reused. Today
|
||||
# there are four stages:
|
||||
#
|
||||
# stage 1 handle_classify_job spool/ → classify → known/ (+known queue) | unknown/ (+unknown queue)
|
||||
# stage 2 handle_known_job known/ → exiftool enrich → done/ (+ .meta.json)
|
||||
# stage 3 handle_unknown_job unknown/ → binary-vs-text sniff → binary/ | text/ (+text queue)
|
||||
# stage 4 handle_text_job text/ → language enrich → text_done/ (+ .meta.json)
|
||||
#
|
||||
# Adding a stage later is just another queue + pool + handler; the loop below
|
||||
# doesn't change.
|
||||
|
||||
# How long a stage-1 worker backs off before retrying an enqueue onto a full
|
||||
# downstream queue (known or unknown). Blocking backpressure: a classified file
|
||||
# is never dropped, so the stage-1 worker parks until the next stage makes room.
|
||||
# Keeps intake decoupled — the HTTP path's `enqueue!` stays non-blocking; only
|
||||
# this worker-to-worker handoff blocks.
|
||||
const ROUTE_ENQUEUE_RETRY_SECONDS = 0.05
|
||||
|
||||
# Per-file logging in stage 1 is `@debug`, not `@info`, because it is the
|
||||
# stage's dominant cost. Measured by bin/bench_stage1.jl (2000 x 64 KiB files,
|
||||
# min of 5 trials): the two log lines cost ~71 µs of the ~118 µs
|
||||
# `handle_classify_job` spent per file — roughly 6x the classifier (10.6 µs) and
|
||||
# 6x the rename (11.6 µs). Nearly all of it is `ConsoleLogger` formatting
|
||||
# (~64 µs); the FlushLogger's per-message flush is only ~8 µs on top. Demoting
|
||||
# them takes stage 1 from ~8.5k files/s to ~35k files/s on one worker.
|
||||
#
|
||||
# `@debug` is compiled to a min-level check that doesn't evaluate its arguments,
|
||||
# so a disabled line costs ~0.15 µs rather than ~36 µs. The messages are still
|
||||
# there when wanted: run with `JULIA_DEBUG=FileServer` to get them back. Errors,
|
||||
# quarantines and lifecycle events stay at `@error`/`@info` — they are rare and
|
||||
# their cost doesn't scale with throughput. `GET /stats` (src/stats.jl) is the
|
||||
# per-file observability that survives, and it is counted, not formatted.
|
||||
|
||||
"""
|
||||
handle_job(job, cfg, worker_id)
|
||||
handle_classify_job(job, cfg, worker_id, known_queue, unknown_queue)
|
||||
|
||||
Do the work for a single job, then move the file to `done/`.
|
||||
Stage 1. Classify the spooled file and route it to the next stage's queue,
|
||||
retrying on a full queue rather than dropping the file:
|
||||
* `:known` → move to `known/`, enqueue onto the known queue for stage 2.
|
||||
* `:unknown` → move to `unknown/`, enqueue onto the unknown queue for stage 3.
|
||||
|
||||
For now the "work" is just logging the received filename to prove the flow —
|
||||
this is the seam where real heavy-lifting will go later.
|
||||
In both cases move first so the file physically lives in its stage dir before the
|
||||
reference is visible downstream; the moved path becomes the routed job's location.
|
||||
|
||||
Sub-`MIN_FILE_BYTES` files short-circuit to `:unknown` inside `classify`.
|
||||
"""
|
||||
function handle_job(job::Job, cfg::Config, worker_id::Int)
|
||||
# Classify the spooled file (annotate-only for now: the result is logged but
|
||||
# every file still moves to done/ regardless of known/unknown). Sub-32-byte
|
||||
# files short-circuit to :unknown inside classify without touching the model.
|
||||
function handle_classify_job(job::Job, cfg::Config, worker_id::Int,
|
||||
known_queue::JobQueue, unknown_queue::JobQueue,
|
||||
stats::StageStats)
|
||||
classification = classify(CLASSIFIER[], job.path)
|
||||
@info "received file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification
|
||||
dest = move_to(cfg.done_dir, job)
|
||||
@info "completed" worker=worker_id id=job.id dest=dest
|
||||
@debug "classified file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification
|
||||
|
||||
if classification === :known
|
||||
dest = move_to(cfg.known_dir, job)
|
||||
routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
|
||||
# known queue full → park and retry, don't drop (time charged to blocked_ns)
|
||||
enqueue_blocking!(known_queue, routed, stats;
|
||||
retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
|
||||
@debug "routed to enrichment" worker=worker_id id=job.id dest=dest
|
||||
else
|
||||
dest = move_to(cfg.unknown_dir, job)
|
||||
routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
|
||||
enqueue_blocking!(unknown_queue, routed, stats;
|
||||
retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
|
||||
@debug "routed to content triage" worker=worker_id id=job.id dest=dest
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
worker_loop(worker_id, cfg, queue)
|
||||
handle_known_job(job, cfg, worker_id)
|
||||
|
||||
Consume jobs until the queue is closed and drained. A failure on one job is
|
||||
logged and the file is quarantined in `failed/` — it must never kill the
|
||||
worker, or the pool would silently shrink.
|
||||
Stage 2. Extract metadata (exiftool, with timeout) and enrich: build the
|
||||
normalized sidecar and commit both to `done/` sidecar-first. Extraction
|
||||
failure/timeout yields a *degraded* sidecar (the file is still a wanted known
|
||||
file), so the only way to land in `failed/` is a genuine I/O error writing the
|
||||
sidecar or moving the file — handled by `worker_loop`'s quarantine.
|
||||
"""
|
||||
function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue)
|
||||
function handle_known_job(job::Job, cfg::Config, worker_id::Int)
|
||||
meta = build_metadata(job, cfg)
|
||||
file_dest, sidecar = finalize_known!(cfg, job, meta)
|
||||
@info "enriched" worker=worker_id id=job.id dest=file_dest sidecar=basename(sidecar) file_type=meta.file_type created_by=meta.created_by degraded=(meta.error !== nothing)
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
handle_unknown_job(job, cfg, worker_id, text_queue)
|
||||
|
||||
Stage 3. Sort an unrecognized file into a coarse content bucket by sniffing its
|
||||
first bytes: `binary/` (terminal — no further stage) if it looks like binary
|
||||
data, `text/` otherwise. A text file is then routed onward to the stage-4
|
||||
language-enrichment queue, retrying on a full queue rather than dropping the file
|
||||
(the same blocking backpressure stage 1 uses for its downstream queues).
|
||||
"""
|
||||
function handle_unknown_job(job::Job, cfg::Config, worker_id::Int,
|
||||
text_queue::JobQueue, stats::StageStats)
|
||||
if is_binary(job.path)
|
||||
dest = move_to(cfg.binary_dir, job)
|
||||
@info "sorted unknown" worker=worker_id id=job.id name=job.original_name kind=:binary dest=dest
|
||||
else
|
||||
dest = move_to(cfg.text_dir, job)
|
||||
routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
|
||||
# text queue full → park and retry, don't drop (time charged to blocked_ns)
|
||||
enqueue_blocking!(text_queue, routed, stats;
|
||||
retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
|
||||
@info "routed to language enrichment" worker=worker_id id=job.id dest=dest
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
handle_text_job(job, cfg, worker_id, detector)
|
||||
|
||||
Stage 4. Enrich a text file with its natural language (via `detector`) and
|
||||
programming language (via github-linguist): build the sidecar and commit both to
|
||||
`text_done/` sidecar-first. Detection failure yields a *degraded* sidecar (the
|
||||
file is still wanted text), so the only way to land in `failed/` is a genuine I/O
|
||||
error committing — handled by `worker_loop`'s quarantine.
|
||||
"""
|
||||
function handle_text_job(job::Job, cfg::Config, worker_id::Int, detector)
|
||||
meta = build_text_metadata(detector, job, cfg)
|
||||
file_dest, sidecar = finalize_text!(cfg, job, meta)
|
||||
@info "enriched text" worker=worker_id id=job.id dest=file_dest sidecar=basename(sidecar) language=meta.language confidence=meta.language_confidence programming_language=meta.programming_language degraded=(meta.error !== nothing)
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
worker_loop(worker_id, cfg, queue, handler, stats)
|
||||
|
||||
Consume jobs from `queue` until it is closed and drained, running `handler` on
|
||||
each. A failure on one job is logged and the file quarantined in `failed/` — it
|
||||
must never kill the worker, or the pool would silently shrink.
|
||||
|
||||
This loop is also where per-stage metrics are recorded (`stats`, see
|
||||
src/stats.jl). Instrumenting here rather than in each handler means every stage
|
||||
is measured the same way, by construction, and a new stage is measured the
|
||||
moment it is wired up — there is no per-handler bookkeeping to forget.
|
||||
|
||||
The timed region is the handler alone, excluding the `dequeue!` above it: time
|
||||
parked waiting for work is idleness, and counting it as service time would make
|
||||
an idle stage look as busy as a saturated one. A quarantined job still counts
|
||||
its time — the work was done, it just ended in `failed/`.
|
||||
"""
|
||||
function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue, handler,
|
||||
stats::StageStats)
|
||||
@info "worker started" worker=worker_id
|
||||
while true
|
||||
job = dequeue!(queue)
|
||||
job === nothing && break # queue closed and drained → exit
|
||||
Threads.atomic_add!(stats.in_flight, 1)
|
||||
t0 = time_ns()
|
||||
ok = true
|
||||
try
|
||||
handle_job(job, cfg, worker_id)
|
||||
handler(job, cfg, worker_id)
|
||||
catch e
|
||||
ok = false
|
||||
@error "processing failed" worker=worker_id id=job.id name=job.original_name exception=(e, catch_backtrace())
|
||||
try
|
||||
move_to(cfg.failed_dir, job)
|
||||
catch e2
|
||||
@error "could not quarantine failed file" worker=worker_id id=job.id path=job.path exception=(e2, catch_backtrace())
|
||||
end
|
||||
finally
|
||||
# In a `finally` so an InterruptException during shutdown can't leave
|
||||
# in_flight permanently above zero, which would read as a stuck job.
|
||||
record_job!(stats, ok, job.size, Int(time_ns() - t0))
|
||||
Threads.atomic_sub!(stats.in_flight, 1)
|
||||
end
|
||||
end
|
||||
@info "worker stopped" worker=worker_id
|
||||
|
||||
1074
test/runtests.jl
Normal file
1074
test/runtests.jl
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user