Initial text cleanup
This commit is contained in:
@@ -119,7 +119,7 @@ end
|
||||
@test p.filename == "a b.txt"
|
||||
@test p.content_type == "text/plain"
|
||||
|
||||
# `name=` must not match inside `filename=` — that would label every
|
||||
# `name=` must not match inside `filename=`; that would label every
|
||||
# file part with a bogus name and (worse) hide a missing real name.
|
||||
p = parse_part_headers("Content-Disposition: form-data; filename=\"only.txt\"")
|
||||
@test p.name === nothing
|
||||
@@ -290,7 +290,7 @@ end
|
||||
@testset "run_with_timeout: returns as soon as the child exits" begin
|
||||
# Regression guard. The original implementation polled with sleep(0.1)
|
||||
# and joined the polling task, so every call paid the remainder of an
|
||||
# in-flight sleep after the child had already exited — ~101 ms on a
|
||||
# in-flight sleep after the child had already exited: ~101 ms on a
|
||||
# process that exits instantly, on the hot path of stages 2 and 4. The
|
||||
# bound here is deliberately loose (a loaded CI box is slow) but far
|
||||
# under the 100 ms floor the polling version could not beat.
|
||||
@@ -369,7 +369,7 @@ end
|
||||
write(txt, "hello, world\nsecond line\n")
|
||||
@test is_binary(txt) == false
|
||||
|
||||
# Non-ASCII UTF-8 (accents, CJK, emoji) is valid text — the whole
|
||||
# Non-ASCII UTF-8 (accents, CJK, emoji) is valid text, which is the
|
||||
# point of moving off the printable-ASCII/NUL heuristic.
|
||||
uni = joinpath(root, "unicode.txt")
|
||||
write(uni, "café — 日本語 — 🚀\n")
|
||||
@@ -434,7 +434,7 @@ end
|
||||
# A text file is routed onto the stage-4 queue and *does not move*:
|
||||
# stage 4 is still to come, so the bytes stay in spool/ and the same
|
||||
# reference is handed on. Regression guard against re-introducing an
|
||||
# intermediate hop — the queue, not a directory, is what records that
|
||||
# intermediate hop: the queue, not a directory, is what records that
|
||||
# this file has cleared triage.
|
||||
tpath = joinpath(cfg.spool_dir, "id-t-notes.log")
|
||||
write(tpath, "just some log text\n")
|
||||
@@ -451,7 +451,7 @@ end
|
||||
@testset "handle_classify_job: routes by enqueue only, never moves the file" begin
|
||||
# Stage 1's entire job is picking a queue. Whichever way the classifier
|
||||
# votes, the bytes must stay where intake wrote them and the *same* Job
|
||||
# must be handed on — the queue records the phase, not a directory.
|
||||
# must be handed on: the queue records the phase, not a directory.
|
||||
# Asserted against both queues at once so the test doesn't depend on how
|
||||
# the committed model happens to label these bytes.
|
||||
CLASSIFIER[] = load_classifier(joinpath(@__DIR__, "..", "model", "classifier.jld2"))
|
||||
@@ -660,7 +660,7 @@ end
|
||||
end
|
||||
|
||||
@testset "cluster: §10.1 discovers nothing from noise" begin
|
||||
# 25 independent random blobs — the shape of data/binary (structureless
|
||||
# 25 independent random blobs: the shape of data/binary (structureless
|
||||
# junk). Correct output: ZERO promoted clusters (random headers never
|
||||
# form a ≥20-member, ≥3-magic-byte signature). See DESIGN §10.1.
|
||||
rng = MersenneTwister(20260703)
|
||||
@@ -681,9 +681,9 @@ end
|
||||
@testset "cluster: §10.2 recovers known (synthetic) formats" begin
|
||||
# Four synthetic "formats": a fixed magic prefix + random tail, mirroring
|
||||
# gzip/PDF/JPEG/ELF. Calibrated settings must recover them as clean,
|
||||
# promotable clusters at high ARI — the magic-collapsed recovery of §10.2,
|
||||
# promotable clusters at high ARI: the magic-collapsed recovery of §10.2,
|
||||
# here with a hermetic, deterministic corpus.
|
||||
# ~12-byte constant headers + random tails — the shape of a real file
|
||||
# ~12-byte constant headers + random tails: the shape of a real file
|
||||
# header (a fixed magic/version region, then variable content). A too-short
|
||||
# magic over a fully-random tail is adversarially hard and lets a format
|
||||
# over-split; real headers anchor a cluster with ~12+ constant bytes.
|
||||
@@ -713,7 +713,7 @@ end
|
||||
for (id, c) in r.clusters
|
||||
sig = signature(c)
|
||||
if is_promotable(c, sig; min_members=20, min_magic=3)
|
||||
# ...and every nomination is PURE — the whole point of the human
|
||||
# ...and every nomination is PURE. The whole point of the human
|
||||
# gate is that we never hand it a garbage merged signature.
|
||||
labels = unique(breakdown(id))
|
||||
@test Base.length(labels) == 1
|
||||
@@ -770,7 +770,7 @@ end
|
||||
@testset "recover_dir!: blocks on a full queue, recovers every file" begin
|
||||
# The regression this guards is data loss, not slowness. `spool/` now
|
||||
# holds every in-flight file at every stage, so a crash under load leaves
|
||||
# far more leftovers than one queue's capacity — and the old non-blocking
|
||||
# far more leftovers than one queue's capacity, and the old non-blocking
|
||||
# recovery warned and abandoned the excess, which is exactly the work
|
||||
# recovery exists to save. With the pool live, recovery must park until
|
||||
# the consumer makes room and come back with all of it.
|
||||
@@ -887,7 +887,7 @@ end
|
||||
@test "match-01" in cat.processed
|
||||
@test "blob-01" in cat.processed
|
||||
|
||||
# Re-sweeping the same pile is idempotent — nothing new is seen.
|
||||
# Re-sweeping the same pile is idempotent; nothing new is seen.
|
||||
s2 = catalog_sweep!(cat, cfg)
|
||||
@test s2.n_seen == 0
|
||||
@test cat.clusters[1].members == 41
|
||||
@@ -911,7 +911,7 @@ end
|
||||
r = run_cluster_sweep(cfg; rng=MersenneTwister(10))
|
||||
@test r.mode == :compact
|
||||
@test isfile(cfg.cluster_catalog_path)
|
||||
# The mission-critical assertion: ZERO promoted clusters from pure noise.
|
||||
# The assertion that matters: ZERO promoted clusters from pure noise.
|
||||
@test r.n_nominated == 0
|
||||
@test isempty(readdir(cfg.nominated_dir))
|
||||
# Every file was accounted for (clustered-as-singleton or background).
|
||||
@@ -923,13 +923,13 @@ end
|
||||
mktempdir() do root
|
||||
n = 32
|
||||
# β=0.1 over-splits a format into pure sub-clusters (DESIGN §11 known
|
||||
# limitation) — each still carries the full magic and nominates
|
||||
# limitation). Each still carries the full magic and nominates
|
||||
# independently, so a modest min_members catches those sub-clusters.
|
||||
cfg = tmp_config(root; cluster_n=n, cluster_alpha=1.0,
|
||||
cluster_pseudocount=0.1, cluster_bg_mass=5.0,
|
||||
promote_min_members=10, promote_min_magic=3)
|
||||
rng = MersenneTwister(21)
|
||||
# 30 files sharing a fixed 6-byte magic then random payload — a format.
|
||||
# 30 files sharing a fixed 6-byte magic then random payload: a format.
|
||||
magic = UInt8[0x89, 0x46, 0x4d, 0x54, 0x21, 0x0a]
|
||||
for i in 1:30
|
||||
drop_binary(cfg.cluster_dir, vcat(magic, rand(rng, UInt8, 40)); name="fmt-$(lpad(i,2,'0'))")
|
||||
@@ -1029,7 +1029,7 @@ end
|
||||
@test stats.failed[] == 1
|
||||
@test stats.bytes[] == 30
|
||||
# Each of the three handlers slept 20ms before its outcome, so
|
||||
# busy time covers the failure too — the work was done either way.
|
||||
# busy time covers the failure too; the work was done either way.
|
||||
@test stats.busy_ns[] > 3 * 15_000_000
|
||||
@test stats.blocked_ns[] == 0 # nothing downstream to block on
|
||||
@test stats.in_flight[] == 0 # the finally in worker_loop
|
||||
@@ -1118,7 +1118,7 @@ end
|
||||
@test snap.intake.files == 9
|
||||
@test snap.uptime_seconds >= 0
|
||||
|
||||
# It has to survive the trip through JSON — /stats is the only
|
||||
# It has to survive the trip through JSON: /stats is the only
|
||||
# consumer, and bin/bench.jl reads these exact field names.
|
||||
round_tripped = JSON3.read(JSON3.write(snap))
|
||||
@test round_tripped.stages[2].busy_seconds ≈ 2.0
|
||||
|
||||
Reference in New Issue
Block a user