Add stage-5 unknown-format discovery: header clustering + calibration
Implements phase A of the DESIGN_clustering.md design: a Dirichlet-process mixture of per-position categoricals over the first 32 header bytes (257-symbol alphabet) that clusters the binary/ pile by file format, plus signature extraction and promotion nomination. All base-Julia (a Lanczos loggamma keeps the Dirichlet-multinomial marginal dependency-free). - src/cluster.jl: header_symbols feature extraction, collapsed Gibbs sampler (phase A), sequential CRP-predictive assignment (phase B core), signatures/ promotion, and ARI/V-measure calibration metrics. - bin/cluster_calibrate.jl: grid-tunes hyperparameters against magic-collapsed ground truth and cross-checks a model-free NCD (gzip) baseline. - FS_CLUSTER_*/FS_PROMOTE_* config knobs; wire cluster.jl into the module. - Tests for the three DESIGN §10 assertions plus the model primitives. Calibrated defaults (n=32, alpha=1.0, beta=0.1) recover known formats at ARI 0.77 (0.885 excl. tar); docx+zip and the ELF family merge correctly and the NCD baseline agrees. DESIGN §11 records the results and three assumptions the data corrected (tar/ELF header-zero merge, the cold-start seeding deadlock, and the Bernoulli signature / Occam-penalized restart scoring).
This commit is contained in:
168
test/runtests.jl
168
test/runtests.jl
@@ -11,7 +11,12 @@ using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, length,
|
||||
is_binary, handle_unknown_job,
|
||||
detect_natural_language, run_linguist, detect_programming_language,
|
||||
read_text_sample, build_text_metadata, finalize_text!, handle_text_job,
|
||||
linguist_available
|
||||
linguist_available,
|
||||
header_symbols, header_matrix, ClusterStats, add!, remove!,
|
||||
log_predictive, loggamma, gibbs_cluster, assign_file,
|
||||
signature, magic_positions, is_promotable,
|
||||
adjusted_rand_index, v_measure, HEADER_N, ALPHABET, PAST_EOF
|
||||
using Random: MersenneTwister
|
||||
using Languages: LanguageDetector
|
||||
|
||||
# A minimal, valid 1×1 PNG. Lets the real-exiftool tests assert stable facts
|
||||
@@ -334,6 +339,167 @@ end
|
||||
end
|
||||
end
|
||||
|
||||
@testset "cluster: header_symbols feature extraction" begin
|
||||
mktempdir() do root
|
||||
# Bytes map to 1-based symbols (b -> b+1); positions past EOF -> PAST_EOF.
|
||||
p = joinpath(root, "f.bin")
|
||||
write(p, UInt8[0x00, 0x7f, 0xff])
|
||||
s = header_symbols(p; n=6)
|
||||
@test s[1:3] == [1, 128, 256] # 0->1, 0x7f->128, 0xff->256
|
||||
@test all(==(PAST_EOF), s[4:6]) # 3 bytes short of n=6 -> past EOF
|
||||
@test PAST_EOF == ALPHABET == 257
|
||||
@test Base.length(header_symbols(p)) == HEADER_N
|
||||
|
||||
# An empty file is all past-EOF (real signal, not an error).
|
||||
e = joinpath(root, "empty"); write(e, UInt8[])
|
||||
@test all(==(PAST_EOF), header_symbols(e; n=8))
|
||||
|
||||
# header_matrix stacks one column per file.
|
||||
q = joinpath(root, "g.bin"); write(q, UInt8[0x41, 0x42])
|
||||
X = header_matrix([p, q]; n=4)
|
||||
@test size(X) == (4, 2)
|
||||
@test X[:, 2] == [0x42, 0x43, PAST_EOF, PAST_EOF] # 'A'->66,'B'->67
|
||||
end
|
||||
end
|
||||
|
||||
@testset "cluster: loggamma matches known values" begin
|
||||
@test loggamma(1.0) ≈ 0.0 atol=1e-10
|
||||
@test loggamma(2.0) ≈ 0.0 atol=1e-10
|
||||
@test loggamma(5.0) ≈ log(24) atol=1e-10 # Γ(5) = 4! = 24
|
||||
@test loggamma(0.5) ≈ 0.5log(π) atol=1e-10 # Γ(1/2) = √π
|
||||
@test loggamma(10.0) ≈ log(362880) atol=1e-8 # Γ(10) = 9!
|
||||
end
|
||||
|
||||
@testset "cluster: sufficient stats and predictive" begin
|
||||
c = ClusterStats(3)
|
||||
x = [10, 20, 30]
|
||||
# Empty cluster's predictive equals the uniform prior (1/ALPHABET)^n.
|
||||
@test log_predictive(c, x, 0.5) ≈ -3 * log(ALPHABET) atol=1e-9
|
||||
# add! then remove! is an exact round-trip back to empty.
|
||||
add!(c, x); remove!(c, x)
|
||||
@test c.members == 0
|
||||
@test all(==(0), c.counts)
|
||||
# A cluster holding a matching point scores it far above uniform.
|
||||
add!(c, x)
|
||||
@test log_predictive(c, x, 0.5) > -3 * log(ALPHABET)
|
||||
end
|
||||
|
||||
@testset "cluster: ARI and V-measure" begin
|
||||
# Identical labelings (up to relabeling) score 1.0.
|
||||
@test adjusted_rand_index([1,1,2,2], [7,7,9,9]) ≈ 1.0
|
||||
@test adjusted_rand_index(["a","a","b"], ["b","b","a"]) ≈ 1.0
|
||||
v, h, comp = v_measure([1,1,2,2], [5,5,6,6])
|
||||
@test v ≈ 1.0 && h ≈ 1.0 && comp ≈ 1.0
|
||||
# A partition that merges two true classes into one is complete but not
|
||||
# homogeneous, and ARI drops below 1.
|
||||
@test adjusted_rand_index([1,1,2,2], [1,1,1,1]) < 1.0
|
||||
_, h2, comp2 = v_measure([1,1,2,2], [1,1,1,1])
|
||||
@test comp2 ≈ 1.0 # everything from each class stays together
|
||||
@test h2 < 1.0 # but the cluster mixes two classes
|
||||
end
|
||||
|
||||
@testset "cluster: signature, magic length, promotability" begin
|
||||
n = 8
|
||||
c = ClusterStats(n)
|
||||
# 30 files sharing bytes 0xDE 0xAD 0xBE 0xEF at positions 1-4, random after.
|
||||
rng = MersenneTwister(1)
|
||||
for _ in 1:30
|
||||
x = vcat([0xDE, 0xAD, 0xBE, 0xEF] .+ 1, rand(rng, 1:256, 4))
|
||||
add!(c, x)
|
||||
end
|
||||
sig = signature(c)
|
||||
@test sig[1:4] == [0xDE, 0xAD, 0xBE, 0xEF] # spiked -> required bytes
|
||||
@test all(isnothing, sig[5:8]) # flat -> wildcards
|
||||
@test magic_positions(sig) == 4
|
||||
@test is_promotable(c, sig; min_members=20, min_magic=3)
|
||||
# Too few members, or too few magic positions, blocks nomination.
|
||||
@test !is_promotable(c, sig; min_members=50, min_magic=3)
|
||||
@test !is_promotable(c, sig; min_members=20, min_magic=5)
|
||||
end
|
||||
|
||||
@testset "cluster: §10.1 discovers nothing from noise" begin
|
||||
# 25 independent random blobs — the shape of data/binary (structureless
|
||||
# junk). Correct output: ZERO promoted clusters (random headers never
|
||||
# form a ≥20-member, ≥3-magic-byte signature). See DESIGN §10.1.
|
||||
rng = MersenneTwister(20260703)
|
||||
X = reduce(hcat, [rand(rng, 1:256, HEADER_N) for _ in 1:25])
|
||||
r = gibbs_cluster(X; α=1.0, β=0.1, bg_mass=5.0, sweeps=60, restarts=3,
|
||||
rng=MersenneTwister(1))
|
||||
promoted = count(c -> is_promotable(c, signature(c); min_members=20, min_magic=3),
|
||||
values(r.clusters))
|
||||
@test promoted == 0
|
||||
|
||||
# And a lone structured file (a singleton, like the giant PDF in the pile)
|
||||
# never promotes on its own: N=1 < min_members.
|
||||
one = ClusterStats(HEADER_N)
|
||||
add!(one, vcat([0x25,0x50,0x44,0x46] .+ 1, fill(1, HEADER_N - 4)))
|
||||
@test !is_promotable(one, signature(one); min_members=20, min_magic=3)
|
||||
end
|
||||
|
||||
@testset "cluster: §10.2 recovers known (synthetic) formats" begin
|
||||
# Four synthetic "formats": a fixed magic prefix + random tail, mirroring
|
||||
# gzip/PDF/JPEG/ELF. Calibrated settings must recover them as clean,
|
||||
# promotable clusters at high ARI — the magic-collapsed recovery of §10.2,
|
||||
# here with a hermetic, deterministic corpus.
|
||||
# ~12-byte constant headers + random tails — the shape of a real file
|
||||
# header (a fixed magic/version region, then variable content). A too-short
|
||||
# magic over a fully-random tail is adversarially hard and lets a format
|
||||
# over-split; real headers anchor a cluster with ~12+ constant bytes.
|
||||
rng = MersenneTwister(7)
|
||||
magics = Dict(
|
||||
"gzip" => UInt8[0x1f,0x8b,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x2d,0x00],
|
||||
"pdf" => UInt8[0x25,0x50,0x44,0x46,0x2d,0x31,0x2e,0x34,0x0a,0x25,0xe2,0xe3],
|
||||
"jpeg" => UInt8[0xff,0xd8,0xff,0xe0,0x00,0x10,0x4a,0x46,0x49,0x46,0x00,0x01],
|
||||
"elf" => UInt8[0x7f,0x45,0x4c,0x46,0x02,0x01,0x01,0x00,0x00,0x00,0x00,0x00],
|
||||
)
|
||||
cols = Vector{Int}[]; truth = String[]
|
||||
for (label, magic) in magics, _ in 1:50
|
||||
tail = rand(rng, 1:256, HEADER_N - Base.length(magic))
|
||||
push!(cols, vcat(Int.(magic) .+ 1, tail))
|
||||
push!(truth, label)
|
||||
end
|
||||
X = reduce(hcat, cols)
|
||||
r = gibbs_cluster(X; α=1.0, β=0.1, bg_mass=5.0, sweeps=120, restarts=6,
|
||||
rng=MersenneTwister(3))
|
||||
@test adjusted_rand_index(truth, r.assignments) > 0.9
|
||||
|
||||
# Truth breakdown of each cluster, keyed by cluster id.
|
||||
breakdown(id) = [truth[i] for i in eachindex(r.assignments) if r.assignments[i] == id]
|
||||
# Nominations cover most formats (a format may over-split below the size
|
||||
# threshold, but the recovery is not allowed to miss more than one)...
|
||||
nominated_labels = Set{String}()
|
||||
for (id, c) in r.clusters
|
||||
sig = signature(c)
|
||||
if is_promotable(c, sig; min_members=20, min_magic=3)
|
||||
# ...and every nomination is PURE — the whole point of the human
|
||||
# gate is that we never hand it a garbage merged signature.
|
||||
labels = unique(breakdown(id))
|
||||
@test Base.length(labels) == 1
|
||||
push!(nominated_labels, only(labels))
|
||||
end
|
||||
end
|
||||
@test Base.length(nominated_labels) >= 3
|
||||
end
|
||||
|
||||
@testset "cluster: §5B sequential assignment (phase B)" begin
|
||||
# Build a catalog with one strong cluster (magic 0xCA 0xFE ...).
|
||||
n = 8
|
||||
clusters = Dict{Int,ClusterStats}()
|
||||
c = ClusterStats(n)
|
||||
rng = MersenneTwister(2)
|
||||
for _ in 1:40
|
||||
add!(c, vcat([0xCA,0xFE,0xBA,0xBE] .+ 1, rand(rng, 1:256, 4)))
|
||||
end
|
||||
clusters[1] = c
|
||||
ids = collect(keys(clusters))
|
||||
# A file that matches the cluster's magic joins it.
|
||||
match = vcat([0xCA,0xFE,0xBA,0xBE] .+ 1, rand(rng, 1:256, 4))
|
||||
@test assign_file(match, clusters, ids; α=1.0, β=0.1, bg_mass=5.0) == 1
|
||||
# A structured-but-novel file (different magic) spawns a new cluster (-1).
|
||||
novel = vcat([0x12,0x34,0x56,0x78] .+ 1, fill(1, 4))
|
||||
@test assign_file(novel, clusters, ids; α=1.0, β=0.1, bg_mass=5.0) in (-1, 0)
|
||||
end
|
||||
|
||||
@testset "recover_dir!: re-enqueues work, skips sidecars" begin
|
||||
mktempdir() do root
|
||||
dir = joinpath(root, "known"); mkpath(dir)
|
||||
|
||||
Reference in New Issue
Block a user