Add stage-5 unknown-format discovery: header clustering + calibration
Implements phase A of the DESIGN_clustering.md design: a Dirichlet-process mixture of per-position categoricals over the first 32 header bytes (257-symbol alphabet) that clusters the binary/ pile by file format, plus signature extraction and promotion nomination. All base-Julia (a Lanczos loggamma keeps the Dirichlet-multinomial marginal dependency-free). - src/cluster.jl: header_symbols feature extraction, collapsed Gibbs sampler (phase A), sequential CRP-predictive assignment (phase B core), signatures/ promotion, and ARI/V-measure calibration metrics. - bin/cluster_calibrate.jl: grid-tunes hyperparameters against magic-collapsed ground truth and cross-checks a model-free NCD (gzip) baseline. - FS_CLUSTER_*/FS_PROMOTE_* config knobs; wire cluster.jl into the module. - Tests for the three DESIGN §10 assertions plus the model primitives. Calibrated defaults (n=32, alpha=1.0, beta=0.1) recover known formats at ARI 0.77 (0.885 excl. tar); docx+zip and the ELF family merge correctly and the NCD baseline agrees. DESIGN §11 records the results and three assumptions the data corrected (tar/ELF header-zero merge, the cold-start seeding deadlock, and the Bernoulli signature / Occam-penalized restart scoring).
This commit is contained in:
217
bin/cluster_calibrate.jl
Normal file
217
bin/cluster_calibrate.jl
Normal file
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env julia
|
||||
#
|
||||
# Phase-A calibration for stage-5 header clustering (model/DESIGN_clustering.md
|
||||
# §7). Runs labeled known files through the exact clustering pipeline, scores the
|
||||
# recovered partition against magic-collapsed ground truth with ARI / V-measure,
|
||||
# grid-tunes (α, β, bg_mass, n), and cross-checks the winning config against a
|
||||
# model-free NCD (gzip) baseline (§8). The settings printed here are the ones the
|
||||
# machine rediscovers known formats at — copy the winner into config.jl.
|
||||
#
|
||||
# julia --project=. bin/cluster_calibrate.jl [training_set_dir]
|
||||
#
|
||||
# Defaults to ../training_set. Prints a report; writes nothing.
|
||||
|
||||
using Random
|
||||
using Printf
|
||||
|
||||
include(joinpath(@__DIR__, "..", "src", "cluster.jl"))
|
||||
|
||||
# --- ground truth: magic-collapsed classes, NOT extensions (DESIGN §7.2) -----
|
||||
|
||||
"""
|
||||
truth_label(path) -> String
|
||||
|
||||
The magic-collapsed format class of a file, read from its actual bytes (so
|
||||
docx≡zip and the whole ELF family merge, exactly the answer we want the
|
||||
clustering to reproduce). `tar` is detected by the `ustar` magic at offset 257 —
|
||||
outside the model's front window, so tars are the accepted blind spot that
|
||||
scatters to background.
|
||||
"""
|
||||
function truth_label(path::AbstractString)
|
||||
b = zeros(UInt8, 262)
|
||||
open(path) do io
|
||||
chunk = read(io, 262)
|
||||
copyto!(b, 1, chunk, 1, length(chunk))
|
||||
end
|
||||
b[1] == 0x1f && b[2] == 0x8b && return "gzip"
|
||||
b[1] == 0x50 && b[2] == 0x4b && return "pkzip"
|
||||
b[1] == 0x25 && b[2] == 0x50 && b[3] == 0x44 && b[4] == 0x46 && return "pdf"
|
||||
b[1] == 0xff && b[2] == 0xd8 && b[3] == 0xff && return "jpeg"
|
||||
b[1] == 0x7f && b[2] == 0x45 && b[3] == 0x4c && b[4] == 0x46 && return "elf"
|
||||
(b[258] == 0x75 && b[259] == 0x73 && b[260] == 0x74 && b[261] == 0x61 && b[262] == 0x72) && return "tar"
|
||||
return "other"
|
||||
end
|
||||
|
||||
# --- NCD (Normalized Compression Distance) baseline, model-free (DESIGN §8) ---
|
||||
|
||||
"gzip-compressed size of a byte buffer, via the gzip CLI (no CodecZlib dep)."
|
||||
function gz_size(bytes::Vector{UInt8})
|
||||
out = IOBuffer()
|
||||
open(pipeline(`gzip -c`; stdout=out); write=true) do io
|
||||
write(io, bytes)
|
||||
end
|
||||
return length(take!(out))
|
||||
end
|
||||
|
||||
"NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)) — 0 = identical, ~1 = unrelated."
|
||||
function ncd(xb, yb, cx, cy)
|
||||
cxy = gz_size(vcat(xb, yb))
|
||||
return (cxy - min(cx, cy)) / max(cx, cy)
|
||||
end
|
||||
|
||||
"""
|
||||
ncd_1nn_purity(paths, truth; head_bytes) -> Float64
|
||||
|
||||
Fraction of files whose NCD-nearest neighbour shares its true label — a cheap,
|
||||
O(N²) sanity read on how well raw gzip-similarity alone separates formats on the
|
||||
same input. The Bayesian clusters should broadly agree; a big gap is a red flag
|
||||
(DESIGN §10.3). Uses each file's first `head_bytes` so the giant files don't
|
||||
dominate compression time.
|
||||
"""
|
||||
function ncd_1nn_purity(paths::Vector{String}, truth::Vector{String}; head_bytes::Int=4096)
|
||||
bufs = map(paths) do p
|
||||
open(io -> read(io, head_bytes), p)
|
||||
end
|
||||
csz = gz_size.(bufs)
|
||||
N = length(paths)
|
||||
correct = 0
|
||||
for i in 1:N
|
||||
best_j = 0; best_d = Inf
|
||||
for j in 1:N
|
||||
i == j && continue
|
||||
d = ncd(bufs[i], bufs[j], csz[i], csz[j])
|
||||
if d < best_d
|
||||
best_d = d; best_j = j
|
||||
end
|
||||
end
|
||||
best_j != 0 && truth[best_j] == truth[i] && (correct += 1)
|
||||
end
|
||||
return correct / N
|
||||
end
|
||||
|
||||
"1-NN label purity of a *cluster* assignment vs truth (same yardstick as NCD's)."
|
||||
function cluster_1nn_purity(pred::Vector{Int}, truth::Vector{String})
|
||||
# For each file, its 'nearest neighbour' is any other file in the same
|
||||
# cluster; purity = P(a random same-cluster neighbour shares the true label).
|
||||
groups = Dict{Int,Vector{Int}}()
|
||||
for (i, k) in enumerate(pred)
|
||||
push!(get!(groups, k, Int[]), i)
|
||||
end
|
||||
correct = 0; total = 0
|
||||
for (_, idxs) in groups
|
||||
length(idxs) < 2 && continue
|
||||
for i in idxs
|
||||
same = count(j -> j != i && truth[j] == truth[i], idxs)
|
||||
total += 1
|
||||
same > 0 && (correct += 1)
|
||||
end
|
||||
end
|
||||
return total == 0 ? 0.0 : correct / total
|
||||
end
|
||||
|
||||
# --- data ---------------------------------------------------------------------
|
||||
|
||||
function load_corpus(dir::AbstractString)
|
||||
paths = String[]
|
||||
for name in readdir(dir; join=true)
|
||||
isfile(name) && push!(paths, name)
|
||||
end
|
||||
truth = truth_label.(paths)
|
||||
return paths, truth
|
||||
end
|
||||
|
||||
# --- grid search --------------------------------------------------------------
|
||||
|
||||
function evaluate(X, truth; α, β, bg_mass, sweeps, restarts, seed)
|
||||
r = gibbs_cluster(X; α=α, β=β, bg_mass=bg_mass, sweeps=sweeps,
|
||||
restarts=restarts, rng=MersenneTwister(seed))
|
||||
pred = r.assignments
|
||||
ari = adjusted_rand_index(truth, pred)
|
||||
keep = truth .!= "tar"
|
||||
ari_notar = adjusted_rand_index(truth[keep], pred[keep])
|
||||
v, h, comp = v_measure(truth, pred)
|
||||
return (; ari, ari_notar, v, h, comp, k=length(r.clusters),
|
||||
bg=count(==(0), pred), result=r)
|
||||
end
|
||||
|
||||
function main()
|
||||
dir = length(ARGS) >= 1 ? ARGS[1] : joinpath(@__DIR__, "..", "..", "training_set")
|
||||
isdir(dir) || error("training set dir not found: $dir")
|
||||
paths, truth = load_corpus(dir)
|
||||
classes = sort(unique(truth))
|
||||
counts = [(c, count(==(c), truth)) for c in classes]
|
||||
@printf("corpus: %d files from %s\n", length(paths), dir)
|
||||
println("magic-collapsed truth classes: ", join(["$c=$n" for (c, n) in counts], " "))
|
||||
println()
|
||||
|
||||
sweeps = 150
|
||||
restarts = 6
|
||||
seed = 20260703
|
||||
|
||||
# Grid. n is expensive to re-featurize, so loop it outermost. Ranges are
|
||||
# centred where the coarse sweep found the optimum: small β (peaked
|
||||
# per-position priors) is what separates formats whose headers differ in only
|
||||
# a few magic bytes; large β over-merges. bg_mass barely moves the result
|
||||
# here (almost nothing lands in background on this corpus), so it is fixed.
|
||||
αs = [1.0, 2.0]
|
||||
βs = [0.05, 0.08, 0.1, 0.15, 0.2]
|
||||
bgs = [5.0]
|
||||
ns = [32, 64]
|
||||
|
||||
println("grid search (sweeps=$sweeps, restarts=$restarts):")
|
||||
@printf(" %-4s %-5s %-5s %-6s | %-6s %-8s %-6s %-6s %-6s %-4s %-4s\n",
|
||||
"n", "alpha", "beta", "bgmss", "ARI", "ARI-tar", "V", "homog", "compl", "k", "bg")
|
||||
results = Vector{Any}()
|
||||
for n in ns
|
||||
X = header_matrix(paths; n=n)
|
||||
for α in αs, β in βs, bg in bgs
|
||||
e = evaluate(X, truth; α=α, β=β, bg_mass=bg, sweeps=sweeps, restarts=restarts, seed=seed)
|
||||
push!(results, (; n, α, β, bg, e))
|
||||
@printf(" %-4d %-5.1f %-5.2f %-6.1f | %-6.3f %-8.3f %-6.3f %-6.3f %-6.3f %-4d %-4d\n",
|
||||
n, α, β, bg, e.ari, e.ari_notar, e.v, e.h, e.comp, e.k, e.bg)
|
||||
end
|
||||
end
|
||||
|
||||
# Rank by ARI-excluding-tar (tar is the accepted blind spot; scoring it would
|
||||
# penalise the correct answer of scattering tars to background — DESIGN §7.2).
|
||||
sort!(results; by=r -> r.e.ari_notar, rev=true)
|
||||
best = results[1]
|
||||
println()
|
||||
@printf("BEST (by ARI excl. tar): n=%d α=%.1f β=%.2f bg_mass=%.1f\n",
|
||||
best.n, best.α, best.β, best.bg)
|
||||
@printf(" ARI=%.3f ARI(excl tar)=%.3f V=%.3f homogeneity=%.3f completeness=%.3f clusters=%d background=%d\n",
|
||||
best.e.ari, best.e.ari_notar, best.e.v, best.e.h, best.e.comp, best.e.k, best.e.bg)
|
||||
|
||||
# Per-cluster composition of the winning partition, and promotion nominations.
|
||||
pred = best.e.result.assignments
|
||||
println("\nwinning partition — cluster composition (truth breakdown):")
|
||||
for (id, c) in sort(collect(best.e.result.clusters); by=x -> -x[2].members)
|
||||
members = [truth[i] for i in eachindex(pred) if pred[i] == id]
|
||||
comp = sort([(l, count(==(l), members)) for l in unique(members)]; by=x -> -x[2])
|
||||
sig = signature(c)
|
||||
promo = is_promotable(c, sig; min_members=20, min_magic=3) ? " ✓NOMINATE" : ""
|
||||
@printf(" cluster %-4d n=%-3d magic=%-2d %s%s\n",
|
||||
id, c.members, magic_positions(sig),
|
||||
join(["$l:$k" for (l, k) in comp], " "), promo)
|
||||
end
|
||||
nbg = count(==(0), pred)
|
||||
bg_truth = [truth[i] for i in eachindex(pred) if pred[i] == 0]
|
||||
bgc = sort([(l, count(==(l), bg_truth)) for l in unique(bg_truth)]; by=x -> -x[2])
|
||||
@printf(" background n=%-3d %s\n", nbg, join(["$l:$k" for (l, k) in bgc], " "))
|
||||
|
||||
# NCD baseline cross-check on a subsample (O(N²), so keep it small).
|
||||
println("\nNCD (gzip) baseline cross-check:")
|
||||
subn = min(150, length(paths))
|
||||
sub = shuffle(MersenneTwister(seed), collect(1:length(paths)))[1:subn]
|
||||
subpaths = paths[sub]; subtruth = truth[sub]
|
||||
ncd_pur = ncd_1nn_purity(subpaths, subtruth)
|
||||
Xsub = header_matrix(subpaths; n=best.n)
|
||||
rsub = gibbs_cluster(Xsub; α=best.α, β=best.β, bg_mass=best.bg,
|
||||
sweeps=sweeps, restarts=restarts, rng=MersenneTwister(seed))
|
||||
bay_pur = cluster_1nn_purity(rsub.assignments, subtruth)
|
||||
@printf(" subsample=%d NCD 1-NN label purity=%.3f Bayesian same-cluster purity=%.3f\n",
|
||||
subn, ncd_pur, bay_pur)
|
||||
println(" (both high ⇒ header-byte signal agrees with model-free gzip similarity — DESIGN §10.3)")
|
||||
end
|
||||
|
||||
main()
|
||||
Reference in New Issue
Block a user