jeff/file-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial text cleanup

Browse Source
This commit is contained in:
Jeffrey Ward
2026-08-19 12:55:19 -04:00
parent 819a4cce7a
commit cc84d70d52
26 changed files with 454 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
bin/cluster_calibrate.jl
View File

@@ -5,7 +5,7 @@
# recovered partition against magic-collapsed ground truth with ARI / V-measure,
# grid-tunes (α, β, bg_mass, n), and cross-checks the winning config against a
# model-free NCD (gzip) baseline (§8). The settings printed here are the ones the
# machine rediscovers known formats at — copy the winner into config.jl.
# machine rediscovers known formats at; copy the winner into config.jl.
#
# julia --project=. bin/cluster_calibrate.jl [training_set_dir]
#
@@ -23,7 +23,7 @@ include(joinpath(@__DIR__, "..", "src", "cluster.jl"))
The magic-collapsed format class of a file, read from its actual bytes (so
docx≡zip and the whole ELF family merge, exactly the answer we want the
clustering to reproduce). `tar` is detected by the `ustar` magic at offset 257 —
clustering to reproduce). `tar` is detected by the `ustar` magic at offset 257,
outside the model's front window, so tars are the accepted blind spot that
scatters to background.
"""
@@ -53,7 +53,7 @@ function gz_size(bytes::Vector{UInt8})
return length(take!(out))
end
"NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)) — 0 = identical, ~1 = unrelated."
"NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)); 0 = identical, ~1 = unrelated."
function ncd(xb, yb, cx, cy)
cxy = gz_size(vcat(xb, yb))
return (cxy - min(cx, cy)) / max(cx, cy)
@@ -62,7 +62,7 @@ end
"""
ncd_1nn_purity(paths, truth; head_bytes) -> Float64
Fraction of files whose NCD-nearest neighbour shares its true label — a cheap,
Fraction of files whose NCD-nearest neighbour shares its true label: a cheap,
O(N²) sanity read on how well raw gzip-similarity alone separates formats on the
same input. The Bayesian clusters should broadly agree; a big gap is a red flag
(DESIGN §10.3). Uses each file's first `head_bytes` so the giant files don't
@@ -173,7 +173,7 @@ function main()
end
# Rank by ARI-excluding-tar (tar is the accepted blind spot; scoring it would
# penalise the correct answer of scattering tars to background — DESIGN §7.2).
# penalise the correct answer of scattering tars to background; DESIGN §7.2).
sort!(results; by=r -> r.e.ari_notar, rev=true)
best = results[1]
println()
@@ -184,7 +184,7 @@ function main()
# Per-cluster composition of the winning partition, and promotion nominations.
pred = best.e.result.assignments
println("\nwinning partition — cluster composition (truth breakdown):")
println("\nwinning partition, cluster composition (truth breakdown):")
for (id, c) in sort(collect(best.e.result.clusters); by=x -> -x[2].members)
members = [truth[i] for i in eachindex(pred) if pred[i] == id]
comp = sort([(l, count(==(l), members)) for l in unique(members)]; by=x -> -x[2])
@@ -211,7 +211,7 @@ function main()
bay_pur = cluster_1nn_purity(rsub.assignments, subtruth)
@printf(" subsample=%d NCD 1-NN label purity=%.3f Bayesian same-cluster purity=%.3f\n",
subn, ncd_pur, bay_pur)
println(" (both high ⇒ header-byte signal agrees with model-free gzip similarity — DESIGN §10.3)")
println(" (both high ⇒ header-byte signal agrees with model-free gzip similarity; DESIGN §10.3)")
end
main()