Add stage-5 phase B: durable single-owner format catalog
Wraps the assign_file scoring core (cluster.jl) in the live catalog the design's phase B calls for (DESIGN §5B/§9): - src/catalog.jl: Catalog durable state (frozen-id clusters + sufficient stats + processed set + examples); sparse, sidecar-first durable save/load; incremental catalog_sweep! (deterministic CRP-predictive assignment of new binary/ files); offline compact! that seeds on first run and recompacts later; write_nominations! emitting one JSON per promotable cluster with a hex magic template. - bin/cluster_sweep.jl: cron/periodic single-owner runner (--compact forces a recluster; first run auto-compacts to seed). - config.jl: cluster_catalog_path + nominated_dir knobs (FS_CLUSTER_CATALOG, FS_NOMINATED_DIR), wired into config_from_env and ensure_dirs. - Tests: durable round-trip, incremental sweep growth + idempotency, §10.1 nothing-from-noise end-to-end (zero promotions), a recurring format self-nominating, seed-then-live-assign (165 pass). - DESIGN_clustering.md: mark phase B built.
This commit is contained in:
182
test/runtests.jl
182
test/runtests.jl
@@ -15,7 +15,10 @@ using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, length,
|
||||
header_symbols, header_matrix, ClusterStats, add!, remove!,
|
||||
log_predictive, loggamma, gibbs_cluster, assign_file,
|
||||
signature, magic_positions, is_promotable,
|
||||
adjusted_rand_index, v_measure, HEADER_N, ALPHABET, PAST_EOF
|
||||
adjusted_rand_index, v_measure, HEADER_N, ALPHABET, PAST_EOF,
|
||||
Catalog, load_catalog, save_catalog!, catalog_sweep!, compact!,
|
||||
write_nominations!, run_cluster_sweep, binary_files, record_example!,
|
||||
signature_hex, ensure_dirs
|
||||
using Random: MersenneTwister
|
||||
using Languages: LanguageDetector
|
||||
|
||||
@@ -36,6 +39,9 @@ function tmp_config(root; kwargs...)
|
||||
done_dir = joinpath(root, "done"),
|
||||
text_done_dir = joinpath(root, "text_done"),
|
||||
failed_dir = joinpath(root, "failed"),
|
||||
cluster_dir = joinpath(root, "binary"), # stage-5 sweeps the binary sink
|
||||
cluster_catalog_path = joinpath(root, "catalog.json"),
|
||||
nominated_dir = joinpath(root, "nominated"),
|
||||
kwargs...,
|
||||
)
|
||||
FileServer.ensure_dirs(cfg)
|
||||
@@ -525,4 +531,178 @@ end
|
||||
end
|
||||
end
|
||||
|
||||
# Helper: write a "file" of raw bytes into a dir with a UUID-ish unique name,
|
||||
# returning its path. Mirrors what stage-3 deposits into binary/.
|
||||
function drop_binary(dir, bytes; name=string(rand(UInt128)))
|
||||
mkpath(dir)
|
||||
p = joinpath(dir, name)
|
||||
open(p, "w") do io; write(io, Vector{UInt8}(bytes)); end
|
||||
return p
|
||||
end
|
||||
|
||||
@testset "catalog: durable save/load round-trip" begin
|
||||
mktempdir() do root
|
||||
n = 8
|
||||
cat = Catalog(n)
|
||||
c = ClusterStats(n)
|
||||
add!(c, [0xCA+1, 0xFE+1, 0xBA+1, 0xBE+1, 1, 2, 3, 4])
|
||||
add!(c, [0xCA+1, 0xFE+1, 0xBA+1, 0xBE+1, 5, 6, 7, 8])
|
||||
cat.clusters[7] = c
|
||||
cat.next_id = 8
|
||||
record_example!(cat, 7, "alpha.bin")
|
||||
push!(cat.processed, "alpha.bin"); push!(cat.processed, "beta.bin")
|
||||
|
||||
path = joinpath(root, "catalog.json")
|
||||
save_catalog!(path, cat)
|
||||
@test isfile(path)
|
||||
|
||||
back = load_catalog(path; n=n)
|
||||
@test back.n == n
|
||||
@test back.next_id == 8
|
||||
@test back.processed == cat.processed
|
||||
@test haskey(back.clusters, 7)
|
||||
@test back.clusters[7].members == 2
|
||||
@test back.clusters[7].counts == c.counts # sparse round-trips exactly
|
||||
@test back.examples[7] == ["alpha.bin"]
|
||||
end
|
||||
end
|
||||
|
||||
@testset "catalog: load of a missing file is a fresh catalog" begin
|
||||
mktempdir() do root
|
||||
cat = load_catalog(joinpath(root, "nope.json"); n=16)
|
||||
@test cat.n == 16
|
||||
@test isempty(cat.clusters)
|
||||
@test isempty(cat.processed)
|
||||
@test cat.next_id == 1
|
||||
end
|
||||
end
|
||||
|
||||
@testset "catalog: binary_files skips sidecars, tmp, dirs; sorts" begin
|
||||
mktempdir() do root
|
||||
drop_binary(root, "a"; name="002-file")
|
||||
drop_binary(root, "b"; name="001-file")
|
||||
write(joinpath(root, "003-file.meta.json"), "{}") # sidecar
|
||||
write(joinpath(root, "004-file.tmp"), "x") # scratch
|
||||
mkpath(joinpath(root, "subdir")) # not a file
|
||||
fs = binary_files(root)
|
||||
@test basename.(fs) == ["001-file", "002-file"]
|
||||
end
|
||||
end
|
||||
|
||||
@testset "catalog: incremental sweep grows an existing cluster" begin
|
||||
mktempdir() do root
|
||||
n = 8
|
||||
cfg = tmp_config(root; cluster_n=n, cluster_alpha=1.0,
|
||||
cluster_pseudocount=0.1, cluster_bg_mass=5.0)
|
||||
# Seed a strong cluster (magic 0xCA 0xFE 0xBA 0xBE, random tail).
|
||||
cat = Catalog(n)
|
||||
c = ClusterStats(n)
|
||||
rng = MersenneTwister(3)
|
||||
for _ in 1:40
|
||||
add!(c, vcat([0xCA,0xFE,0xBA,0xBE] .+ 1, rand(rng, 1:256, 4)))
|
||||
end
|
||||
cat.clusters[1] = c
|
||||
cat.next_id = 2
|
||||
|
||||
# A brand-new file that matches the magic must JOIN cluster 1.
|
||||
drop_binary(cfg.cluster_dir, vcat(UInt8[0xCA,0xFE,0xBA,0xBE], rand(rng, UInt8, 4)); name="match-01")
|
||||
# A structureless random blob must park in the background.
|
||||
drop_binary(cfg.cluster_dir, rand(rng, UInt8, 64); name="blob-01")
|
||||
|
||||
s = catalog_sweep!(cat, cfg)
|
||||
@test s.n_seen == 2
|
||||
@test s.n_joined == 1
|
||||
@test s.n_bg == 1
|
||||
@test s.n_minted == 0
|
||||
@test cat.clusters[1].members == 41 # grew by the matching file
|
||||
@test "match-01" in cat.processed
|
||||
@test "blob-01" in cat.processed
|
||||
|
||||
# Re-sweeping the same pile is idempotent — nothing new is seen.
|
||||
s2 = catalog_sweep!(cat, cfg)
|
||||
@test s2.n_seen == 0
|
||||
@test cat.clusters[1].members == 41
|
||||
end
|
||||
end
|
||||
|
||||
@testset "catalog: §10.1 nothing from noise (end-to-end, no promotion)" begin
|
||||
mktempdir() do root
|
||||
n = 32
|
||||
cfg = tmp_config(root; cluster_n=n, cluster_alpha=1.0,
|
||||
cluster_pseudocount=0.1, cluster_bg_mass=5.0,
|
||||
promote_min_members=20, promote_min_magic=3)
|
||||
rng = MersenneTwister(10)
|
||||
# The §10.1 pile: 20 small random blobs + 1 lone structured "PDF".
|
||||
for i in 1:20
|
||||
drop_binary(cfg.cluster_dir, rand(rng, UInt8, 40); name="blob-$(lpad(i,2,'0'))")
|
||||
end
|
||||
drop_binary(cfg.cluster_dir, vcat(UInt8[0x25,0x50,0x44,0x46], rand(rng, UInt8, 60)); name="lone-pdf")
|
||||
|
||||
# First run auto-compacts (empty catalog) to seed, then persists + nominates.
|
||||
r = run_cluster_sweep(cfg; rng=MersenneTwister(10))
|
||||
@test r.mode == :compact
|
||||
@test isfile(cfg.cluster_catalog_path)
|
||||
# The mission-critical assertion: ZERO promoted clusters from pure noise.
|
||||
@test r.n_nominated == 0
|
||||
@test isempty(readdir(cfg.nominated_dir))
|
||||
# Every file was accounted for (clustered-as-singleton or background).
|
||||
@test r.n_processed == 21
|
||||
end
|
||||
end
|
||||
|
||||
@testset "catalog: a real recurring format self-nominates" begin
|
||||
mktempdir() do root
|
||||
n = 32
|
||||
# β=0.1 over-splits a format into pure sub-clusters (DESIGN §11 known
|
||||
# limitation) — each still carries the full magic and nominates
|
||||
# independently, so a modest min_members catches those sub-clusters.
|
||||
cfg = tmp_config(root; cluster_n=n, cluster_alpha=1.0,
|
||||
cluster_pseudocount=0.1, cluster_bg_mass=5.0,
|
||||
promote_min_members=10, promote_min_magic=3)
|
||||
rng = MersenneTwister(21)
|
||||
# 30 files sharing a fixed 6-byte magic then random payload — a format.
|
||||
magic = UInt8[0x89, 0x46, 0x4d, 0x54, 0x21, 0x0a]
|
||||
for i in 1:30
|
||||
drop_binary(cfg.cluster_dir, vcat(magic, rand(rng, UInt8, 40)); name="fmt-$(lpad(i,2,'0'))")
|
||||
end
|
||||
r = run_cluster_sweep(cfg; rng=MersenneTwister(21))
|
||||
@test r.n_nominated >= 1
|
||||
files = readdir(cfg.nominated_dir; join=true)
|
||||
@test !isempty(files)
|
||||
payload = JSON3.read(read(first(files), String))
|
||||
@test payload.members >= 10
|
||||
@test payload.magic_length >= 3
|
||||
# The hex template exposes the shared magic bytes for the human gate.
|
||||
@test occursin("89 46 4d 54", payload.signature_hex)
|
||||
end
|
||||
end
|
||||
|
||||
@testset "catalog: seeded catalog then live-assigns a matching arrival" begin
|
||||
mktempdir() do root
|
||||
n = 32
|
||||
cfg = tmp_config(root; cluster_n=n, cluster_alpha=1.0,
|
||||
cluster_pseudocount=0.1, cluster_bg_mass=5.0,
|
||||
promote_min_members=20, promote_min_magic=3)
|
||||
rng = MersenneTwister(31)
|
||||
magic = UInt8[0x7a, 0x7a, 0x01, 0x02, 0x03]
|
||||
for i in 1:25
|
||||
drop_binary(cfg.cluster_dir, vcat(magic, rand(rng, UInt8, 40)); name="seed-$(lpad(i,2,'0'))")
|
||||
end
|
||||
# Seed pass.
|
||||
run_cluster_sweep(cfg; rng=MersenneTwister(31))
|
||||
cat = load_catalog(cfg.cluster_catalog_path; n=n)
|
||||
@test !isempty(cat.clusters)
|
||||
members_before = sum(c.members for c in values(cat.clusters))
|
||||
|
||||
# A new matching file arrives; an incremental sweep must fold it in
|
||||
# (mode :sweep, not compact) without re-clustering the world.
|
||||
drop_binary(cfg.cluster_dir, vcat(magic, rand(rng, UInt8, 40)); name="arrival-01")
|
||||
r2 = run_cluster_sweep(cfg; rng=MersenneTwister(99))
|
||||
@test r2.mode == :sweep
|
||||
cat2 = load_catalog(cfg.cluster_catalog_path; n=n)
|
||||
members_after = sum(c.members for c in values(cat2.clusters))
|
||||
@test members_after == members_before + 1 # the arrival joined a cluster
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user