Add stage-4 language enrichment for text files
Text files sorted by stage 3 now flow onto a new work queue and worker pool that enrich them with natural language (Languages.jl LanguageDetector: name, ISO 639-3 code, confidence) and programming language (github-linguist), writing a .meta.json sidecar to data/text_done/ like the stage-2 known-file pipeline. github-linguist reads the git blob of a path inside a repo, so untracked data/ files are copied to /tmp (outside any repo, name preserved for extension heuristics) before detection. Programming-language lookup is best-effort (startup warning if missing, degraded/null on failure); natural-language failure yields a degraded sidecar, not a quarantine. Factored exiftool's timeout-kill into shared run_with_timeout and the durable sidecar-first commit into commit_enriched!, both reused by stage 4. Recovery re-drives data/text/; graceful drain closes the text queue after its stage-3 producers finish.
This commit is contained in:
131
test/runtests.jl
131
test/runtests.jl
@@ -8,7 +8,11 @@ using JSON3
|
||||
using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, length,
|
||||
sanitize_filename, recover_dir!, normalize_metadata,
|
||||
build_metadata, finalize_known!, run_exiftool,
|
||||
is_binary, handle_unknown_job
|
||||
is_binary, handle_unknown_job,
|
||||
detect_natural_language, run_linguist, detect_programming_language,
|
||||
read_text_sample, build_text_metadata, finalize_text!, handle_text_job,
|
||||
linguist_available
|
||||
using Languages: LanguageDetector
|
||||
|
||||
# A minimal, valid 1×1 PNG. Lets the real-exiftool tests assert stable facts
|
||||
# (FileType == "PNG", 1×1 dimensions) that don't drift across exiftool versions.
|
||||
@@ -25,6 +29,7 @@ function tmp_config(root; kwargs...)
|
||||
binary_dir = joinpath(root, "binary"),
|
||||
text_dir = joinpath(root, "text"),
|
||||
done_dir = joinpath(root, "done"),
|
||||
text_done_dir = joinpath(root, "text_done"),
|
||||
failed_dir = joinpath(root, "failed"),
|
||||
kwargs...,
|
||||
)
|
||||
@@ -195,25 +200,137 @@ end
|
||||
end
|
||||
end
|
||||
|
||||
@testset "handle_unknown_job: routes to binary/ and text/" begin
|
||||
@testset "handle_unknown_job: binary terminal, text routed to stage 4" begin
|
||||
mktempdir() do root
|
||||
cfg = tmp_config(root)
|
||||
text_queue = ChannelQueue(10)
|
||||
|
||||
# A binary file (embedded NUL) lands in binary/.
|
||||
# A binary file (embedded NUL) lands in binary/ and is NOT enqueued.
|
||||
bpath = joinpath(cfg.unknown_dir, "id-b-blob.dat")
|
||||
write(bpath, UInt8[0x00, 0xFF, 0x10])
|
||||
bjob = Job("id-b", "blob.dat", bpath, filesize(bpath), 0.0)
|
||||
handle_unknown_job(bjob, cfg, 1)
|
||||
handle_unknown_job(bjob, cfg, 1, text_queue)
|
||||
@test isfile(joinpath(cfg.binary_dir, "id-b-blob.dat"))
|
||||
@test !isfile(bpath)
|
||||
@test length(text_queue) == 0
|
||||
|
||||
# A text file lands in text/.
|
||||
# A text file lands in text/ AND is routed onto the stage-4 queue,
|
||||
# with its path updated to the new text/ location.
|
||||
tpath = joinpath(cfg.unknown_dir, "id-t-notes.log")
|
||||
write(tpath, "just some log text\n")
|
||||
tjob = Job("id-t", "notes.log", tpath, filesize(tpath), 0.0)
|
||||
handle_unknown_job(tjob, cfg, 1)
|
||||
@test isfile(joinpath(cfg.text_dir, "id-t-notes.log"))
|
||||
handle_unknown_job(tjob, cfg, 1, text_queue)
|
||||
moved = joinpath(cfg.text_dir, "id-t-notes.log")
|
||||
@test isfile(moved)
|
||||
@test !isfile(tpath)
|
||||
@test length(text_queue) == 1
|
||||
routed = dequeue!(text_queue)
|
||||
@test routed.id == "id-t"
|
||||
@test routed.path == moved
|
||||
end
|
||||
end
|
||||
|
||||
@testset "detect_natural_language" begin
|
||||
d = LanguageDetector()
|
||||
name, code, conf = detect_natural_language(d,
|
||||
"The quick brown fox jumps over the lazy dog and then runs away quickly today.")
|
||||
@test name == "English"
|
||||
@test code == "eng"
|
||||
@test conf isa Real && 0.0 <= conf <= 1.0
|
||||
|
||||
# Empty / whitespace-only text yields no result rather than throwing
|
||||
# (the detector itself errors on empty input).
|
||||
@test detect_natural_language(d, "") == (nothing, nothing, nothing)
|
||||
@test detect_natural_language(d, " \n\t ") == (nothing, nothing, nothing)
|
||||
end
|
||||
|
||||
@testset "read_text_sample: bounded, UTF-8 safe" begin
|
||||
mktempdir() do root
|
||||
p = joinpath(root, "notes.txt")
|
||||
write(p, "café — 日本語 — hello\n")
|
||||
@test read_text_sample(p) == "café — 日本語 — hello\n"
|
||||
|
||||
# Reads at most LANG_SAMPLE_BYTES, and doesn't choke on a multi-byte
|
||||
# char straddling that boundary (trailing 'é' half-in the window).
|
||||
big = joinpath(root, "big.txt")
|
||||
write(big, vcat(fill(UInt8('a'), FileServer.LANG_SAMPLE_BYTES - 1),
|
||||
UInt8[0xc3, 0xa9])) # 'é' split by the edge
|
||||
s = read_text_sample(big)
|
||||
@test Base.length(s) == FileServer.LANG_SAMPLE_BYTES - 1 # trailing half-char trimmed
|
||||
@test all(==('a'), s)
|
||||
end
|
||||
end
|
||||
|
||||
@testset "run_linguist: real detection on source vs. prose" begin
|
||||
if !linguist_available()
|
||||
@info "github-linguist not on PATH; skipping run_linguist tests"
|
||||
else
|
||||
mktempdir() do root
|
||||
# A Python source file → linguist names the language.
|
||||
py = joinpath(root, "script.py")
|
||||
write(py, "import sys\ndef main():\n print('hi')\nmain()\n")
|
||||
@test run_linguist(py, 30) == "Python"
|
||||
|
||||
# Plain prose reports as "Text", which collapses to nothing.
|
||||
prose = joinpath(root, "notes.txt")
|
||||
write(prose, "The quarterly report shows steady growth this year.\n")
|
||||
@test run_linguist(prose, 30) === nothing
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@testset "build_text_metadata + finalize_text!: end to end" begin
|
||||
mktempdir() do root
|
||||
cfg = tmp_config(root)
|
||||
d = LanguageDetector()
|
||||
|
||||
src = joinpath(cfg.text_dir, "id-x-script.py")
|
||||
write(src, join(["# a short program in English prose comment",
|
||||
"import sys",
|
||||
"def greet(name):",
|
||||
" print('hello ' + name + ' welcome to the show today')",
|
||||
"greet('world')", ""], "\n"))
|
||||
job = Job("id-x", "script.py", src, filesize(src), 0.0)
|
||||
|
||||
meta = build_text_metadata(d, job, cfg)
|
||||
@test meta.id == "id-x"
|
||||
@test meta.content_type == "text"
|
||||
@test meta.file_size == filesize(src)
|
||||
@test meta.language !== nothing # some natural language detected
|
||||
@test meta.error === nothing
|
||||
# programming_language is best-effort; present only when linguist is.
|
||||
if linguist_available()
|
||||
@test meta.programming_language == "Python"
|
||||
end
|
||||
|
||||
file_dest, sidecar = finalize_text!(cfg, job, meta)
|
||||
# File moved into text_done/, original gone from text/.
|
||||
@test isfile(file_dest)
|
||||
@test dirname(file_dest) == cfg.text_done_dir
|
||||
@test !isfile(src)
|
||||
# Sidecar committed alongside it, valid JSON, no leftover .tmp.
|
||||
@test isfile(sidecar)
|
||||
@test endswith(sidecar, ".meta.json")
|
||||
@test !isfile(string(sidecar, ".tmp"))
|
||||
parsed = JSON3.read(read(sidecar, String))
|
||||
@test parsed.content_type == "text"
|
||||
@test parsed.file_size == filesize(file_dest)
|
||||
end
|
||||
end
|
||||
|
||||
@testset "handle_text_job: enriches and commits to text_done/" begin
|
||||
mktempdir() do root
|
||||
cfg = tmp_config(root)
|
||||
d = LanguageDetector()
|
||||
|
||||
src = joinpath(cfg.text_dir, "id-h-readme.md")
|
||||
write(src, "# Project\n\nThis project does something useful and interesting for everyone.\n")
|
||||
job = Job("id-h", "readme.md", src, filesize(src), 0.0)
|
||||
|
||||
handle_text_job(job, cfg, 1, d)
|
||||
@test isfile(joinpath(cfg.text_done_dir, "id-h-readme.md"))
|
||||
@test isfile(joinpath(cfg.text_done_dir, "id-h-readme.md.meta.json"))
|
||||
@test !isfile(src)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user