Jeffrey Ward
1c7d7d6cad
Known-classified files now flow to a second queue with its own worker pool that extracts metadata via exiftool and writes a normalized JSON sidecar next to the file in done/, leaving the original bytes untouched. - Two-stage pipeline: spool/ → classify → known/ → enrich → done/; unknowns park in unknown/ as a seam for a future pool - src/metadata.jl: exiftool -json -G -n with timeout, normalized schema (file_type, mime_type, author, created_by, dimensions, ...) + raw dump; degraded sidecar on extraction failure rather than quarantine - Sidecar-first commit so a file in done/ always has its sidecar - Parametrized worker_loop with classify/enrich handlers; blocking backpressure on a full known queue (never drop a classified file) - Stage-aware recovery: spool/ and known/ resume at their correct stage - Ordered drain: close stage-1 and wait its workers (the known queue's only producer) before closing the known queue - exiftool required at startup (fail-fast); new FS_KNOWN_*/FS_UNKNOWN_DIR/ FS_EXIFTOOL_TIMEOUT config knobs; combined-pool thread warning
75 lines
2.9 KiB
Julia
75 lines
2.9 KiB
Julia
# Disk lifecycle: sanitize names, spool bytes to disk, move processed files,
|
|
# and recover leftover files on startup.
|
|
|
|
const MAX_NAME_LEN = 100
|
|
|
|
"""
|
|
sanitize_filename(name) -> String
|
|
|
|
Turn an untrusted, client-supplied filename into something safe to place in a
|
|
path. Strips directory components, replaces anything outside a conservative
|
|
charset, removes leading dots (so `..` and dotfiles can't sneak through), caps
|
|
the length, and falls back to `"unnamed"` if nothing usable remains.
|
|
"""
|
|
function sanitize_filename(name::AbstractString)::String
|
|
base = basename(String(name)) # drop any path components
|
|
base = replace(base, r"[^A-Za-z0-9._-]" => "_") # ASCII-only safe charset
|
|
base = lstrip(base, '.') # kill "..", ".hidden", etc.
|
|
isempty(base) && (base = "unnamed")
|
|
return first(base, MAX_NAME_LEN)
|
|
end
|
|
|
|
"Write `bytes` to the spool dir under `<uuid>-<sanitized>` and return the Job."
|
|
function spool_file(cfg::Config, original_name::AbstractString, bytes::Vector{UInt8})::Job
|
|
id = string(uuid4())
|
|
safe = sanitize_filename(original_name)
|
|
path = joinpath(cfg.spool_dir, string(id, "-", safe))
|
|
open(path, "w") do io
|
|
write(io, bytes)
|
|
end
|
|
return Job(id, String(original_name), path, length(bytes), time())
|
|
end
|
|
|
|
"Move a spooled file into `dir` (done/ or failed/), returning the destination."
|
|
function move_to(dir::AbstractString, job::Job)::String
|
|
dest = joinpath(dir, basename(job.path))
|
|
mv(job.path, dest; force=true)
|
|
return dest
|
|
end
|
|
|
|
# The UUID string produced by `uuid4()` is always 36 chars, followed by '-',
|
|
# then the sanitized name. That fixed width lets us split reliably on recovery.
|
|
const UUID_LEN = 36
|
|
|
|
"""
|
|
recover_dir!(dir, queue) -> Int
|
|
|
|
Re-enqueue any files sitting in `dir` (left by a crash, hard shutdown, or an
|
|
intake that never finished) onto `queue`. This is the payoff of spooling to
|
|
disk: a restart resumes work instead of stranding it. Stage-aware recovery uses
|
|
one call per stage — `spool/` → stage-1 queue, `known/` → known queue — so each
|
|
file re-enters at the correct stage rather than being reclassified from scratch.
|
|
Returns the number of files recovered.
|
|
|
|
Skips `.meta.json` sidecars: those are stage-2 output, not work to redo.
|
|
"""
|
|
function recover_dir!(dir::AbstractString, queue::JobQueue)::Int
|
|
n = 0
|
|
for path in sort(readdir(dir; join=true))
|
|
isfile(path) || continue
|
|
endswith(path, ".meta.json") && continue # sidecar, not a work item
|
|
fname = basename(path)
|
|
if length(fname) > UUID_LEN + 1
|
|
id = fname[1:UUID_LEN]
|
|
name = fname[(UUID_LEN + 2):end] # skip the '-'
|
|
else
|
|
id = string(uuid4()) # unexpected name; give it an id
|
|
name = fname
|
|
end
|
|
job = Job(id, name, path, filesize(path), time())
|
|
enqueue!(queue, job) || @warn "queue full during recovery; leaving file for next start" path
|
|
n += 1
|
|
end
|
|
return n
|
|
end
|