Initial file-ingestion service
REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded files to disk, enqueues lightweight references onto a bounded thread-safe work queue, and hands off immediately (202 + job IDs; 503 when full). A configurable pool of worker threads pulls jobs off the queue, logs the received filename (placeholder for real processing), and moves files to done/ on success or failed/ on error. - Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap - Startup recovery: re-enqueues leftover files in spool/ - Graceful drain on SIGINT and SIGTERM (via atexit) - Env-var config; filenames sanitized + UUID-prefixed on disk Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
69
src/spool.jl
Normal file
69
src/spool.jl
Normal file
@@ -0,0 +1,69 @@
|
||||
# Disk lifecycle: sanitize names, spool bytes to disk, move processed files,
|
||||
# and recover leftover files on startup.
|
||||
|
||||
const MAX_NAME_LEN = 100
|
||||
|
||||
"""
|
||||
sanitize_filename(name) -> String
|
||||
|
||||
Turn an untrusted, client-supplied filename into something safe to place in a
|
||||
path. Strips directory components, replaces anything outside a conservative
|
||||
charset, removes leading dots (so `..` and dotfiles can't sneak through), caps
|
||||
the length, and falls back to `"unnamed"` if nothing usable remains.
|
||||
"""
|
||||
function sanitize_filename(name::AbstractString)::String
|
||||
base = basename(String(name)) # drop any path components
|
||||
base = replace(base, r"[^A-Za-z0-9._-]" => "_") # ASCII-only safe charset
|
||||
base = lstrip(base, '.') # kill "..", ".hidden", etc.
|
||||
isempty(base) && (base = "unnamed")
|
||||
return first(base, MAX_NAME_LEN)
|
||||
end
|
||||
|
||||
"Write `bytes` to the spool dir under `<uuid>-<sanitized>` and return the Job."
|
||||
function spool_file(cfg::Config, original_name::AbstractString, bytes::Vector{UInt8})::Job
|
||||
id = string(uuid4())
|
||||
safe = sanitize_filename(original_name)
|
||||
path = joinpath(cfg.spool_dir, string(id, "-", safe))
|
||||
open(path, "w") do io
|
||||
write(io, bytes)
|
||||
end
|
||||
return Job(id, String(original_name), path, length(bytes), time())
|
||||
end
|
||||
|
||||
"Move a spooled file into `dir` (done/ or failed/), returning the destination."
|
||||
function move_to(dir::AbstractString, job::Job)::String
|
||||
dest = joinpath(dir, basename(job.path))
|
||||
mv(job.path, dest; force=true)
|
||||
return dest
|
||||
end
|
||||
|
||||
# The UUID string produced by `uuid4()` is always 36 chars, followed by '-',
|
||||
# then the sanitized name. That fixed width lets us split reliably on recovery.
|
||||
const UUID_LEN = 36
|
||||
|
||||
"""
|
||||
recover_spool!(cfg, queue) -> Int
|
||||
|
||||
Re-enqueue any files already sitting in the spool directory (left by a crash,
|
||||
a hard shutdown, or an intake that never got processed). This is the payoff of
|
||||
spooling to disk: a restart resumes work instead of stranding it. Returns the
|
||||
number of files recovered.
|
||||
"""
|
||||
function recover_spool!(cfg::Config, queue::JobQueue)::Int
|
||||
n = 0
|
||||
for path in sort(readdir(cfg.spool_dir; join=true))
|
||||
isfile(path) || continue
|
||||
fname = basename(path)
|
||||
if length(fname) > UUID_LEN + 1
|
||||
id = fname[1:UUID_LEN]
|
||||
name = fname[(UUID_LEN + 2):end] # skip the '-'
|
||||
else
|
||||
id = string(uuid4()) # unexpected name; give it an id
|
||||
name = fname
|
||||
end
|
||||
job = Job(id, name, path, filesize(path), time())
|
||||
enqueue!(queue, job) || @warn "queue full during recovery; leaving file for next start" path
|
||||
n += 1
|
||||
end
|
||||
return n
|
||||
end
|
||||
Reference in New Issue
Block a user