Initial file-ingestion service

REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded
files to disk, enqueues lightweight references onto a bounded thread-safe
work queue, and hands off immediately (202 + job IDs; 503 when full). A
configurable pool of worker threads pulls jobs off the queue, logs the
received filename (placeholder for real processing), and moves files to
done/ on success or failed/ on error.

- Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap
- Startup recovery: re-enqueues leftover files in spool/
- Graceful drain on SIGINT and SIGTERM (via atexit)
- Env-var config; filenames sanitized + UUID-prefixed on disk

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 10:53:39 -04:00
commit a6dbcaef8b
11 changed files with 577 additions and 0 deletions

69
src/spool.jl Normal file
View File

@@ -0,0 +1,69 @@
# Disk lifecycle: sanitize names, spool bytes to disk, move processed files,
# and recover leftover files on startup.
const MAX_NAME_LEN = 100
"""
sanitize_filename(name) -> String
Turn an untrusted, client-supplied filename into something safe to place in a
path. Strips directory components, replaces anything outside a conservative
charset, removes leading dots (so `..` and dotfiles can't sneak through), caps
the length, and falls back to `"unnamed"` if nothing usable remains.
"""
function sanitize_filename(name::AbstractString)::String
base = basename(String(name)) # drop any path components
base = replace(base, r"[^A-Za-z0-9._-]" => "_") # ASCII-only safe charset
base = lstrip(base, '.') # kill "..", ".hidden", etc.
isempty(base) && (base = "unnamed")
return first(base, MAX_NAME_LEN)
end
"Write `bytes` to the spool dir under `<uuid>-<sanitized>` and return the Job."
function spool_file(cfg::Config, original_name::AbstractString, bytes::Vector{UInt8})::Job
id = string(uuid4())
safe = sanitize_filename(original_name)
path = joinpath(cfg.spool_dir, string(id, "-", safe))
open(path, "w") do io
write(io, bytes)
end
return Job(id, String(original_name), path, length(bytes), time())
end
"Move a spooled file into `dir` (done/ or failed/), returning the destination."
function move_to(dir::AbstractString, job::Job)::String
dest = joinpath(dir, basename(job.path))
mv(job.path, dest; force=true)
return dest
end
# The UUID string produced by `uuid4()` is always 36 chars, followed by '-',
# then the sanitized name. That fixed width lets us split reliably on recovery.
const UUID_LEN = 36
"""
recover_spool!(cfg, queue) -> Int
Re-enqueue any files already sitting in the spool directory (left by a crash,
a hard shutdown, or an intake that never got processed). This is the payoff of
spooling to disk: a restart resumes work instead of stranding it. Returns the
number of files recovered.
"""
function recover_spool!(cfg::Config, queue::JobQueue)::Int
n = 0
for path in sort(readdir(cfg.spool_dir; join=true))
isfile(path) || continue
fname = basename(path)
if length(fname) > UUID_LEN + 1
id = fname[1:UUID_LEN]
name = fname[(UUID_LEN + 2):end] # skip the '-'
else
id = string(uuid4()) # unexpected name; give it an id
name = fname
end
job = Job(id, name, path, filesize(path), time())
enqueue!(queue, job) || @warn "queue full during recovery; leaving file for next start" path
n += 1
end
return n
end