Initial file-ingestion service

REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded
files to disk, enqueues lightweight references onto a bounded thread-safe
work queue, and hands off immediately (202 + job IDs; 503 when full). A
configurable pool of worker threads pulls jobs off the queue, logs the
received filename (placeholder for real processing), and moves files to
done/ on success or failed/ on error.

- Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap
- Startup recovery: re-enqueues leftover files in spool/
- Graceful drain on SIGINT and SIGTERM (via atexit)
- Env-var config; filenames sanitized + UUID-prefixed on disk

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 10:53:39 -04:00
commit a6dbcaef8b
11 changed files with 577 additions and 0 deletions

46
src/worker.jl Normal file
View File

@@ -0,0 +1,46 @@
# Worker task: pull jobs off the queue and process them. One of these runs per
# configured worker, each as its own `Threads.@spawn`'d task.
"""
handle_job(job, cfg, worker_id)
Do the work for a single job, then move the file to `done/`.
For now the "work" is just logging the received filename to prove the flow —
this is the seam where real heavy-lifting will go later.
"""
function handle_job(job::Job, cfg::Config, worker_id::Int)
# --- placeholder for real heavy-lifting work -------------------------
@info "received file" worker=worker_id id=job.id name=job.original_name size=job.size
# --------------------------------------------------------------------
dest = move_to(cfg.done_dir, job)
@info "completed" worker=worker_id id=job.id dest=dest
return nothing
end
"""
worker_loop(worker_id, cfg, queue)
Consume jobs until the queue is closed and drained. A failure on one job is
logged and the file is quarantined in `failed/` — it must never kill the
worker, or the pool would silently shrink.
"""
function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue)
@info "worker started" worker=worker_id
while true
job = dequeue!(queue)
job === nothing && break # queue closed and drained → exit
try
handle_job(job, cfg, worker_id)
catch e
@error "processing failed" worker=worker_id id=job.id name=job.original_name exception=(e, catch_backtrace())
try
move_to(cfg.failed_dir, job)
catch e2
@error "could not quarantine failed file" worker=worker_id id=job.id path=job.path exception=(e2, catch_backtrace())
end
end
end
@info "worker stopped" worker=worker_id
return nothing
end