Jeffrey Ward
a6dbcaef8b
REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded files to disk, enqueues lightweight references onto a bounded thread-safe work queue, and hands off immediately (202 + job IDs; 503 when full). A configurable pool of worker threads pulls jobs off the queue, logs the received filename (placeholder for real processing), and moves files to done/ on success or failed/ on error. - Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap - Startup recovery: re-enqueues leftover files in spool/ - Graceful drain on SIGINT and SIGTERM (via atexit) - Env-var config; filenames sanitized + UUID-prefixed on disk Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
47 lines
1.7 KiB
Julia
47 lines
1.7 KiB
Julia
# Worker task: pull jobs off the queue and process them. One of these runs per
|
|
# configured worker, each as its own `Threads.@spawn`'d task.
|
|
|
|
"""
|
|
handle_job(job, cfg, worker_id)
|
|
|
|
Do the work for a single job, then move the file to `done/`.
|
|
|
|
For now the "work" is just logging the received filename to prove the flow —
|
|
this is the seam where real heavy-lifting will go later.
|
|
"""
|
|
function handle_job(job::Job, cfg::Config, worker_id::Int)
|
|
# --- placeholder for real heavy-lifting work -------------------------
|
|
@info "received file" worker=worker_id id=job.id name=job.original_name size=job.size
|
|
# --------------------------------------------------------------------
|
|
dest = move_to(cfg.done_dir, job)
|
|
@info "completed" worker=worker_id id=job.id dest=dest
|
|
return nothing
|
|
end
|
|
|
|
"""
|
|
worker_loop(worker_id, cfg, queue)
|
|
|
|
Consume jobs until the queue is closed and drained. A failure on one job is
|
|
logged and the file is quarantined in `failed/` — it must never kill the
|
|
worker, or the pool would silently shrink.
|
|
"""
|
|
function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue)
|
|
@info "worker started" worker=worker_id
|
|
while true
|
|
job = dequeue!(queue)
|
|
job === nothing && break # queue closed and drained → exit
|
|
try
|
|
handle_job(job, cfg, worker_id)
|
|
catch e
|
|
@error "processing failed" worker=worker_id id=job.id name=job.original_name exception=(e, catch_backtrace())
|
|
try
|
|
move_to(cfg.failed_dir, job)
|
|
catch e2
|
|
@error "could not quarantine failed file" worker=worker_id id=job.id path=job.path exception=(e2, catch_backtrace())
|
|
end
|
|
end
|
|
end
|
|
@info "worker stopped" worker=worker_id
|
|
return nothing
|
|
end
|