Initial file-ingestion service
REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded files to disk, enqueues lightweight references onto a bounded thread-safe work queue, and hands off immediately (202 + job IDs; 503 when full). A configurable pool of worker threads pulls jobs off the queue, logs the received filename (placeholder for real processing), and moves files to done/ on success or failed/ on error. - Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap - Startup recovery: re-enqueues leftover files in spool/ - Graceful drain on SIGINT and SIGTERM (via atexit) - Env-var config; filenames sanitized + UUID-prefixed on disk Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
46
src/worker.jl
Normal file
46
src/worker.jl
Normal file
@@ -0,0 +1,46 @@
|
||||
# Worker task: pull jobs off the queue and process them. One of these runs per
|
||||
# configured worker, each as its own `Threads.@spawn`'d task.
|
||||
|
||||
"""
|
||||
handle_job(job, cfg, worker_id)
|
||||
|
||||
Do the work for a single job, then move the file to `done/`.
|
||||
|
||||
For now the "work" is just logging the received filename to prove the flow —
|
||||
this is the seam where real heavy-lifting will go later.
|
||||
"""
|
||||
function handle_job(job::Job, cfg::Config, worker_id::Int)
|
||||
# --- placeholder for real heavy-lifting work -------------------------
|
||||
@info "received file" worker=worker_id id=job.id name=job.original_name size=job.size
|
||||
# --------------------------------------------------------------------
|
||||
dest = move_to(cfg.done_dir, job)
|
||||
@info "completed" worker=worker_id id=job.id dest=dest
|
||||
return nothing
|
||||
end
|
||||
|
||||
"""
|
||||
worker_loop(worker_id, cfg, queue)
|
||||
|
||||
Consume jobs until the queue is closed and drained. A failure on one job is
|
||||
logged and the file is quarantined in `failed/` — it must never kill the
|
||||
worker, or the pool would silently shrink.
|
||||
"""
|
||||
function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue)
|
||||
@info "worker started" worker=worker_id
|
||||
while true
|
||||
job = dequeue!(queue)
|
||||
job === nothing && break # queue closed and drained → exit
|
||||
try
|
||||
handle_job(job, cfg, worker_id)
|
||||
catch e
|
||||
@error "processing failed" worker=worker_id id=job.id name=job.original_name exception=(e, catch_backtrace())
|
||||
try
|
||||
move_to(cfg.failed_dir, job)
|
||||
catch e2
|
||||
@error "could not quarantine failed file" worker=worker_id id=job.id path=job.path exception=(e2, catch_backtrace())
|
||||
end
|
||||
end
|
||||
end
|
||||
@info "worker stopped" worker=worker_id
|
||||
return nothing
|
||||
end
|
||||
Reference in New Issue
Block a user