Initial file-ingestion service

REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded
files to disk, enqueues lightweight references onto a bounded thread-safe
work queue, and hands off immediately (202 + job IDs; 503 when full). A
configurable pool of worker threads pulls jobs off the queue, logs the
received filename (placeholder for real processing), and moves files to
done/ on success or failed/ on error.

- Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap
- Startup recovery: re-enqueues leftover files in spool/
- Graceful drain on SIGINT and SIGTERM (via atexit)
- Env-var config; filenames sanitized + UUID-prefixed on disk

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 10:53:39 -04:00
commit a6dbcaef8b
11 changed files with 577 additions and 0 deletions

92
src/queue.jl Normal file
View File

@@ -0,0 +1,92 @@
# The queue seam.
#
# The rest of the app only ever calls `enqueue!`, `dequeue!`, and `close!`.
# Today those are backed by an in-process, bounded, thread-safe buffer
# (the Go-channel / Julia-`Channel` model). To move to RabbitMQ (or any broker)
# later, implement a new `JobQueue` subtype with these three methods and swap
# the construction in `run` — no HTTP handler or worker code needs to change.
abstract type JobQueue end
"""
enqueue!(q, job) -> Bool
Non-blocking. Returns `true` if the job was accepted, `false` if the queue is
full (the HTTP layer turns `false` into a 503) or closed. Never blocks the
calling request thread.
"""
function enqueue! end
"""
dequeue!(q) -> Union{Job,Nothing}
Blocks until a job is available and returns it. Returns `nothing` only when the
queue has been closed *and* fully drained — the signal for a worker to exit.
"""
function dequeue! end
"""
close!(q)
Mark the queue closed and wake all waiting workers. Buffered jobs are still
handed out (drain-then-exit); no new jobs are accepted.
"""
function close! end
# --- In-process bounded implementation --------------------------------------
mutable struct ChannelQueue <: JobQueue
const buffer::Vector{Job}
const capacity::Int
const cond::Threads.Condition # its internal lock guards `buffer` + `closed`
closed::Bool
end
ChannelQueue(capacity::Integer) =
ChannelQueue(Job[], Int(capacity), Threads.Condition(), false)
function enqueue!(q::ChannelQueue, job::Job)::Bool
lock(q.cond)
try
(q.closed || length(q.buffer) >= q.capacity) && return false
push!(q.buffer, job)
notify(q.cond) # wake a waiting worker (notify wakes all by default)
return true
finally
unlock(q.cond)
end
end
function dequeue!(q::ChannelQueue)::Union{Job,Nothing}
lock(q.cond)
try
while isempty(q.buffer)
q.closed && return nothing # closed and drained → tell worker to stop
wait(q.cond) # releases lock while parked
end
return popfirst!(q.buffer)
finally
unlock(q.cond)
end
end
function close!(q::ChannelQueue)
lock(q.cond)
try
q.closed = true
notify(q.cond) # wake every parked worker so they can drain/exit
finally
unlock(q.cond)
end
return nothing
end
"Number of jobs currently buffered (for logging/introspection)."
function Base.length(q::ChannelQueue)
lock(q.cond)
try
return length(q.buffer)
finally
unlock(q.cond)
end
end