Initial file-ingestion service

REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded
files to disk, enqueues lightweight references onto a bounded thread-safe
work queue, and hands off immediately (202 + job IDs; 503 when full). A
configurable pool of worker threads pulls jobs off the queue, logs the
received filename (placeholder for real processing), and moves files to
done/ on success or failed/ on error.

- Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap
- Startup recovery: re-enqueues leftover files in spool/
- Graceful drain on SIGINT and SIGTERM (via atexit)
- Env-var config; filenames sanitized + UUID-prefixed on disk

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 10:53:39 -04:00
commit a6dbcaef8b
11 changed files with 577 additions and 0 deletions

92
src/FileServer.jl Normal file
View File

@@ -0,0 +1,92 @@
module FileServer
using Logging
using UUIDs
using HTTP
using JSON3
using Oxygen
include("config.jl")
include("job.jl")
include("queue.jl")
include("spool.jl")
include("worker.jl")
# Globals the HTTP handlers read at request time. Set once in `run`, before the
# server starts accepting connections. Declared after the includes above so the
# `Config`/`ChannelQueue` types exist.
const CONFIG = Ref{Config}()
const QUEUE = Ref{ChannelQueue}()
include("server.jl") # registers routes (references CONFIG/QUEUE at call time)
export run
"""
run(; overrides...)
Start the file server: build config, ensure directories, recover any leftover
spooled files, spawn the worker pool, then serve HTTP until interrupted
(Ctrl-C / SIGINT or SIGTERM). On shutdown it stops accepting uploads, drains the
queue, waits for workers to finish in-flight files, and exits cleanly.
Keyword `overrides` (e.g. `port=9000`) take precedence over environment vars.
"""
function run(; overrides...)
# By default a Julia script exits immediately on SIGINT (Ctrl-C). Disable
# that so the interrupt surfaces as a catchable InterruptException, letting
# us drain the queue gracefully below.
Base.exit_on_sigint(false)
cfg = config_from_env(; overrides...)
ensure_dirs(cfg)
if cfg.worker_count > Threads.nthreads()
@warn "worker_count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" worker_count=cfg.worker_count nthreads=Threads.nthreads()
end
queue = ChannelQueue(cfg.queue_capacity)
CONFIG[] = cfg
QUEUE[] = queue
recovered = recover_spool!(cfg, queue)
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count capacity=cfg.queue_capacity recovered=recovered
workers = [Threads.@spawn worker_loop(i, cfg, queue) for i in 1:cfg.worker_count]
register_routes()
serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false)
# Idempotent graceful drain: stop accepting uploads, let workers finish the
# buffered jobs, then exit. Called from two places:
# * the `finally` below, for SIGINT (Ctrl-C) and normal return, and
# * an `atexit` hook, for SIGTERM (systemd/Docker/k8s `stop`).
# We can't intercept SIGTERM directly — Julia blocks it on worker threads and
# handles it in its own runtime, so a user signal() handler never fires. But
# Julia's SIGTERM path runs `atexit` hooks, which gives us a reliable seam.
drained = Threads.Atomic{Bool}(false)
function drain()
Threads.atomic_xchg!(drained, true) && return # run at most once
@info "draining queue and stopping workers"
terminate() # stop accepting new HTTP requests
close!(queue) # let workers drain buffered jobs, then exit
foreach(wait, workers)
@info "shutdown complete"
end
atexit(drain)
try
while true
sleep(0.5) # interruptible; SIGINT throws in here
end
catch e
e isa InterruptException || rethrow(e)
@info "shutdown requested (SIGINT)"
finally
drain()
end
return nothing
end
end # module