Initial file-ingestion service
REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded files to disk, enqueues lightweight references onto a bounded thread-safe work queue, and hands off immediately (202 + job IDs; 503 when full). A configurable pool of worker threads pulls jobs off the queue, logs the received filename (placeholder for real processing), and moves files to done/ on success or failed/ on error. - Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap - Startup recovery: re-enqueues leftover files in spool/ - Graceful drain on SIGINT and SIGTERM (via atexit) - Env-var config; filenames sanitized + UUID-prefixed on disk Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
92
src/FileServer.jl
Normal file
92
src/FileServer.jl
Normal file
@@ -0,0 +1,92 @@
|
||||
module FileServer
|
||||
|
||||
using Logging
|
||||
using UUIDs
|
||||
using HTTP
|
||||
using JSON3
|
||||
using Oxygen
|
||||
|
||||
include("config.jl")
|
||||
include("job.jl")
|
||||
include("queue.jl")
|
||||
include("spool.jl")
|
||||
include("worker.jl")
|
||||
|
||||
# Globals the HTTP handlers read at request time. Set once in `run`, before the
|
||||
# server starts accepting connections. Declared after the includes above so the
|
||||
# `Config`/`ChannelQueue` types exist.
|
||||
const CONFIG = Ref{Config}()
|
||||
const QUEUE = Ref{ChannelQueue}()
|
||||
|
||||
include("server.jl") # registers routes (references CONFIG/QUEUE at call time)
|
||||
|
||||
export run
|
||||
|
||||
"""
|
||||
run(; overrides...)
|
||||
|
||||
Start the file server: build config, ensure directories, recover any leftover
|
||||
spooled files, spawn the worker pool, then serve HTTP until interrupted
|
||||
(Ctrl-C / SIGINT or SIGTERM). On shutdown it stops accepting uploads, drains the
|
||||
queue, waits for workers to finish in-flight files, and exits cleanly.
|
||||
|
||||
Keyword `overrides` (e.g. `port=9000`) take precedence over environment vars.
|
||||
"""
|
||||
function run(; overrides...)
|
||||
# By default a Julia script exits immediately on SIGINT (Ctrl-C). Disable
|
||||
# that so the interrupt surfaces as a catchable InterruptException, letting
|
||||
# us drain the queue gracefully below.
|
||||
Base.exit_on_sigint(false)
|
||||
|
||||
cfg = config_from_env(; overrides...)
|
||||
ensure_dirs(cfg)
|
||||
|
||||
if cfg.worker_count > Threads.nthreads()
|
||||
@warn "worker_count exceeds available threads; workers will share threads (start Julia with -t N for real parallelism)" worker_count=cfg.worker_count nthreads=Threads.nthreads()
|
||||
end
|
||||
|
||||
queue = ChannelQueue(cfg.queue_capacity)
|
||||
CONFIG[] = cfg
|
||||
QUEUE[] = queue
|
||||
|
||||
recovered = recover_spool!(cfg, queue)
|
||||
@info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count capacity=cfg.queue_capacity recovered=recovered
|
||||
|
||||
workers = [Threads.@spawn worker_loop(i, cfg, queue) for i in 1:cfg.worker_count]
|
||||
|
||||
register_routes()
|
||||
serve(; host = cfg.host, port = cfg.port, async = true, show_banner = false)
|
||||
|
||||
# Idempotent graceful drain: stop accepting uploads, let workers finish the
|
||||
# buffered jobs, then exit. Called from two places:
|
||||
# * the `finally` below, for SIGINT (Ctrl-C) and normal return, and
|
||||
# * an `atexit` hook, for SIGTERM (systemd/Docker/k8s `stop`).
|
||||
# We can't intercept SIGTERM directly — Julia blocks it on worker threads and
|
||||
# handles it in its own runtime, so a user signal() handler never fires. But
|
||||
# Julia's SIGTERM path runs `atexit` hooks, which gives us a reliable seam.
|
||||
drained = Threads.Atomic{Bool}(false)
|
||||
function drain()
|
||||
Threads.atomic_xchg!(drained, true) && return # run at most once
|
||||
@info "draining queue and stopping workers"
|
||||
terminate() # stop accepting new HTTP requests
|
||||
close!(queue) # let workers drain buffered jobs, then exit
|
||||
foreach(wait, workers)
|
||||
@info "shutdown complete"
|
||||
end
|
||||
atexit(drain)
|
||||
|
||||
try
|
||||
while true
|
||||
sleep(0.5) # interruptible; SIGINT throws in here
|
||||
end
|
||||
catch e
|
||||
e isa InterruptException || rethrow(e)
|
||||
@info "shutdown requested (SIGINT)"
|
||||
finally
|
||||
drain()
|
||||
end
|
||||
|
||||
return nothing
|
||||
end
|
||||
|
||||
end # module
|
||||
Reference in New Issue
Block a user