Initial file-ingestion service
REST endpoint (Oxygen.jl POST /upload, multipart) that spools uploaded files to disk, enqueues lightweight references onto a bounded thread-safe work queue, and hands off immediately (202 + job IDs; 503 when full). A configurable pool of worker threads pulls jobs off the queue, logs the received filename (placeholder for real processing), and moves files to done/ on success or failed/ on error. - Queue behind an enqueue!/dequeue!/close! seam for a future RabbitMQ swap - Startup recovery: re-enqueues leftover files in spool/ - Graceful drain on SIGINT and SIGTERM (via atexit) - Env-var config; filenames sanitized + UUID-prefixed on disk Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
46
src/config.jl
Normal file
46
src/config.jl
Normal file
@@ -0,0 +1,46 @@
|
||||
# Runtime configuration. Built once at startup, then treated as immutable.
|
||||
# Every knob has a default so the service runs with zero configuration, and
|
||||
# every knob can be overridden by an environment variable (see `config_from_env`).
|
||||
|
||||
Base.@kwdef struct Config
|
||||
host::String = "127.0.0.1"
|
||||
port::Int = 8080
|
||||
worker_count::Int = Threads.nthreads()
|
||||
queue_capacity::Int = 1000
|
||||
spool_dir::String = "data/spool" # files land here on intake (pending)
|
||||
done_dir::String = "data/done" # files move here after successful processing
|
||||
failed_dir::String = "data/failed" # files move here if a worker throws
|
||||
end
|
||||
|
||||
"""
|
||||
config_from_env(; overrides...)
|
||||
|
||||
Build a `Config` from environment variables, falling back to the struct
|
||||
defaults. Any keyword `overrides` win over the environment (useful for tests
|
||||
and for `FileServer.run(; port=...)`).
|
||||
|
||||
Recognised variables:
|
||||
FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
|
||||
FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR
|
||||
"""
|
||||
function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
|
||||
queue_capacity=nothing, spool_dir=nothing,
|
||||
done_dir=nothing, failed_dir=nothing)
|
||||
Config(
|
||||
host = something(host, get(ENV, "FS_HOST", "127.0.0.1")),
|
||||
port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))),
|
||||
worker_count = something(worker_count, parse(Int, get(ENV, "FS_WORKERS", string(Threads.nthreads())))),
|
||||
queue_capacity = something(queue_capacity, parse(Int, get(ENV, "FS_QUEUE_CAPACITY", "1000"))),
|
||||
spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")),
|
||||
done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")),
|
||||
failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")),
|
||||
)
|
||||
end
|
||||
|
||||
"Create the spool/done/failed directories if they don't already exist."
|
||||
function ensure_dirs(cfg::Config)
|
||||
for d in (cfg.spool_dir, cfg.done_dir, cfg.failed_dir)
|
||||
mkpath(d)
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
Reference in New Issue
Block a user