file-server/src/queue.jl

# The queue seam.
#
# The rest of the app only ever calls `enqueue!`, `dequeue!`, `ack!`, `nack!`
# and `close!`. Two implementations sit behind those five methods:
#
#   ChannelQueue  in-process, bounded, thread-safe (the Go-channel / Julia-
#                 `Channel` model). Lost on crash: everything still in `spool/`
#                 is re-driven from stage 1 by `recover_dir!`.
#   RabbitQueue   durable RabbitMQ queues (src/rabbit.jl). Survives a crash: a
#                 job stays unacked until its handler commits, so a restart
#                 resumes each file at the stage it had actually reached.
#
# `FS_QUEUE_BACKEND` picks one; `run` constructs accordingly. No HTTP handler or
# worker code knows which it got.
#
# The ack pair is what makes the broker worth having. Without it a delivery is
# settled the moment it is handed to a worker, so a crash mid-handler loses the
# job exactly as the in-process queue does, and the network hop buys nothing.
# `ChannelQueue` implements both as no-ops, which is honest rather than lazy:
# an in-process queue has no delivery to settle, and its recovery story is
# `recover_dir!`.

abstract type JobQueue end

# How long a producer backs off before retrying an enqueue onto a full queue.
# Blocking backpressure: a file that is already on disk is never dropped, so the
# producer parks until the consumer makes room. Used by the stage-1 and stage-3
# routing handoffs and by startup recovery (see `enqueue_blocking!`). Intake is
# deliberately *not* a user: the HTTP path's `enqueue!` stays non-blocking and
# turns a full queue into a 503, so a saturated pipeline slows uploads down
# rather than holding request threads hostage.
const ROUTE_ENQUEUE_RETRY_SECONDS = 0.05

"""
    enqueue!(q, job) -> Bool

Non-blocking. Returns `true` if the job was accepted, `false` if the queue is
full (the HTTP layer turns `false` into a 503) or closed. Never blocks the
calling request thread.
"""
function enqueue! end

"""
    dequeue!(q) -> Union{Job,Nothing}

Blocks until a job is available and returns it. Returns `nothing` only when the
queue has been closed *and* fully drained, which is a worker's signal to exit.
"""
function dequeue! end

"""
    ack!(q, job)

Settle `job` as done: the broker may forget it. Called by `worker_loop` after
the handler returns, and after a quarantine, so the message outlives the process
for exactly as long as the work is unfinished.

No-op for `ChannelQueue`.
"""
function ack! end

"""
    nack!(q, job)

Settle `job` as rejected, without requeueing. Requeueing is deliberately not
offered: a job that failed on its bytes will fail again, and redelivering it is
a poison-message loop. With no dead-letter exchange configured the broker simply
discards it, so this differs from `ack!` only in what it says, not in what
happens — which is the point at the one call site that uses it (see
`worker_loop`).

No-op for `ChannelQueue`.
"""
function nack! end

"""
    close!(q)

Mark the queue closed and wake all waiting workers. Buffered jobs are still
handed out (drain-then-exit); no new jobs are accepted.
"""
function close! end

# --- In-process bounded implementation --------------------------------------

mutable struct ChannelQueue <: JobQueue
    const buffer::Vector{Job}
    const capacity::Int
    const cond::Threads.Condition   # its internal lock guards `buffer` + `closed`
    closed::Bool
end

ChannelQueue(capacity::Integer) =
    ChannelQueue(Job[], Int(capacity), Threads.Condition(), false)

function enqueue!(q::ChannelQueue, job::Job)::Bool
    lock(q.cond)
    try
        (q.closed || length(q.buffer) >= q.capacity) && return false
        push!(q.buffer, job)
        notify(q.cond)   # wake a waiting worker (notify wakes all by default)
        return true
    finally
        unlock(q.cond)
    end
end

function dequeue!(q::ChannelQueue)::Union{Job,Nothing}
    lock(q.cond)
    try
        while isempty(q.buffer)
            q.closed && return nothing   # closed and drained → tell worker to stop
            wait(q.cond)                 # releases lock while parked
        end
        return popfirst!(q.buffer)
    finally
        unlock(q.cond)
    end
end

# Nothing to settle: an in-process job was never a delivery.
ack!(::ChannelQueue, ::Job) = nothing
nack!(::ChannelQueue, ::Job) = nothing

function close!(q::ChannelQueue)
    lock(q.cond)
    try
        q.closed = true
        notify(q.cond)   # wake every parked worker so they can drain/exit
    finally
        unlock(q.cond)
    end
    return nothing
end

"""
    capacity(q) -> Int

How many jobs the queue can hold before `enqueue!` starts refusing. Part of the
introspection seam alongside `length`: `/stats` reports depth against capacity,
because a depth of 900 means nothing without knowing whether the limit is 1000
or 1_000_000.
"""
capacity(q::ChannelQueue) = q.capacity

"Number of jobs currently buffered (for logging/introspection)."
function Base.length(q::ChannelQueue)
    lock(q.cond)
    try
        return length(q.buffer)
    finally
        unlock(q.cond)
    end
end