# Stage-3 content triage for unknown files.
#
# A file that stage-1 couldn't recognize is still sorted into one of two coarse
# buckets so downstream tooling can treat them differently: text (human-readable)
# and binary (everything else). Only binary is a directory — it is terminal, so
# the file is committed to `binary/`; a text file has stage 4 still to come, so it
# stays in `spool/` and only its queue reference moves on. We sniff only the first
# `CONTENT_SNIFF_BYTES` (no full read) and ask two questions: does the window
# decode as valid UTF-8, and are any of its control bytes ones that don't belong
# in text? This is the Unicode-aware successor to the classic "NUL byte" test —
# it accepts non-ASCII text (accents, CJK, emoji) instead of misfiling it as
# binary, while still rejecting binary formats, which almost never form valid
# UTF-8 near their start (and a NUL is never a valid UTF-8 scalar, so it still
# reads as binary for free).

const CONTENT_SNIFF_BYTES = 8000

# Control bytes (< 0x20) that appear legitimately in text: BS, TAB, LF, VT, FF,
# CR, and ESC (ANSI-colored logs). Any *other* control byte is a binary signal.
const TEXT_CONTROL_BYTES = (0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x1b)

# Drop a trailing UTF-8 sequence that the sniff window cut in half, so a
# multi-byte character straddling the boundary isn't mistaken for invalid bytes.
# Continuation bytes are 0x80–0xBF; a lead byte encodes its own sequence length
# in its high bits. We walk back over the trailing continuation bytes, and if
# the lead byte we land on expects more bytes than the window actually holds,
# trim the whole incomplete sequence.
function trim_truncated_utf8(chunk::AbstractVector{UInt8})
    n = length(chunk)
    n == 0 && return chunk
    # Find the start of the final byte sequence: skip back over continuations.
    i = n
    while i > 0 && (chunk[i] & 0xc0) == 0x80
        i -= 1
    end
    i == 0 && return chunk                 # all continuations; leave as-is
    lead = chunk[i]
    # How many bytes does this lead byte announce?
    expected = lead < 0x80 ? 1 :           # ASCII
               lead < 0xe0 ? 2 :           # 110xxxxx
               lead < 0xf0 ? 3 :           # 1110xxxx
                             4             # 11110xxx
    have = n - i + 1
    return have < expected ? view(chunk, 1:i-1) : chunk
end

"""
    is_binary(path) -> Bool

Classify a file as binary (`true`) or text (`false`) by sniffing its first
`CONTENT_SNIFF_BYTES` bytes. A file is text when that window (minus any
multi-byte character truncated by the window edge) is valid UTF-8 and contains
no control bytes outside the text-safe set (`TEXT_CONTROL_BYTES`). An empty file
is treated as text.
"""
function is_binary(path::AbstractString)::Bool
    open(path, "r") do io
        chunk = read(io, CONTENT_SNIFF_BYTES)
        isempty(chunk) && return false
        window = trim_truncated_utf8(chunk)
        # Malformed UTF-8 → binary.
        isvalid(String(copy(window))) || return true
        # Valid UTF-8, but a stray non-text control byte still means binary.
        # (NUL is a valid UTF-8 scalar, so it's rejected here, not above.)
        return any(b -> b < 0x20 && !(b in TEXT_CONTROL_BYTES), window)
    end
end