file-server/src/language.jl

# Stage-4 language enrichment for text files.
#
# A file that stage-3 sorted as text is human-readable, but we don't yet know
# *what* it is. This stage answers two questions and records them in a
# `.meta.json` sidecar, exactly like the stage-2 known-file enrichment:
#
#   * natural language  — via Languages.jl's `LanguageDetector` (a Julia port of
#     the `whatlang` n-gram model): English vs. French vs. Japanese, plus a
#     confidence score. Pure Julia, no subprocess.
#   * programming language — via the `github-linguist` CLI, which recognizes
#     source and markup by extension + content heuristics. There is no
#     comparable native Julia library, so we shell out (mirroring stage-2's
#     exiftool dependency).
#
# Neither detector failing quarantines the file: a text file is wanted whether
# or not we can name its language, so a failure yields a *degraded* sidecar
# (what we know plus an `error` note), just like stage 2.
#
# The github-linguist quirk that shapes this code: run against a path *inside* a
# git repository, linguist reads the file's committed git blob, not the bytes on
# disk — and an untracked file (which every file under `data/` is) has no blob,
# so it crashes. We sidestep this by copying the file to a fresh temp dir outside
# any repo (preserving its name so linguist's extension heuristics still fire)
# and pointing linguist there.

# How much of a text file to feed the natural-language detector. The whatlang
# model saturates quickly, so a bounded prefix keeps memory flat on huge logs
# while still giving the detector plenty of signal.
const LANG_SAMPLE_BYTES = 65_536

"Return true if the `github-linguist` binary is on PATH."
function linguist_available()
    try
        Base.run(pipeline(`github-linguist --version`; stdout=devnull, stderr=devnull))
        return true
    catch
        return false
    end
end

"""
    read_text_sample(path) -> String

Read up to `LANG_SAMPLE_BYTES` of `path` as UTF-8 text, trimming a multi-byte
character the window may have cut in half (reusing stage-3's `trim_truncated_utf8`)
so the tail isn't misread as garbage.
"""
function read_text_sample(path::AbstractString)::String
    open(path, "r") do io
        chunk = read(io, LANG_SAMPLE_BYTES)
        return String(copy(trim_truncated_utf8(chunk)))
    end
end

"""
    detect_natural_language(detector, text) -> (name, code, confidence)

Run the `LanguageDetector` on `text`, returning the language's English name
(e.g. `"English"`), its ISO 639-3 code (e.g. `"eng"`), and the model's
confidence in `[0,1]`. Returns `(nothing, nothing, nothing)` when there is no
usable text (empty/whitespace) or the detector errors — the caller records that
as a degraded result rather than failing the file.
"""
function detect_natural_language(detector, text::AbstractString)
    isempty(strip(text)) && return (nothing, nothing, nothing)
    try
        lang, _script, confidence = detector(text)
        return (Languages.english_name(lang), Languages.isocode(lang), confidence)
    catch
        return (nothing, nothing, nothing)
    end
end

"""
    run_linguist(path, timeout) -> Union{String,Nothing}

Ask `github-linguist --json` for the programming/markup language of the file at
`path`, returning the language name (e.g. `"Python"`, `"Markdown"`) or `nothing`
when linguist can't identify one. Plain prose reports as `"Text"` and
unrecognized content as JSON `null`; both collapse to `nothing` here, since only
a real programming/markup language is worth recording.

`path` MUST be outside any git repository — see the module header for why.
"""
function run_linguist(path::AbstractString, timeout::Integer)
    bytes = run_with_timeout(`github-linguist --json $path`, timeout)
    bytes === nothing && return nothing

    parsed = try
        JSON3.read(String(bytes))
    catch
        return nothing
    end
    # linguist --json emits a single object keyed by the file path; pull the one
    # entry rather than depend on the exact key spelling.
    isempty(parsed) && return nothing
    entry = first(values(parsed))
    lang = get(entry, :language, nothing)
    (lang === nothing || lang == "Text") && return nothing
    return String(lang)
end

"""
    detect_programming_language(job, cfg) -> Union{String,Nothing}

Programming/markup language of a text file, or `nothing`. Copies the file to a
throwaway temp dir *outside* the git repo — under its sanitized original name so
linguist's extension heuristics still apply — runs linguist there, and cleans up.
"""
function detect_programming_language(job::Job, cfg::Config)
    lang = nothing
    mktempdir() do dir                       # tempdir() → /tmp, outside the repo
        safe = sanitize_filename(job.original_name)
        tmp  = joinpath(dir, safe)
        cp(job.path, tmp; force=true)
        lang = run_linguist(tmp, cfg.linguist_timeout)
    end
    return lang
end

"""
    build_text_metadata(detector, job, cfg) -> NamedTuple

Build the stage-4 sidecar payload for a text file: its natural language (name +
ISO code + confidence) and programming/markup language, plus the Job's
authoritative id/name/size. `error` is set only when natural-language detection
produced nothing usable (the file is still enriched and committed); programming
language is best-effort and its absence is normal, not an error.
"""
function build_text_metadata(detector, job::Job, cfg::Config)
    text = read_text_sample(job.path)
    name, code, confidence = detect_natural_language(detector, text)
    programming_language   = detect_programming_language(job, cfg)

    return (
        id                   = job.id,
        original_name        = job.original_name,
        file_size            = job.size,                 # authoritative, from intake
        content_type         = "text",
        language             = name,
        language_code        = code,
        language_confidence  = confidence,
        programming_language = programming_language,
        error                = name === nothing ? "language detection produced no result" : nothing,
    )
end

"""
    finalize_text!(cfg, job, meta) -> (file_dest, sidecar_dest)

Commit an enriched text file (stage 4) to `text_done/` via the shared
sidecar-first `commit_enriched!`, giving text files the same crash-safe
"file implies sidecar" guarantee as stage-2 known files.
"""
finalize_text!(cfg::Config, job::Job, meta) = commit_enriched!(cfg.text_done_dir, job, meta)