From 819a4cce7ac2fb00f6f62231d0d0df70443f5b61 Mon Sep 17 00:00:00 2001
From: Jeffrey Ward <wardjm@gmail.com>
Date: Fri, 7 Aug 2026 15:38:33 -0400
Subject: [PATCH] Containerize the server: multi-stage build, compose, data
 volume
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The runtime image carries only what serving needs — the Julia runtime, an
already-precompiled depot, exiftool, and optionally github-linguist. The
package registry, git clones, and the Ruby/C toolchain that builds rugged
all stay in earlier stages.

Two things shape the build. Dependencies are instantiated and precompiled in
a layer keyed only on Project.toml/Manifest.toml, so a src/ edit rebuilds in
seconds rather than minutes; a stub src/FileServer.jl satisfies Pkg's
root-package check there. And github-linguist, the one heavy optional
dependency, is behind WITH_LINGUIST: it degrades gracefully (stage 4 keeps
natural-language enrichment and warns), so building it out is a supported
~160MB saving.

All stage directories are pointed under /data via the FS_*_DIR variables so
one volume holds the whole in-flight working set, and data/ is excluded from
the build context.

Claude-Session: https://claude.ai/code/session_01KMNwyqLS3ns6uwWHGDGdAQ
---
 .dockerignore      |  16 +++++
 Dockerfile         | 146 +++++++++++++++++++++++++++++++++++++++++++++
 docker-compose.yml |  27 +++++++++
 3 files changed, 189 insertions(+)
 create mode 100644 .dockerignore
 create mode 100644 Dockerfile
 create mode 100644 docker-compose.yml

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..303eadd
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,16 @@
+# The pipeline working set (17G here) belongs in a volume, not the image.
+data/
+.git/
+test/
+# Benchmark / training / calibration harnesses are dev tools, not runtime.
+bin/bench.jl
+bin/bench_model.jl
+bin/bench_stage1.jl
+bin/bench_stage2.jl
+bin/cluster_calibrate.jl
+bin/train.jl
+bin/send_dir.sh
+Dockerfile
+docker-compose.yml
+.dockerignore
+*.md
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..0205f6b
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,146 @@
+# Multi-stage build for the FileServer pipeline.
+#
+# The runtime image carries only what serving needs: the Julia runtime, a depot
+# that is already instantiated *and* precompiled (so startup is load-only, no
+# compilation), exiftool, and optionally github-linguist. Nothing from the build
+# — no compilers, no package registry, no gem toolchain — survives into it.
+#
+#   docker build -t file-server .
+#   docker build --build-arg WITH_LINGUIST=false -t file-server .   # ~200MB smaller
+#
+# github-linguist (stage-4 programming-language detection) is the one heavy
+# optional dependency: it needs a Ruby toolchain to build rugged. It degrades
+# gracefully — without it, text files still get natural-language enrichment and
+# the server logs a warning at startup — so it can be built out.
+
+ARG JULIA_VERSION=1.12.6
+ARG DEBIAN_RELEASE=bookworm
+ARG WITH_LINGUIST=true
+
+# ---------------------------------------------------------------------------
+# Stage 1: instantiate and precompile the Julia depot.
+# ---------------------------------------------------------------------------
+FROM julia:${JULIA_VERSION}-${DEBIAN_RELEASE} AS julia-depot
+
+ENV JULIA_DEPOT_PATH=/opt/julia-depot \
+    JULIA_PKG_PRECOMPILE_AUTO=0
+
+WORKDIR /app
+
+# Manifest-first so a source-only edit doesn't re-resolve or rebuild the whole
+# dependency tree. FileServer is its own project's root package, so Pkg insists
+# on a module file being there — a stub satisfies it while the (slow, rarely
+# invalidated) dependency layer is built; the real source lands below.
+COPY Project.toml Manifest.toml ./
+RUN mkdir -p src \
+ && printf 'module FileServer\nend\n' > src/FileServer.jl \
+ && julia --project=/app -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+# Precompiling FileServer itself means the runtime image loads cached code
+# instead of compiling Lux/Zygote on first request.
+COPY src/ ./src/
+RUN julia --project=/app -e 'using Pkg; Pkg.precompile()' \
+ && julia --project=/app -e 'using FileServer'
+
+# Everything the depot keeps for *resolving* packages is dead weight once the
+# Manifest is pinned: the registry, git clones, scratch and log dirs.
+RUN rm -rf /opt/julia-depot/registries \
+           /opt/julia-depot/clones \
+           /opt/julia-depot/scratchspaces \
+           /opt/julia-depot/logs \
+ && find /opt/julia-depot/packages -type d \
+        \( -name test -o -name docs -o -name .git \) -prune -exec rm -rf {} + \
+ && find /opt/julia-depot/artifacts -type d -name doc -prune -exec rm -rf {} +
+
+# ---------------------------------------------------------------------------
+# Stage 2: build github-linguist into a relocatable GEM_HOME (optional).
+#
+# Built against Debian's system Ruby — the same package the runtime installs —
+# so rugged's native extension is ABI-compatible with the interpreter there.
+# ---------------------------------------------------------------------------
+FROM debian:${DEBIAN_RELEASE}-slim AS linguist
+ARG WITH_LINGUIST
+
+ENV GEM_HOME=/opt/gems
+
+RUN set -eux; \
+    mkdir -p "$GEM_HOME"; \
+    if [ "$WITH_LINGUIST" = "true" ]; then \
+      apt-get update; \
+      apt-get install -y --no-install-recommends \
+          ruby ruby-dev build-essential cmake pkg-config \
+          libicu-dev zlib1g-dev libssl-dev; \
+      gem install --no-document github-linguist; \
+      # Drop sources, docs and test fixtures from the installed gems; only the
+      # built .so and the .rb load path matter at runtime.
+      rm -rf "$GEM_HOME"/cache "$GEM_HOME"/doc; \
+      find "$GEM_HOME"/gems -type d \( -name test -o -name spec -o -name samples \) \
+          -prune -exec rm -rf {} +; \
+      find "$GEM_HOME"/extensions -type d -name tmp -prune -exec rm -rf {} +; \
+      rm -rf /var/lib/apt/lists/*; \
+    fi
+
+# ---------------------------------------------------------------------------
+# Stage 3: runtime.
+# ---------------------------------------------------------------------------
+FROM debian:${DEBIAN_RELEASE}-slim AS runtime
+ARG WITH_LINGUIST
+
+# exiftool is a hard startup prerequisite (stage-2 enrichment fails fast without
+# it). Ruby + libicu are only needed to *run* the linguist gem built above.
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        libimage-exiftool-perl ca-certificates curl; \
+    if [ "$WITH_LINGUIST" = "true" ]; then \
+      apt-get install -y --no-install-recommends ruby libicu72; \
+    fi; \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd --system --uid 10001 --create-home --home-dir /app app
+
+# Depot ownership matters: Julia rewrites a precompile cache it considers stale,
+# and a read-only depot would turn that into a startup crash instead of a
+# recompile. --chown does it in the copy, so it costs no extra layer.
+COPY --from=julia-depot /usr/local/julia /usr/local/julia
+COPY --from=julia-depot --chown=app:app /opt/julia-depot /opt/julia-depot
+COPY --from=linguist /opt/gems /opt/gems
+
+ENV PATH=/usr/local/julia/bin:/opt/gems/bin:$PATH \
+    GEM_HOME=/opt/gems \
+    GEM_PATH=/opt/gems \
+    JULIA_DEPOT_PATH=/opt/julia-depot \
+    JULIA_NUM_THREADS=auto \
+    JULIA_PKG_OFFLINE=true
+
+WORKDIR /app
+COPY Project.toml Manifest.toml ./
+COPY src/ ./src/
+COPY bin/server.jl bin/cluster_sweep.jl ./bin/
+COPY model/classifier.jld2 ./model/
+
+# Every stage directory hangs off /data so one volume covers the whole pipeline.
+# 0.0.0.0 rather than the 127.0.0.1 default: the port is only reachable from
+# outside if it is published.
+ENV FS_HOST=0.0.0.0 \
+    FS_PORT=8080 \
+    FS_SPOOL_DIR=/data/spool \
+    FS_BINARY_DIR=/data/binary \
+    FS_DONE_DIR=/data/done \
+    FS_TEXT_DONE_DIR=/data/text_done \
+    FS_FAILED_DIR=/data/failed \
+    FS_CLUSTER_DIR=/data/binary \
+    FS_CLUSTER_CATALOG=/data/catalog.json \
+    FS_NOMINATED_DIR=/data/nominated \
+    FS_MODEL_PATH=/app/model/classifier.jld2
+
+RUN mkdir -p /data && chown app:app /data
+USER app
+VOLUME ["/data"]
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+    CMD curl -fsS "http://127.0.0.1:${FS_PORT}/health" || exit 1
+
+ENTRYPOINT ["julia", "--project=/app", "-t", "auto"]
+CMD ["bin/server.jl"]
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..d3ddead
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,27 @@
+services:
+  file-server:
+    build:
+      context: .
+      args:
+        # false drops the Ruby toolchain and github-linguist (~160MB smaller).
+        # Stage-4 text files then get natural-language enrichment but no
+        # programming language — the server warns at startup and carries on.
+        WITH_LINGUIST: "true"
+    image: file-server:latest
+    ports:
+      - "8080:8080"
+    volumes:
+      # Every pipeline stage (spool, done, text_done, binary, failed, nominated)
+      # lives under /data, so one volume holds the whole in-flight working set.
+      - fs-data:/data
+    environment:
+      # The four pools each default to nthreads(), which oversubscribes the
+      # threads Julia actually has. Size them for the host instead.
+      FS_WORKERS: "4"
+      FS_KNOWN_WORKERS: "4"
+      FS_UNKNOWN_WORKERS: "2"
+      FS_TEXT_WORKERS: "2"
+    restart: unless-stopped
+
+volumes:
+  fs-data: