Renamed to DarkStruct

The runtime image carries only what serving needs — the Julia runtime, an
already-precompiled depot, exiftool, and optionally github-linguist. The
package registry, git clones, and the Ruby/C toolchain that builds rugged
all stay in earlier stages.

Two things shape the build. Dependencies are instantiated and precompiled in
a layer keyed only on Project.toml/Manifest.toml, so a src/ edit rebuilds in
seconds rather than minutes; a stub src/FileServer.jl satisfies Pkg's
root-package check there. And github-linguist, the one heavy optional
dependency, is behind WITH_LINGUIST: it degrades gracefully (stage 4 keeps
natural-language enrichment and warns), so building it out is a supported
~160MB saving.

All stage directories are pointed under /data via the FS_*_DIR variables so
one volume holds the whole in-flight working set, and data/ is excluded from
the build context.

Claude-Session: https://claude.ai/code/session_01KMNwyqLS3ns6uwWHGDGdAQ

.dockerignore

@@ -0,0 +1,16 @@
+# The pipeline working set (17G here) belongs in a volume, not the image.
+data/
+.git/
+test/
+# Benchmark / training / calibration harnesses are dev tools, not runtime.
+bin/bench.jl
+bin/bench_model.jl
+bin/bench_stage1.jl
+bin/bench_stage2.jl
+bin/cluster_calibrate.jl
+bin/train.jl
+bin/send_dir.sh
+Dockerfile
+docker-compose.yml
+.dockerignore
+*.md

Dockerfile

@@ -0,0 +1,146 @@
+# Multi-stage build for the DarkStruct pipeline.
+#
+# The runtime image carries only what serving needs: the Julia runtime, a depot
+# that is already instantiated *and* precompiled (so startup is load-only, no
+# compilation), exiftool, and optionally github-linguist. Nothing from the build
+# survives into it: no compilers, no package registry, no gem toolchain.
+#
+#   docker build -t darkstruct .
+#   docker build --build-arg WITH_LINGUIST=false -t darkstruct .   # ~200MB smaller
+#
+# github-linguist (stage-4 programming-language detection) is the one heavy
+# optional dependency: it needs a Ruby toolchain to build rugged. It degrades
+# gracefully: without it, text files still get natural-language enrichment and
+# the server logs a warning at startup, so it can be built out.
+
+ARG JULIA_VERSION=1.12.6
+ARG DEBIAN_RELEASE=bookworm
+ARG WITH_LINGUIST=true
+
+# ---------------------------------------------------------------------------
+# Stage 1: instantiate and precompile the Julia depot.
+# ---------------------------------------------------------------------------
+FROM julia:${JULIA_VERSION}-${DEBIAN_RELEASE} AS julia-depot
+
+ENV JULIA_DEPOT_PATH=/opt/julia-depot \
+    JULIA_PKG_PRECOMPILE_AUTO=0
+
+WORKDIR /app
+
+# Manifest-first so a source-only edit doesn't re-resolve or rebuild the whole
+# dependency tree. DarkStruct is its own project's root package, so Pkg insists
+# on a module file being there; a stub satisfies it while the (slow, rarely
+# invalidated) dependency layer is built; the real source lands below.
+COPY Project.toml Manifest.toml ./
+RUN mkdir -p src \
+ && printf 'module DarkStruct\nend\n' > src/DarkStruct.jl \
+ && julia --project=/app -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+# Precompiling DarkStruct itself means the runtime image loads cached code
+# instead of compiling Lux/Zygote on first request.
+COPY src/ ./src/
+RUN julia --project=/app -e 'using Pkg; Pkg.precompile()' \
+ && julia --project=/app -e 'using DarkStruct'
+
+# Everything the depot keeps for *resolving* packages is dead weight once the
+# Manifest is pinned: the registry, git clones, scratch and log dirs.
+RUN rm -rf /opt/julia-depot/registries \
+           /opt/julia-depot/clones \
+           /opt/julia-depot/scratchspaces \
+           /opt/julia-depot/logs \
+ && find /opt/julia-depot/packages -type d \
+        \( -name test -o -name docs -o -name .git \) -prune -exec rm -rf {} + \
+ && find /opt/julia-depot/artifacts -type d -name doc -prune -exec rm -rf {} +
+
+# ---------------------------------------------------------------------------
+# Stage 2: build github-linguist into a relocatable GEM_HOME (optional).
+#
+# Built against Debian's system Ruby, the same package the runtime installs,
+# so rugged's native extension is ABI-compatible with the interpreter there.
+# ---------------------------------------------------------------------------
+FROM debian:${DEBIAN_RELEASE}-slim AS linguist
+ARG WITH_LINGUIST
+
+ENV GEM_HOME=/opt/gems
+
+RUN set -eux; \
+    mkdir -p "$GEM_HOME"; \
+    if [ "$WITH_LINGUIST" = "true" ]; then \
+      apt-get update; \
+      apt-get install -y --no-install-recommends \
+          ruby ruby-dev build-essential cmake pkg-config \
+          libicu-dev zlib1g-dev libssl-dev; \
+      gem install --no-document github-linguist; \
+      # Drop sources, docs and test fixtures from the installed gems; only the
+      # built .so and the .rb load path matter at runtime.
+      rm -rf "$GEM_HOME"/cache "$GEM_HOME"/doc; \
+      find "$GEM_HOME"/gems -type d \( -name test -o -name spec -o -name samples \) \
+          -prune -exec rm -rf {} +; \
+      find "$GEM_HOME"/extensions -type d -name tmp -prune -exec rm -rf {} +; \
+      rm -rf /var/lib/apt/lists/*; \
+    fi
+
+# ---------------------------------------------------------------------------
+# Stage 3: runtime.
+# ---------------------------------------------------------------------------
+FROM debian:${DEBIAN_RELEASE}-slim AS runtime
+ARG WITH_LINGUIST
+
+# exiftool is a hard startup prerequisite (stage-2 enrichment fails fast without
+# it). Ruby + libicu are only needed to *run* the linguist gem built above.
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        libimage-exiftool-perl ca-certificates curl; \
+    if [ "$WITH_LINGUIST" = "true" ]; then \
+      apt-get install -y --no-install-recommends ruby libicu72; \
+    fi; \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd --system --uid 10001 --create-home --home-dir /app app
+
+# Depot ownership matters: Julia rewrites a precompile cache it considers stale,
+# and a read-only depot would turn that into a startup crash instead of a
+# recompile. --chown does it in the copy, so it costs no extra layer.
+COPY --from=julia-depot /usr/local/julia /usr/local/julia
+COPY --from=julia-depot --chown=app:app /opt/julia-depot /opt/julia-depot
+COPY --from=linguist /opt/gems /opt/gems
+
+ENV PATH=/usr/local/julia/bin:/opt/gems/bin:$PATH \
+    GEM_HOME=/opt/gems \
+    GEM_PATH=/opt/gems \
+    JULIA_DEPOT_PATH=/opt/julia-depot \
+    JULIA_NUM_THREADS=auto \
+    JULIA_PKG_OFFLINE=true
+
+WORKDIR /app
+COPY Project.toml Manifest.toml ./
+COPY src/ ./src/
+COPY bin/server.jl bin/cluster_sweep.jl ./bin/
+COPY model/classifier.jld2 ./model/
+
+# Every stage directory hangs off /data so one volume covers the whole pipeline.
+# 0.0.0.0 rather than the 127.0.0.1 default: the port is only reachable from
+# outside if it is published.
+ENV FS_HOST=0.0.0.0 \
+    FS_PORT=8080 \
+    FS_SPOOL_DIR=/data/spool \
+    FS_BINARY_DIR=/data/binary \
+    FS_DONE_DIR=/data/done \
+    FS_TEXT_DONE_DIR=/data/text_done \
+    FS_FAILED_DIR=/data/failed \
+    FS_CLUSTER_DIR=/data/binary \
+    FS_CLUSTER_CATALOG=/data/catalog.json \
+    FS_NOMINATED_DIR=/data/nominated \
+    FS_MODEL_PATH=/app/model/classifier.jld2
+
+RUN mkdir -p /data && chown app:app /data
+USER app
+VOLUME ["/data"]
+
+EXPOSE 8080
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+    CMD curl -fsS "http://127.0.0.1:${FS_PORT}/health" || exit 1
+
+ENTRYPOINT ["julia", "--project=/app", "-t", "auto"]
+CMD ["bin/server.jl"]

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.12.6"
 manifest_format = "2.0"
-project_hash = "ed6bd1b772452682c906ce1236b89ccb1b0876fc"
+project_hash = "d7c84379a20829ec867546b4a887bc3c10df8311"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "d9aaef7c63466eee4de23b4d9dad03629df54bea"
@@ -15,6 +15,12 @@ weakdeps = ["ChainRulesCore", "ConstructionBase", "EnzymeCore"]
     ADTypesConstructionBaseExt = "ConstructionBase"
     ADTypesEnzymeCoreExt = "EnzymeCore"
 
+[[deps.AMQPClient]]
+deps = ["Logging", "MbedTLS", "Sockets"]
+git-tree-sha1 = "508457ed7a2afb432590247dc363fffc51f242fc"
+uuid = "79c8b4cd-a41a-55fa-907c-fab5288e1383"
+version = "0.5.1"
+
 [[deps.AbstractFFTs]]
 deps = ["LinearAlgebra"]
 git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef"
@@ -308,7 +314,7 @@ weakdeps = ["HTTP"]
     [deps.FileIO.extensions]
     HTTPExt = "HTTP"
 
-[[deps.FileServer]]
+[[deps.DarkStruct]]
 deps = ["HTTP", "JLD2", "JSON3", "Languages", "Logging", "Lux", "Optimisers", "Oxygen", "Random", "UUIDs", "Zygote"]
 path = "."
 uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"

Project.toml

@@ -1,9 +1,10 @@
-name = "FileServer"
+name = "DarkStruct"
 uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e"
 version = "0.1.0"
 authors = ["wardjm@gmail.com"]
 
 [deps]
+AMQPClient = "79c8b4cd-a41a-55fa-907c-fab5288e1383"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
@@ -17,6 +18,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
+AMQPClient = "0.5.1"
 HTTP = "1.11.0"
 JLD2 = "0.6.4"
 JSON3 = "1.14.3"

README.md

@@ -1,13 +1,13 @@
-# FileServer
+# DarkStruct
 
 A minimal Julia service that receives files over HTTP and hands them off to a
 pool of worker threads for processing. The HTTP endpoint does no real work: it
 spools each uploaded file to disk, pushes a lightweight reference onto a work
-queue, and responds immediately — staying free to accept the next upload.
+queue, and responds immediately, staying free to accept the next upload.
 
-The per-file "processing" runs each file through a small neural-network
+The per-file processing is deliberately thin. Each file runs through a small
-classifier that labels it **known** (a file type resembling the training set) or
+neural-network classifier that labels it **known** (a file type resembling the
-**unknown**, and logs the result. See "File classifier" below.
+training set) or **unknown**, and logs the result. See "File classifier" below.
 
 ## Architecture
 
@@ -22,12 +22,13 @@ enrichment mixes CPU with a subprocess):
                  ▼
         ┌─────────────────┐     stream bytes to disk (never buffered)
         │  HTTP handler    │────────────────────────► data/spool/<uuid>-<name>
-        │  (streaming)     │
+        │  (streaming)     │                          the file's home for its
-        └────────┬─────────┘     enqueue reference (non-blocking)
+        └────────┬─────────┘     enqueue reference    whole time in flight
-                 │                        │
+                 │               (non-blocking)
-                 ▼                        ▼
+                 ▼                        │
-          202 + job IDs        ┌────────────────────┐
+          202 + job IDs                   ▼
-          (503 if full)        │  stage-1 queue     │  classification
+          (503 if full)        ┌────────────────────┐
+                               │  stage-1 queue     │  classification
                                └─────────┬──────────┘
                                          │ dequeue
                      ┌───────────────────┼───────────────────┐
@@ -35,9 +36,10 @@ enrichment mixes CPU with a subprocess):
               classify wkr 1      classify wkr 2   …   classify wkr N
                      │
         ┌────────────┴────────────┐
-    :unknown                    :known
+    :unknown                    :known     the file itself never moves;
-        │  move to data/unknown/,  │  move to data/known/, then
+        │                         │        routing is the enqueue alone
-        ▼  then enqueue (blocking) ▼  enqueue (blocking backpressure)
+        │  enqueue (blocking)     │  enqueue (blocking backpressure)
+        ▼                         ▼
   ┌────────────────────┐        ┌────────────────────┐
   │  unknown queue     │        │  known queue       │  enrichment
   └─────────┬──────────┘        └─────────┬──────────┘
@@ -47,10 +49,10 @@ enrichment mixes CPU with a subprocess):
  unk 1  unk 2 … unk K   known wkr 1  known wkr 2 … known wkr M
    │  binary-vs-text sniff       │  exiftool → normalized sidecar
    ├─► data/binary/<uuid>-<name> success ──┴──► data/done/<uuid>-<name>
-   │   (terminal)                           data/done/<uuid>-<name>.meta.json
+   │   (terminal — moved)                    data/done/<uuid>-<name>.meta.json
-   │                                        (sidecar-first commit)
+   │                                         (sidecar-first commit)
-   │  :text  move to data/text/,  failure  ───────► data/failed/<uuid>-<name>
+   │  :text  enqueue (blocking   failure  ───────► data/failed/<uuid>-<name>
-   ▼  then enqueue (blocking backpressure)
+   ▼          backpressure)
   ┌────────────────────┐
   │  text queue        │  language enrichment
   └─────────┬──────────┘
@@ -63,6 +65,16 @@ enrichment mixes CPU with a subprocess):
                                         (sidecar-first commit)
 ```
 
+A file is written once, into `data/spool/`, and stays there for its entire time
+in the pipeline. Stages hand it on by enqueueing its small `Job` reference, never
+by moving bytes: the queue holding the reference *is* the record of which stage
+the file has reached. The only move is the last one, into a terminal sink
+(`data/done/`, `data/text_done/`, `data/binary/`) or into `data/failed/` if a
+worker throws. Stage 1 used to rename each file into `data/known/` or
+`data/unknown/` first, and stage 3 into `data/text/`. Those three directories are
+gone, and with them 11.6 µs per file, the equal of the classifier itself. Stage 1
+now runs at ~85k files/s on one worker instead of ~35k.
+
 Stages 2 (known-file enrichment) and 3 (content triage) run in parallel: stage 1
 feeds both the known and unknown queues. Stage 3 in turn feeds stage 4 (language
 enrichment) for every file it sorts as text.
@@ -71,22 +83,35 @@ Key properties:
 
 - **Fast intake:** the queue only ever carries small references; file bytes live
   on disk, so memory stays flat regardless of file size. This holds end to end:
-  intake **streams** each upload from the socket to the spool file a chunk at a
+  intake streams each upload from the socket to the spool file a chunk at a
   time (`FS_UPLOAD_CHUNK_BYTES`, default 64 KiB) rather than buffering the body,
   and every worker reads only a bounded prefix. Measured: uploads of 256 MiB,
-  1 GiB and 2 GiB each grow resident memory by ~20 MiB — a flat line in file
+  1 GiB and 2 GiB each grow resident memory by ~20 MiB, a flat line in file
   size. See "Streaming intake" and "Benchmarking" below.
 - **Backpressure:** each queue is bounded (default 1000). When the *intake* queue
   is full, uploads get `503 Service Unavailable`. When the *known* queue is full,
   the stage-1 worker blocks and retries (a classified file is never dropped).
-- **Crash-resilient:** files survive on disk. On startup, recovery is
+- **Crash-resilient:** files survive on disk. On startup everything left in
-  stage-aware: leftovers in `data/spool/` re-enter classification, `data/known/`
+  `data/spool/` is re-enqueued (`recovered` in the log) and **replays from stage
-  re-enter enrichment, `data/unknown/` re-enter content triage, and `data/text/`
+  1**. That is safe rather than merely tolerable: classification and the
-  re-enter language enrichment (`recovered` / `recovered_known` /
+  binary/text sniff are pure functions of the file's bytes, and the terminal
-  `recovered_unknown` / `recovered_text` in the log), so a file resumes at its
+  commits rename with `force=true`, so a replayed file lands where it would have
-  correct stage instead of restarting from scratch.
+  landed and overwrites its own sidecar. Recovery *blocks* on a full queue rather
+  than dropping the excess, and runs with the worker pools already live, so a
+  backlog larger than one queue's capacity takes longer to re-drive but none of
+  it is abandoned.
+
+  The cost of replay is redoing stages a file had already cleared. That is a
+  property of the *queue*, not of the directory layout: the default queues are
+  in-process (`src/queue.jl`), so a crash destroys the only record of how far
+  each file got. Per-stage directories used to stand in for that record, at the
+  price of a rename per file per stage on the hot path: a permanent cost on
+  every file, to buy a cheaper restart. The seam in `src/queue.jl` is where that
+  is actually fixed: run with `FS_QUEUE_BACKEND=rabbitmq` and a job stays unacked
+  until its handler commits, so a restart resumes each file at the stage it had
+  reached instead of re-driving `spool/` from stage 1. See "Queue backends".
 - **Graceful shutdown:** SIGINT (Ctrl-C) and SIGTERM (systemd/Docker/k8s `stop`)
-  both stop accepting uploads, then drain the stages *in order* — close the
+  both stop accepting uploads, then drain the stages *in order*: close the
   stage-1 queue and wait out the classify workers (the only producer of the known
   *and* unknown queues), then close those queues and wait out the enrich and
   content-triage workers (content triage being the only producer of the text
@@ -99,22 +124,22 @@ Key properties:
 
 The upload endpoint never holds a file in memory. Bytes go socket → spool file in
 `FS_UPLOAD_CHUNK_BYTES` chunks, so resident memory per in-flight upload is set by
-the chunk size, not the file size — a 2 GiB upload costs about what a 2 KiB one
+the chunk size, not the file size. A 2 GiB upload costs about what a 2 KiB one
 does. Two pieces make that work, and both are deliberate:
 
-- **`src/multipart.jl` — an incremental multipart parser.** HTTP.jl's
+- **`src/multipart.jl`, an incremental multipart parser.** HTTP.jl's
   `parse_multipart_form` takes the *complete* body as a byte vector, so using it
   means every file in the request is in memory at once (and copied again per
   part). The reader here pulls fixed-size chunks and hands each part's bytes
-  straight to its spool file. Its interface is two calls in a loop —
+  straight to its spool file. Its interface is two calls in a loop
-  `next_part!` then `write_part_body!` (or `skip_part_body!`) — so the handler
+  (`next_part!` then `write_part_body!`, or `skip_part_body!`), so the handler
   keeps ordinary control flow instead of inverting into callbacks. The subtle
   part is that a boundary delimiter can straddle two chunks, so the buffer always
   retains the last `length(delimiter)-1` bytes; the test suite parses the same
   body at chunk sizes from 1 byte upward to put that split at every offset.
 - **`/upload` bypasses Oxygen's router.** Oxygen's root handler wraps
   `HTTP.streamhandler`, which does `request.body = read(stream)` *before*
-  dispatching — even for an Oxygen `@stream` route, so no route can stream an
+  dispatching, even for an Oxygen `@stream` route, so no route can stream an
   upload. `run` therefore passes its own `handler` to `serve`
   (`root_stream_handler`), which intercepts `POST /upload` at the stream level and
   delegates everything else to Oxygen unchanged. The trade-off: `/upload` is
@@ -133,28 +158,28 @@ complete) and the event is logged `upload aborted by client`. One cosmetic cavea
 like the SIGTERM one below: when a request body is cut short, HTTP.jl's own
 `closeread` logs an `EOFError` after the handler returns, because the connection
 promised more bytes via `Content-Length` than arrived. It's harmless noise from
-inside HTTP.jl — the partial file is already cleaned up and the connection closed.
+inside HTTP.jl: the partial file is already cleaned up and the connection closed.
 
 ### Metadata enrichment (stage 2)
 
 Files the classifier labels **known** are handed to a second pool that extracts
-metadata with [`exiftool`](https://exiftool.org/) (`exiftool -json -G -n`) —
+metadata with [`exiftool`](https://exiftool.org/) (`exiftool -json -G -n`),
 chosen because no native Julia library comes close to its multi-format coverage.
 The output is normalized into a small, stable, documented schema and written as a
 JSON **sidecar** next to the file in `data/done/`, e.g.
 `data/done/<uuid>-<name>.meta.json`. The original bytes are never modified.
 
 > **Prerequisite:** `exiftool` must be on `PATH` (e.g. `apt install
-> libimage-exiftool-perl`). The server **fails fast at startup** if it's missing.
+> libimage-exiftool-perl`). The server fails fast at startup if it's missing.
 
-Sidecar top-level fields (all nullable — present only when available), plus the
+Sidecar top-level fields (all nullable, present only when available), plus the
 complete raw `exiftool` object under `raw`:
 
 | Field | Meaning |
 |---|---|
 | `id`, `original_name` | job id and client-supplied name |
 | `file_type`, `mime_type` | e.g. `PDF` / `application/pdf` |
-| `file_size` | bytes (authoritative, from intake — not exiftool) |
+| `file_size` | bytes (authoritative, from intake, not exiftool) |
 | `created_date`, `modified_date` | content timestamps |
 | `author` | person (`Author`/`Artist`/`By-line`) |
 | `created_by` | authoring app/tool (`Producer`/`CreatorTool`/`Creator`/`Software`/…) |
@@ -167,12 +192,12 @@ complete raw `exiftool` object under `raw`:
 Each normalized field is a coalesce over a priority list of exiftool tags
 (`src/metadata.jl`); extend a field by appending tag names. If extraction fails
 or `exiftool` times out (`FS_EXIFTOOL_TIMEOUT`, default 30s), the file still
-completes to `data/done/` with a **degraded sidecar** — `file_size`/`file_type`
+completes to `data/done/` with a **degraded sidecar** (`file_size`/`file_type`
-plus an `error` note — rather than being quarantined, because it's still a wanted
+plus an `error` note) rather than being quarantined, because it's still a wanted
 known file. Only genuine I/O errors (can't write the sidecar or move the file)
 send it to `data/failed/`.
 
-The sidecar is committed **before** the file is moved into `data/done/`, so a
+The sidecar is committed *before* the file is moved into `data/done/`, so a
 file's presence there always implies its sidecar is already present; a crash in
 between leaves only a harmless orphan sidecar, and recovery re-enriches
 idempotently.
@@ -182,21 +207,23 @@ idempotently.
 Files the classifier labels **unknown** are handed to a third pool that sorts
 them into two coarse buckets so downstream tooling can treat them differently:
 
-- **`data/binary/`** — the file looks like binary data.
+- **binary:** the file looks like binary data. This is the end of the live path,
-- **`data/text/`** — the file looks like text.
+  so the file is committed to `data/binary/`, which doubles as the corpus for the
+  offline stage-5 discovery sweep below.
+- **text:** the file looks like text. Stage 4 is still to come, so nothing
+  moves; the file stays in `data/spool/` and its reference goes onto the
+  stage-4 queue, ending up in `data/text_done/` once enriched.
 
 The test is a **UTF-8 sniff**: read the first 8000 bytes and call the file text
 when that window is valid UTF-8 and holds no control bytes outside the text-safe
 set (tab, newline, CR, and friends, plus ESC for ANSI-colored logs); otherwise
-binary. It's cheap (no full read) and Unicode-aware — unlike the older NUL-byte
+binary. It's cheap (no full read) and Unicode-aware. Where the older NUL-byte and
-or printable-ASCII heuristics, it keeps non-ASCII text (accents, CJK, emoji) in
+printable-ASCII heuristics misfiled non-ASCII text, this keeps accents, CJK and
-`text/` instead of misfiling it, while binary formats — which rarely form valid
+emoji in the text bucket, while binary formats, which rarely form valid UTF-8
-UTF-8 near their start — still land in `binary/`. A NUL byte is valid UTF-8 but
+near their start, still read as binary. A NUL byte is valid UTF-8 but not a text
-not a text control byte, so it still reads as binary. A multi-byte character
+control byte, so it too reads as binary. A multi-byte character split by the
-split by the 8000-byte boundary is trimmed before the check so it isn't mistaken
+8000-byte boundary is trimmed before the check so it isn't mistaken for malformed
-for malformed bytes. An empty file is treated as text. `binary/` is terminal on
+bytes. An empty file is treated as text (`src/content.jl`).
-the live path (but is the input the offline **stage-5 discovery** sweeps — see
-below); `text/` is handed to stage 4 (`src/content.jl`).
 
 ### Language enrichment (stage 4)
 
@@ -204,12 +231,12 @@ Files that stage 3 sorts as **text** are handed to a fourth pool that identifies
 their language and writes a `.meta.json` sidecar, mirroring the stage-2
 known-file enrichment. Two detectors run per file:
 
-- **natural language** — [`Languages.jl`](https://github.com/JuliaText/Languages.jl)'s
+- **natural language:** [`Languages.jl`](https://github.com/JuliaText/Languages.jl)'s
   `LanguageDetector` (a Julia port of the `whatlang` n-gram model) reads a bounded
   prefix (up to `LANG_SAMPLE_BYTES`, 64 KiB) and reports the language's English
   name, ISO 639-3 code, and a confidence in `[0,1]`. Pure Julia, no subprocess.
   The detector is built once at startup and shared read-only across the pool.
-- **programming language** — the [`github-linguist`](https://github.com/github-linguist/linguist)
+- **programming language:** the [`github-linguist`](https://github.com/github-linguist/linguist)
   CLI recognizes source and markup by extension + content heuristics (e.g.
   `Python`, `Markdown`). Plain prose reports as `Text` and unrecognized content as
   `null`; both collapse to *no programming language*.
@@ -228,42 +255,42 @@ The sidecar schema:
 | `error` | set if natural-language detection produced nothing |
 
 > **`github-linguist` and the git-repo quirk:** run against a path *inside* a git
-> repository, linguist reads the file's committed git blob, not the on-disk bytes
+> repository, linguist reads the file's committed git blob, not the on-disk
-> — and an untracked file (which everything under `data/` is) has no blob, so it
+> bytes, and an untracked file (which everything under `data/` is) has no blob,
-> crashes. Stage 4 sidesteps this by copying each file to a fresh temp dir under
+> so it crashes. Stage 4 sidesteps this by copying each file to a fresh temp dir under
 > `/tmp` (outside any repo, preserving the name so extension heuristics still
 > fire) and pointing linguist there.
 >
-> Programming-language detection is **best-effort**: if `github-linguist` is
+> Programming-language detection is best-effort: if `github-linguist` is
 > missing (a startup warning, not a fatal error, unlike `exiftool`), fails, or
 > times out (`FS_LINGUIST_TIMEOUT`, default 30s), `programming_language` is simply
 > `null` and the file still completes. Natural-language detection failing produces
 > a **degraded sidecar** (with an `error` note) rather than a quarantine, because
 > the file is still wanted text.
 
-Like stage 2, the sidecar is committed **before** the file is moved into
+Like stage 2, the sidecar is committed *before* the file is moved into
 `data/text_done/`, so the file's presence there always implies its sidecar is
 present; recovery re-enriches idempotently (`src/language.jl`).
 
 ### Unknown-format discovery (stage 5, offline)
 
 The `binary/` sink from stage 3 is the pile of genuinely *unrecognized* files.
-Stage 5 mines it for **recurring new file formats** by clustering files on their
+Stage 5 mines it for recurring new file formats by clustering files on their
-header bytes — a growing catalog of discovered formats, each with a magic-byte
+header bytes into a growing catalog of discovered formats, each with a magic-byte
 signature that can eventually be promoted into the classifier's fast path. Unlike
-stages 1–4 it is **not on the request hot path**: it is a single-owner *batch*
+stages 1–4 it is not on the request hot path: it is a single-owner *batch*
 process (the catalog is mutable shared state, the opposite of the stateless
 classifier), and because promotion is human-gated nothing here is
-latency-sensitive. The full rationale — and the assumptions we deliberately
+latency-sensitive. The full rationale, and the assumptions we deliberately
-rejected — live in [`model/DESIGN_clustering.md`](model/DESIGN_clustering.md).
+rejected, live in [`model/DESIGN_clustering.md`](model/DESIGN_clustering.md).
 
 The model (`src/cluster.jl`, base-Julia, no extra deps) is a Dirichlet-process
 mixture of **per-position categoricals** over the first 32 header bytes, on a
 257-symbol alphabet (byte `0–255` plus a `past-EOF` symbol so short fixed-length
-formats are modeled honestly). Bytes are treated as **categorical, not numeric**
+formats are modeled honestly). Bytes are treated as categorical rather than
-— `0x89` and `0x88` are not "close" — so this deliberately does *not* reuse the
+numeric (`0x89` and `0x88` are not "close"), so this deliberately does *not*
-classifier's `[0,1]` byte scaling. A fixed uniform **background** component
+reuse the classifier's `[0,1]` byte scaling. A fixed uniform *background*
-absorbs structureless (compressed/encrypted) blobs so they don't mint spurious
+component absorbs structureless (compressed/encrypted) blobs so they don't mint spurious
 clusters. A cluster's spiked positions become a libmagic-style signature;
 clusters with enough members and enough fixed positions self-**nominate** for
 promotion (a human does the one irreversible step, redefining "known").
@@ -273,8 +300,8 @@ is the science; phase B (`src/catalog.jl`) is the live catalog: a durable
 single-owner state that sweeps `binary/`, folds each new file into a cluster with
 the deterministic CRP-predictive rule, and writes promotion nominations.
 
-The catalog process is run periodically (cron), single-threaded — it is the
+The catalog process is run periodically (cron), single-threaded. It is the only
-**only** writer of the catalog, so it needs no locking:
+writer of the catalog, so it needs no locking:
 
 ```bash
 julia --project=. bin/cluster_sweep.jl            # incremental live sweep of new binary/ files
@@ -284,17 +311,17 @@ julia --project=. bin/cluster_sweep.jl --compact  # offline Gibbs re-cluster (se
 The catalog is a single durable file (`FS_CLUSTER_CATALOG`, default
 `data/catalog.json`) committed with the same sidecar-first
 temp→fsync→rename→fsync-dir discipline as the stage-2 sidecars, so a crash can
-neither corrupt it nor lose a write. On the **first** run (empty catalog) the
+neither corrupt it nor lose a write. On the first run (empty catalog) the sweep
-sweep auto-promotes to a `--compact` pass to seed clusters; later runs assign
+auto-promotes to a `--compact` pass to seed clusters; later runs assign
 incrementally, touching only files they have not seen. A cluster that clears the
-member/magic thresholds writes a nomination — a hex magic template, member count,
+member/magic thresholds writes a nomination (a hex magic template, member count,
-and example filenames — into `FS_NOMINATED_DIR` (default `data/nominated/`) for a
+and example filenames) into `FS_NOMINATED_DIR` (default `data/nominated/`) for a
 human to glance at and promote. Under the calibrated `bg_mass > α`, the live
 sweep never mints single-file clusters; genuinely new formats surface from the
 periodic `--compact` re-clustering of the background residue, not the live path.
 
-Calibration is its own offline script (like training — never in the request
+Calibration is its own offline script, like training, and never in the request
-path), scored against magic-collapsed ground truth (so `docx`≡`zip` and the whole
+path. It is scored against magic-collapsed ground truth (so `docx`≡`zip` and the whole
 ELF family count as one format each, which is the *correct* answer, not an error):
 
 ```bash
@@ -307,16 +334,97 @@ training corpus the calibrated defaults (`n=32`, `α=1.0`, `β=0.1`) recover the
 known formats at **ARI 0.77** (0.885 excluding tar), with `gzip`, `pkzip`
 (`docx`+`zip` merged), and `jpeg` forming clean, promotable clusters; the NCD
 baseline agrees. See `DESIGN_clustering.md` §11 for the full results, including the
-one known limitation (ELF and these tarballs share a long run of header zero-
+one known limitation: ELF and these tarballs share a long run of header
-padding and merge — the v2 fix is inverse-entropy position weighting).
+zero-padding and merge. The v2 fix is inverse-entropy position weighting.
 
-## The queue seam (→ RabbitMQ later)
+## Queue backends
 
-The HTTP handler and workers only ever call `enqueue!`, `dequeue!`, and
+The HTTP handler and the workers only ever call `enqueue!`, `dequeue!`, `ack!`,
-`close!` on a `JobQueue` (see `src/queue.jl`). Today that's an in-process
+`nack!` and `close!` on a `JobQueue` (`src/queue.jl`). Two implementations sit
-`ChannelQueue`. To move to RabbitMQ (or any broker), implement a new `JobQueue`
+behind those five methods, chosen at startup by `FS_QUEUE_BACKEND`:
-subtype with those three methods and swap the construction in `run` — no handler
+
-or worker code changes.
+| | `channel` (default) | `rabbitmq` |
+|---|---|---|
+| Where jobs live | in-process, bounded buffer | durable broker queues, persistent messages |
+| Crash recovery | everything in `spool/` replays from stage 1 | each file resumes at the stage it had reached |
+| External dependency | none | a RabbitMQ broker |
+| Capacity | hard limit, enforced per enqueue | advisory, checked against a polled depth |
+| Delivery | exactly once (nothing to redeliver) | at least once |
+
+`ack!` is the whole difference. A job is settled only after its handler commits,
+so a crash mid-enrichment leaves that job on the enrich queue and the restart
+picks it up there — not at stage 1, and not lost. `ChannelQueue` implements
+`ack!`/`nack!` as no-ops, which is honest rather than lazy: an in-process queue
+has no delivery to settle, and its recovery story is `recover_dir!`.
+
+### Running it
+
+```bash
+# broker + server, both in compose
+docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up
+
+# just the broker, with the server on the host (5672 and the management UI on
+# 15672 are published for exactly this)
+docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
+FS_QUEUE_BACKEND=rabbitmq \
+FS_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ \
+  julia --project=. -t auto bin/server.jl
+```
+
+Queues are named `<FS_AMQP_PREFIX>.<stage>` — `darkstruct.classify`,
+`.enrich`, `.triage`, `.language` — so the broker's queue list reads like the
+pipeline, and `/stats` reports each one's depth from a once-a-second poll.
+
+### What it guarantees, and what it deliberately doesn't
+
+- **At least once, not exactly once.** Stages 1 and 3 publish downstream *then*
+  ack upstream, so a crash in that window redelivers a job that was already
+  routed. Safe for the same reason replay is: every stage is a pure function of
+  the file's bytes and every commit is idempotent. The one new case is that a
+  duplicate can run *concurrently* with the original and find the file already
+  committed; `worker_loop` recognises a vanished `job.path` and settles it
+  quietly, rather than quarantining a file that in fact succeeded.
+- **Publisher confirms on intake only.** A `202` means the broker has the file,
+  because a client may delete its copy on the strength of it. That costs a round
+  trip per file and serializes intake publishes; `FS_AMQP_CONFIRMS=false` turns
+  it off. Inter-stage publishes are fire-and-forget by design — a lost one leaves
+  its source job unacked, and redelivery repairs it for free, on a handoff that
+  otherwise costs 0.12 µs.
+- **Advisory capacity.** `enqueue!` compares `FS_QUEUE_CAPACITY` against a depth
+  polled once a second (and adjusted locally in between), so the existing `503`
+  and `blocked_ns` behaviour still works, but a burst can overshoot by up to a
+  poll interval. Making the limit real would mean `x-max-length` with
+  `overflow: reject-publish`, which needs a confirm per message to detect.
+- **No reconnect.** A dropped connection invalidates every in-flight delivery
+  tag, so reconnecting would silently reprocess whatever the workers were
+  holding. Instead the server drains and exits non-zero, and the restart policy
+  brings it back to redelivered messages and an intact `spool/`.
+- **No dead-letter queue.** A failed job is quarantined to `failed/` and then
+  acked: the file's story and the message's story end in the same place. A DLQ
+  would hold messages pointing at files that had already moved.
+- **One consumer process.** `Job` is a claim check carrying a *local* path, so a
+  second server on another host would be handed jobs whose files it cannot see.
+  The broker buys durable resume across restarts, not horizontal scale; scaling
+  out would additionally need `spool/` on shared storage and an aggregated
+  `/stats`.
+- **Restart recovery skips `spool/`.** The broker is the record of what is in
+  flight, so re-driving `spool/` would duplicate the whole backlog on every
+  restart. `FS_RECOVER_SPOOL=true` forces it, for the one case the broker cannot
+  cover: it was purged or recreated and the spooled files are all that is left.
+
+### Tests
+
+The pure parts (URL parsing, the wire format, backend selection, the duplicate
+branch) run in the normal suite. The round trip against a real broker is gated:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
+FS_TEST_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ \
+  julia --project=. -e 'using Pkg; Pkg.test()'
+```
+
+Without `FS_TEST_AMQP_URL` those tests are skipped with a notice, so the suite
+still passes on a machine with no Docker.
 
 ## Running
 
@@ -325,7 +433,7 @@ or worker code changes.
 julia --project=. -e 'using Pkg; Pkg.instantiate()'
 
 # external tools: exiftool (stage 2, required) and github-linguist (stage 4,
-# optional — programming-language detection). e.g. on Debian/Ubuntu:
+# optional, for programming-language detection). e.g. on Debian/Ubuntu:
 #   apt install libimage-exiftool-perl
 #   gem install github-linguist
 
@@ -333,6 +441,10 @@ julia --project=. -e 'using Pkg; Pkg.instantiate()'
 julia --project=. -t auto bin/server.jl
 ```
 
+By default the pipeline's queues are in-process, and nothing external is needed.
+For durable queues that survive a crash, run against RabbitMQ instead — see
+"Queue backends" above.
+
 ## Shutdown
 
 Both SIGINT and SIGTERM trigger the same idempotent graceful drain
@@ -340,12 +452,25 @@ Both SIGINT and SIGTERM trigger the same idempotent graceful drain
 
 - **SIGINT** is caught as an `InterruptException` (we call
   `Base.exit_on_sigint(false)`), so shutdown is clean and quiet.
-- **SIGTERM** can't be intercepted directly — Julia blocks it on worker threads
+
+  One caveat specific to the RabbitMQ backend: Julia delivers SIGINT to whichever
+  task happens to be running on thread 1, which is usually the main loop but is
+  not guaranteed to be — AMQPClient runs receiver tasks of its own, and an
+  interrupt that lands in one of those kills the broker connection instead of
+  reaching the main loop. That is not a stuck server: the depth poller notices
+  the closed connection within a second and asks for the same drain, so the
+  process still stops (within ~4s, exiting non-zero, and logging it as a lost
+  connection rather than an interrupt). **Under a process manager, prefer SIGTERM
+  for the broker backend** — it goes through `atexit`, which has no such lottery.
+  Shutdown on either signal also prints a few `Consumer ... task exiting` warnings
+  from AMQPClient, which are cosmetic: they are its consumer tasks noticing that
+  we cancelled them.
+- **SIGTERM** can't be intercepted directly: Julia blocks it on worker threads
   and handles it in its own runtime, so a user `signal()` handler never fires.
   Instead we hook the drain into an `atexit` handler, which Julia's SIGTERM path
   does run. Caveat: Julia prints its own `signal 15: Terminated` backtrace
-  *before* `atexit` runs. It's harmless noise — the drain still completes right
+  *before* `atexit` runs. It's harmless noise, and the drain still completes
-  after it — but if you want a fully quiet stop under a process manager,
+  right after it, but for a fully quiet stop under a process manager,
   configure it to send SIGINT instead (systemd: `KillSignal=SIGINT`; Docker:
   `STOPSIGNAL SIGINT`). Give the stop timeout enough headroom to drain
   in-flight work (systemd: `TimeoutStopSec`).
@@ -354,7 +479,7 @@ Both SIGINT and SIGTERM trigger the same idempotent graceful drain
 
 Each file is scored by a fixed-structure neural network (Lux.jl) that answers a
 single binary question: is this file **known** (like the types in the training
-set) or **unknown**? It's novelty detection, not exact file-typing — it won't
+set) or **unknown**? It's novelty detection, not exact file-typing: it won't
 tell you "PDF", just "this looks like something I was trained on, or not".
 
 - **Features:** the first 16 bytes + last 16 bytes of the file, each scaled
@@ -367,24 +492,24 @@ tell you "PDF", just "this looks like something I was trained on, or not".
   fast rather than run without classification.
 - **Effect today:** *active routing*. The class is logged
   (`classification=known|unknown`) and drives the pipeline split: `:known` files
-  go to `known/` for metadata enrichment (stage 2), `:unknown` files go to
+  go onto the stage-2 queue for metadata enrichment, `:unknown` files onto the
-  `unknown/` for content triage (stage 3). The class chooses the downstream
+  stage-3 queue for content triage. The class chooses the downstream stage (the
-  stage; what's still unproven is the model's *accuracy*, not whether the routing
+  file itself stays in `data/spool/` either way); what's still unproven is the
-  path runs.
+  model's *accuracy*, not whether the routing path runs.
 
 The architecture and byte→feature mapping are defined once in `src/model.jl` and
 shared by the trainer and the server, so they can't drift apart.
 
 ### Training
 
-Training is a separate, offline script — it never runs in the request path:
+Training is a separate, offline script. It never runs in the request path:
 
 ```bash
 julia --project=. bin/train.jl <positives_dir> [negatives_dir]
 ```
 
-- **positives_dir** — every file in it (≥32 bytes) is a "known" example.
+- **positives_dir:** every file in it (≥32 bytes) is a "known" example.
-- **negatives_dir** *(optional)* — a grab-bag of *other* real file types used as
+- **negatives_dir** *(optional)*: a grab-bag of *other* real file types used as
   "unknown" examples. Negatives are generated ~1:1 with positives, split 50/50
   between uniform-random byte vectors and grab-bag files. With no grab-bag dir,
   negatives are all random (weaker: the net may just learn "high entropy =
@@ -410,11 +535,8 @@ init, so the artifact is exactly regenerable from the same inputs.
 | `FS_UNKNOWN_QUEUE_CAPACITY` | `1000`       | Max pending triage jobs (then backpressure)  |
 | `FS_TEXT_WORKERS`         | `nthreads()`   | Stage-4 (language enrichment) worker tasks   |
 | `FS_TEXT_QUEUE_CAPACITY`  | `1000`         | Max pending language jobs (then backpressure) |
-| `FS_SPOOL_DIR`            | `data/spool`   | Incoming files (pending classification)      |
+| `FS_SPOOL_DIR`            | `data/spool`   | Every in-flight file, at every stage         |
-| `FS_KNOWN_DIR`            | `data/known`   | Classified-known, awaiting enrichment        |
-| `FS_UNKNOWN_DIR`          | `data/unknown` | Classified-unknown, awaiting content triage  |
 | `FS_BINARY_DIR`           | `data/binary`  | Stage-3 sink: unknown files that look binary  |
-| `FS_TEXT_DIR`             | `data/text`    | Classified-text, awaiting language enrichment |
 | `FS_DONE_DIR`             | `data/done`    | Enriched known files (+ `.meta.json`)        |
 | `FS_TEXT_DONE_DIR`        | `data/text_done` | Enriched text files (+ `.meta.json`)       |
 | `FS_FAILED_DIR`           | `data/failed`  | Files whose processing threw                 |
@@ -431,6 +553,12 @@ init, so the artifact is exactly regenerable from the same inputs.
 | `FS_PROMOTE_MIN_MAGIC`    | `3`            | Required fixed signature positions to nominate |
 | `FS_CLUSTER_CATALOG`      | `data/catalog.json` | Durable stage-5 catalog file (single-owner) |
 | `FS_NOMINATED_DIR`        | `data/nominated` | One JSON per self-nominated cluster (human promote gate) |
+| `FS_QUEUE_BACKEND`        | `channel`      | `channel` (in-process) or `rabbitmq` (durable); see "Queue backends" |
+| `FS_AMQP_URL`             | `amqp://guest:guest@localhost:5672/` | Broker connection, credentials included |
+| `FS_AMQP_PREFIX`          | `darkstruct`   | Queue names are `<prefix>.<stage>`           |
+| `FS_AMQP_PREFETCH`        | worker count   | Unacked messages the broker hands one stage at a time |
+| `FS_AMQP_CONFIRMS`        | `true`         | Wait for a publisher confirm before a `202` (intake only) |
+| `FS_RECOVER_SPOOL`        | `false`        | Re-drive `spool/` at startup even on the broker backend (use after a purged broker) |
 
 > To get real parallelism, start Julia with enough threads (`-t N`) to cover all
 > pools. If `FS_WORKERS + FS_KNOWN_WORKERS + FS_UNKNOWN_WORKERS + FS_TEXT_WORKERS`
@@ -462,9 +590,9 @@ Each file in a request becomes its own job. Responses:
 ### `GET /stats` — per-stage counters
 
 The pipeline counts its own work (`src/stats.jl`), because nothing outside it
-can: `known/`, `unknown/` and `text/` are *transient*, so a file can cross one
+can. Every in-flight file sits in `data/spool/` no matter which stage it has
-between two directory polls and an external sampler will miss exactly the stages
+reached. The stage is a property of the queue holding its reference, and only
-you most want to measure.
+the pipeline can see that; there is no directory to poll.
 
 ```jsonc
 {
@@ -481,7 +609,7 @@ you most want to measure.
 }
 ```
 
-Counters are monotonic since startup, Prometheus-style — rates are the reader's
+Counters are monotonic since startup, Prometheus-style. Rates are the reader's
 job, so a scrape is stateless and two readers can't disturb each other. Take two
 scrapes Δt apart and subtract:
 
@@ -497,14 +625,14 @@ each pool worked to keep up. The bottleneck sits near 1.0 with its queue backing
 up while its neighbours idle.
 
 `blocked_seconds` is what keeps that true. Stages 1 and 3 apply *blocking*
-backpressure — a full downstream queue means the handler parks and retries rather
+backpressure (a full downstream queue means the handler parks and retries rather
-than dropping the file — and that wait happens inside the handler. Counting it as
+than dropping the file), and that wait happens inside the handler. Counting it as
 busy would pin stage 1 at 1.0 whenever stage 2 is the real jam, making every
 stage upstream of a jam look like the jam. Subtracted out, utilization means
 "doing its own work", and a high blocked share becomes its own signal: a stage
 blocked 90% of the time is naming its successor.
 
-The counters are a handful of atomic adds per file, recorded in `worker_loop` —
+The counters are a handful of atomic adds per file, recorded in `worker_loop`:
 the one place every stage's work passes through, so a new stage is instrumented
 the moment it is wired up, and never on the read path.
 
@@ -525,13 +653,13 @@ Running all of them from a clean checkout:
 ```bash
 julia --project=. -e 'using Pkg; Pkg.instantiate()'   # once
 
-# 1. the model, on its own — no server involved
+# 1. the model, on its own; no server involved
 julia --project=. -t auto bin/bench_model.jl
 
-# 1b. stage 1 taken apart — also no server
+# 1b. stage 1 taken apart; also no server
 julia --project=. -t auto bin/bench_stage1.jl
 
-# 1c. stage 2 taken apart — needs a directory of real files, not generated ones
+# 1c. stage 2 taken apart; needs a directory of real files, not generated ones
 julia --project=. -t auto bin/bench_stage2.jl
 
 # 2. the pipeline. Start the server in one terminal…
@@ -550,19 +678,19 @@ julia --project=. bin/cluster_calibrate.jl ../training_set
 The test suite is `julia --project=. -t auto -e 'using Pkg; Pkg.test()'`.
 
 `bin/bench.jl` must run on the same machine as the server (it reads the sink dirs
-and `/proc`); otherwise it only speaks HTTP — `/upload`, `/health`, and `/stats`
+and `/proc`); otherwise it only speaks HTTP: `/upload`, `/health`, and `/stats`
 for the per-stage numbers.
 
 Three properties of this design dictate how it measures:
 
 - **HTTP latency is not throughput.** `/upload` returns `202` once the bytes are
-  spooled and a reference is enqueued — all four stages run *after* the response,
+  spooled and a reference is enqueued, and all four stages run *after* the
-  so `ab`/`hey`/`wrk` would only ever measure intake. The bench instead uploads a
+  response, so `ab`/`hey`/`wrk` would only ever measure intake. The bench
-  corpus and polls the terminal sinks (`done/`, `text_done/`, `binary/`,
+  instead uploads a corpus and polls the terminal sinks (`done/`, `text_done/`, `binary/`,
   `failed/`) until the count stops moving, and reports both numbers separately:
-  intake rate *and* end-to-end completion rate. It also samples the intermediate
+  intake rate *and* end-to-end completion rate. It also samples `spool/`, whose
-  stage dirs, so the peak depth of `spool/`/`known/`/`unknown/`/`text/` shows
+  peak depth is the high-water mark of files in flight. But *which* stage they
-  where work piles up.
+  are waiting on comes from `/stats`, not from disk.
 - **End-to-end throughput doesn't name the slow stage.** The four stages run
   concurrently behind their own queues, so the pipeline's rate *is* the slowest
   stage's rate and the others are invisible in it. The bench scrapes
@@ -580,8 +708,8 @@ Three properties of this design dictate how it measures:
     bottleneck      stage 4 (language) at 87.0% utilization of 16 worker(s)
   ```
 
-  (400 mixed files, 16 KiB each, concurrency 16. Stage 4 is the constraint —
+  (400 mixed files, 16 KiB each, concurrency 16. Stage 4 is the constraint,
-  `github-linguist` is a process spawn per file at ~850 ms — and the 200 text
+  `github-linguist` being a process spawn per file at ~850 ms, and the 200 text
   files queue up behind it while stages 1 and 3 idle at under 10%.) Read *down*
   the util column, not the files/s column: each stage sees a different subset of
   the corpus, so a low rate can just mean little work was routed there.
@@ -599,9 +727,9 @@ Three properties of this design dictate how it measures:
   | 256 MiB × 4 | 4 | none measurable (peak stayed under the baseline) |
 
   **Read the shape, not the digits.** Growth is flat across an 8× range of file
-  sizes — 14–31 MiB whether the upload is 256 MiB or 2 GiB — which is the claim
+  sizes: 14–31 MiB whether the upload is 256 MiB or 2 GiB. That is the claim
-  that matters: nothing scales with file size, so nothing is buffering. The
+  that matters, because nothing scales with file size, so nothing is buffering.
-  absolute figures are *not* precise to the megabyte. A freshly started server
+  The absolute figures are *not* precise to the megabyte. A freshly started server
   settles anywhere in an ~860–985 MiB band, so run-to-run variance in the
   baseline is comparable to the growth being measured; the residual is GC churn
   from the chunk reads, not retained buffers.
@@ -613,20 +741,20 @@ Three properties of this design dictate how it measures:
     (or none at all, as the concurrency-4 row above did).
   - **Linear-in-concurrency is not currently demonstrated.** An earlier
     measurement put 4 concurrent 256 MiB uploads at 86.9 MiB (21.7 per in-flight
-    upload), but that run predates a fix to how the baseline was sampled — it was
+    upload), but that run predates a fix to how the baseline was sampled: it was
-    read *before* the peak counter was reset, against a different origin — and the
+    read *before* the peak counter was reset, against a different origin. The
-    re-measurement above cannot reproduce it: the growth sits under the noise
+    re-measurement above cannot reproduce it, and the growth sits under the noise
     floor. Expect concurrency to cost memory; don't trust a specific coefficient
     without a quieter machine or many more runs.
 
-  Before intake
+  Before intake was streamed, the same 256 MiB upload grew RSS by ~700 MiB and 4
-  was streamed, the same 256 MiB upload grew RSS by ~700 MiB and 4 concurrent ones
+  concurrent ones pushed a 950 MiB baseline past 2 GiB. So a `--size` sweep that
-  pushed a 950 MiB baseline past 2 GiB, so **a `--size` sweep that slopes upward is
+  slopes upward is the regression signal: something has started buffering bodies
-  the regression signal** — it means something has started buffering bodies again.
+  again.
 
 Flags: `--files`, `--size` (`8k`/`64m`/`1g`), `--concurrency`, `--kind`
-(`binary`/`text`/`mixed` — chooses which stages get loaded), `--corpus DIR` (real
+(`binary`/`text`/`mixed`, which chooses the stages that get loaded), `--corpus
-files, the only way to exercise stage 2's exiftool path), `--pid`, `--no-mem`,
+DIR` (real files, the only way to exercise stage 2's exiftool path), `--pid`, `--no-mem`,
 `--no-stats`, `--sample-ms`, `--timeout`, `--json PATH`. Full list in the script
 header. The `--json` output carries the per-stage numbers too, so a sweep can be
 compared run to run.
@@ -634,7 +762,7 @@ compared run to run.
 Two caveats the script reports rather than hides: it counts sink *deltas*, so it
 warns if the pipeline isn't idle at the start (in-flight leftovers would be
 counted as its own throughput); and because Julia's GC returns memory to the OS
-lazily, a second run in the same process starts from an inflated baseline — it
+lazily, a second run in the same process starts from an inflated baseline. It
 resets the kernel's peak-RSS counter (`/proc/<pid>/clear_refs`) per run and flags
 a drifted baseline, but for a clean growth figure restart the server between
 memory runs.
@@ -642,7 +770,7 @@ memory runs.
 ### Stage-1 component benchmark (`bin/bench_stage1.jl`)
 
 `bin/bench.jl` reports stage 1 as one number and `bin/bench_model.jl` takes the
-*classifier* apart — but stage 1 is more than the model. Per file it also
+*classifier* apart, but stage 1 is more than the model. Per file it also
 renames the file into its stage directory, pushes a reference onto the
 downstream queue, and logs. `bin/bench_stage1.jl` times each of those in
 isolation, then times the real `handle_classify_job` end to end so the parts can
@@ -666,33 +794,32 @@ Measured on this machine (Ryzen 7 2700X, 8 cores/16 threads, Julia 1.12; 2000 ×
 | **`handle_classify_job`** | **28.3 µs** | 100% |
 
 **This benchmark is why stage 1's per-file log lines are `@debug` rather than
-`@info`.** As `@info` they cost ~71 µs of the handler's ~118 µs — about 6× the
+`@info`.** As `@info` they cost ~71 µs of the handler's ~118 µs, about 6× the
-classifier and 6× the rename — and nearly all of it was `ConsoleLogger`
+classifier and 6× the rename, and nearly all of it was `ConsoleLogger`
 *formatting* (~64 µs), not the `FlushLogger`'s per-message flush (~8 µs on top).
 Demoting them took stage 1 from 8.5k files/s to 35.3k files/s on a single worker,
 a 4.2× speedup for no algorithmic change. The script still prices a formatted
 line, so the cost of turning them back on is visible: running the handler under
-`JULIA_DEBUG=FileServer` measures 133 µs per file, a 4.7× slowdown. That is the
+`JULIA_DEBUG=DarkStruct` measures 133 µs per file, a 4.7× slowdown. That is the
-trade — per-file tracing is available when you want it, and off by default,
+trade. Per-file tracing is available when you want it and off by default, with
-with `GET /stats` giving per-file observability that is counted rather than
+`GET /stats` giving per-file observability that is counted, not formatted.
-formatted.
 
 What's left is evenly split between the rename and the classifier, and neither
 has an easy 2×. Two things worth knowing:
 
 - **The rename, not the model, is the single largest component** (11.7 µs), and
   it's a plain `mv` within one filesystem. Inside `classify`, the same pattern
-  holds: 7.9 µs of the 10.7 µs is `read_features` — the `open`, the two reads
+  holds: 7.9 µs of the 10.7 µs is `read_features` (the `open`, the two reads
-  and the `seek` — against 2.3 µs of actual inference. Stage 1 is now a
+  and the `seek`) against 2.3 µs of actual inference. Stage 1 is now a
   filesystem-bound stage with a neural network attached, not the reverse.
 - **Stage 1 now peaks at ~4 workers.** With the logger removed from the hot path
   the sweep reads 35.0k/s at 1 worker, 66.6k/s at 2, **73.3k/s at 4**, then
-  *falls back* to 60.1k/s at 8 and 53.3k/s at 16 — every worker renaming into the
+  *falls back* to 60.1k/s at 8 and 53.3k/s at 16, because every worker renaming
-  same two directories contends on the same directory inode. That ceiling
+  into the same two directories contends on the same directory inode. That ceiling
   coincides with the one `bin/bench_model.jl` finds for inference, so ~4 is the
   number from both directions: raising `FS_WORKERS` past it costs throughput.
 
-Reported times are the **minimum** over trials. Flags: `--files`, `--reps`,
+Reported times are the minimum over trials. Flags: `--files`, `--reps`,
 `--trials`, `--size`, `--dir`, `--model`, `--threads`, `--no-threads`,
 `--json PATH`.
 
@@ -701,7 +828,7 @@ Reported times are the **minimum** over trials. Flags: `--files`, `--reps`,
 Stage 2 is the one stage whose cost is dominated by something outside Julia
 entirely: it forks `exiftool`, a Perl program, once per file. `bin/bench.jl`
 reports the stage as a single throughput number, which can't distinguish "the
-extraction is slow" from "the *spawn* is slow" — and those have opposite fixes.
+extraction is slow" from "the *spawn* is slow", and those have opposite fixes.
 `bin/bench_stage2.jl` times each piece in isolation, then times the real
 `handle_known_job` end to end:
 
@@ -713,14 +840,14 @@ Two things make this benchmark different from the stage-1 one:
 
 - **The corpus must be real files.** exiftool's cost depends on what it finds; a
   file of random bytes bails out early and understates the stage by ~10×. The
-  default corpus is `data/done` — files that already went through stage 2 on this
+  default corpus is `data/done`, files that already went through stage 2 on this
   machine. `--corpus PATH` points it elsewhere.
 - **It prices the alternatives to one-fork-per-file**, because if the fork
   dominates then the only fixes are to stop paying it per file. `exiftool
   (batched Nx)` runs the whole corpus through one process; `exiftool
   (-stay_open)` keeps one process alive and feeds it one file at a time over a
-  pipe — the shape a streaming pipeline could actually adopt. Both are measured,
+  pipe, which is the shape a streaming pipeline could actually adopt. Both are
-  not assumed.
+  measured, not assumed.
 
 Measured on this machine (Ryzen 7 2700X, 8 cores/16 threads, Julia 1.12,
 exiftool 12.40; 150 real files / 102 MiB, minimum of 2 trials):
@@ -737,8 +864,8 @@ exiftool 12.40; 150 real files / 102 MiB, minimum of 2 trials):
 | *alt:* `exiftool -stay_open` | 40.1 ms | 29% |
 | *alt:* `exiftool` batched 150× | 37.3 ms | 27% |
 
-**Stage 2 is exiftool and nothing else.** Everything the Julia code does —
+**Stage 2 is exiftool and nothing else.** Everything the Julia code does
-parsing, normalizing, the durable sidecar-first commit, the log line — sums to
+(parsing, normalizing, the durable sidecar-first commit, the log line) sums to
 about 1.5% of the stage. There is no point optimizing any of it.
 
 **More than half the stage is interpreter startup, not metadata extraction.**
@@ -751,7 +878,7 @@ available win in this stage; it is measured here but not yet implemented.
 
 **This benchmark is also why `run_with_timeout` no longer polls.** The original
 watchdog polled with `sleep(0.1)` and then joined the polling task, so every call
-paid the remainder of an in-flight sleep *after* the child had already exited —
+paid the remainder of an in-flight sleep *after* the child had already exited:
 ~25 ms per file here, and a measured 101 ms on a process that exits instantly.
 Replacing it with a one-shot `Timer` took the stage from 164.6 ms to 138.3 ms per
 file (6→8 files/s on one worker) and cost nothing in behavior. Stage 4 shares the
@@ -759,7 +886,7 @@ wrapper and got the same fix for free.
 
 Writing the missing tests for that wrapper turned up a second, worse problem:
 **the timeout was never enforceable.** `wait(proc)` returns only once the
-captured stdout pipe closes, and any grandchild inherits that pipe — so
+captured stdout pipe closes, and any grandchild inherits that pipe, so
 signalling the child alone left the worker blocked until the whole process tree
 finished on its own (a `sh -c "trap '' TERM; sleep 30"` child ran the full 30 s
 against a 1 s timeout, under both the old and new watchdog). The child now runs
@@ -769,7 +896,7 @@ these children are short-lived and timeout-bounded, which is the cheaper side of
 it.
 
 **Stage 2 scales to ~8 workers, then flattens**: 8 files/s at 1 worker, 14 at 2,
-28 at 4, **49 at 8**, and 49 at 16 — the machine runs out of cores to run Perl
+28 at 4, **49 at 8**, and 49 at 16. The machine runs out of cores to run Perl
 on, which is exactly what you'd expect of a stage that is ~100% subprocess. Note
 that the sweep pulls from a shared counter rather than splitting the corpus into
 contiguous slices: per-file exiftool time spans two orders of magnitude on a real
@@ -782,13 +909,13 @@ which is far too fast to be a real disk flush. The durability that
 even though the code is correct. That's a correctness question, not a speed one,
 and it is not yet resolved.
 
-Reported times are the **minimum** over trials. Flags: `--files`, `--reps`,
+Reported times are the minimum over trials. Flags: `--files`, `--reps`,
 `--trials`, `--corpus`, `--dir`, `--timeout`, `--threads`, `--no-threads`,
 `--no-stay-open`, `--json PATH`.
 
 ### Model microbenchmark (`bin/bench_model.jl`)
 
-`bin/bench.jl` reports stage 1 as a single number — the wall time of
+`bin/bench.jl` reports stage 1 as a single number: the wall time of
 `handle_classify_job`, which is a feature read, an inference, a rename, a
 (disabled) debug line, and whatever contention the other three pools create.
 That's the right number for capacity planning and the wrong one for "is the
@@ -808,10 +935,10 @@ Measured on this machine (Ryzen 7 2700X, 8 cores/16 threads, Julia 1.12):
 | `read_features` | **4.2–6.0 µs** | flat across a 262,144× size range (1 KiB → 256 MiB) |
 | `classify()` | **8.4 µs** | 73% feature read, 27% inference |
 
-So the model is **not** the pipeline's problem, by three orders of magnitude: the
+So the model is not the pipeline's problem, by three orders of magnitude: the
 same run measured stage 1 at 38.7 ms per file, ~4,500× the 8.4 µs `classify()`
 costs. Whatever stage 1 spends its time on, it isn't the network. (That 38.7 ms
-predates the `@debug` demotion above and is a whole-pipeline figure — it includes
+predates the `@debug` demotion above and is a whole-pipeline figure. It includes
 time the stage-1 worker spends *blocked* on a full downstream queue, which is why
 it is three orders of magnitude above the 28.3 µs the handler costs in
 isolation. For the uncontended split, see
@@ -829,13 +956,13 @@ Two findings worth acting on if stage 1 ever *does* become the constraint:
   control kernel through the same sweep to place the blame: the control reaches
   14.3× at 16 tasks (90% efficiency) on the same box, so the machine
   parallelizes and `Lux.apply` doesn't. GC is only ~1% of it, so allocation
-  pressure isn't the explanation either — the cause is inside Lux/BLAS and is
+  pressure isn't the explanation either. The cause is inside Lux/BLAS and is
   not diagnosed here. The practical consequence: raising `FS_WORKERS` past ~4
   adds no classification throughput.
 
-Reported times are the **minimum** over trials — for a microbenchmark the floor
+Reported times are the minimum over trials, because for a microbenchmark the
-is the signal and everything above it is scheduler and GC noise — and every timed
+floor is the signal and everything above it is scheduler and GC noise. Every
-loop stores its result in a sink so a pure call can't be hoisted out of the loop.
+timed loop stores its result in a sink so a pure call can't be hoisted out.
 Flags: `--model`, `--reps`, `--trials`, `--batches`, `--sizes`, `--no-threads`,
 `--json PATH`.
 
@@ -843,10 +970,11 @@ Flags: `--model`, `--reps`, `--trials`, `--batches`, `--sizes`, `--no-threads`,
 
 ```
 src/
-  FileServer.jl   module + run() (startup, recovery, workers, serve, shutdown)
+  DarkStruct.jl   module + run() (startup, recovery, workers, serve, shutdown)
   config.jl       Config struct + env parsing
   job.jl          Job (the queue reference)
   queue.jl        JobQueue seam + in-process ChannelQueue
+  rabbit.jl       RabbitMQ-backed JobQueue: durable queues, acks, confirms
   stats.jl        per-stage counters behind GET /stats (throughput, utilization)
   multipart.jl    streaming multipart/form-data reader (intake never buffers a file)
   spool.jl        filename sanitizing, streaming spool/move, startup recovery
@@ -859,6 +987,8 @@ src/
   catalog.jl      durable single-owner format catalog + sweep + nominations (stage 5, phase B)
   worker.jl       parametrized worker loop + classify/enrich/triage/language handlers
   server.jl       HTTP routes + the streaming /upload handler
+docker-compose.yml           the server, in-process queues
+docker-compose.rabbitmq.yml  overlay: adds the broker and switches the backend
 bin/
   server.jl           entry point
   bench.jl            throughput + memory harness against a running server

bin/bench.jl

@@ -1,12 +1,12 @@
 #!/usr/bin/env julia
 #
-# bench.jl — measure end-to-end throughput and server memory for a running FileServer.
+# bench.jl: measure end-to-end throughput and server memory for a running DarkStruct.
 #
 # Two things make this pipeline awkward to benchmark with off-the-shelf tools
 # (ab/hey/wrk), and both shape what this script does:
 #
 #   1. HTTP latency is not throughput. /upload returns 202 as soon as the bytes
-#      are spooled and a reference is enqueued — all four stages run afterwards.
+#      are spooled and a reference is enqueued, and all four stages run afterwards.
 #      So real throughput is the *arrival rate at the terminal sinks*
 #      (done/, text_done/, binary/, failed/), not the response rate. We upload a
 #      corpus, then poll the sinks until the file count stops moving.
@@ -14,8 +14,9 @@
 #   2. End-to-end throughput doesn't name the slow stage. The four stages run
 #      concurrently behind their own queues, so the pipeline's rate is the
 #      slowest stage's rate and the others are invisible. Directory polling can't
-#      recover them either — known/, unknown/ and text/ are transient, and a file
+#      recover them either: every in-flight file sits in spool/ whatever stage it
-#      can cross one between two samples. So the server keeps per-stage counters
+#      is at, since stages route by enqueueing rather than by moving bytes. So the
+#      server keeps per-stage counters
 #      (src/stats.jl) and we scrape GET /stats before and after: the deltas give
 #      each stage's throughput, mean service time, and worker utilization, and
 #      utilization is what actually names the bottleneck (see `stage_report`).
@@ -35,7 +36,7 @@
 #
 # The harness talks to the server only over HTTP (/upload, /health, /stats) and
 # reads the on-disk sink layout; it must run on the same machine (sink dirs and
-# /proc). If /stats is missing — an older build — everything else still works and
+# /proc). If /stats is missing (an older build) everything else still works and
 # the per-stage section is skipped.
 #
 # Usage:
@@ -45,7 +46,7 @@
 #   --files N          number of files to upload (default: 200)
 #   --size S           size of each generated file, e.g. 4k, 512k, 8m, 1g (default: 64k)
 #   --concurrency J    uploads in flight at once (default: 8)
-#   --kind K           binary | text | mixed — what to generate (default: binary)
+#   --kind K           binary | text | mixed: what to generate (default: binary)
 #   --corpus DIR       upload an existing directory instead of generating
 #                      (the only way to exercise stage 2: point it at real known files)
 #   --keep-corpus      don't delete the generated corpus on exit
@@ -142,7 +143,7 @@ end
 #
 # Resolved from the same environment variables src/config.jl reads, so a server
 # started with custom dirs is benchmarked correctly. Kept as a standalone table
-# rather than `using FileServer` so the harness doesn't pay to load Lux.
+# rather than `using DarkStruct` so the harness doesn't pay to load Lux.
 
 sinkdirs() = (
     done      = get(ENV, "FS_DONE_DIR",      "data/done"),
@@ -151,12 +152,12 @@ sinkdirs() = (
     failed    = get(ENV, "FS_FAILED_DIR",    "data/failed"),
 )
 
-stagedirs() = (
+# One directory, not four. Files no longer move between stages: spool/ holds
-    spool   = get(ENV, "FS_SPOOL_DIR",   "data/spool"),
+# every in-flight file at every stage, and which stage it has reached lives in
-    known   = get(ENV, "FS_KNOWN_DIR",   "data/known"),
+# the queue holding its reference (src/worker.jl header). So this depth is
-    unknown = get(ENV, "FS_UNKNOWN_DIR", "data/unknown"),
+# "files in flight", full stop; per-stage depth comes from /stats, which is the
-    text    = get(ENV, "FS_TEXT_DIR",    "data/text"),
+# only place that can see it at all.
-)
+stagedirs() = (spool = get(ENV, "FS_SPOOL_DIR", "data/spool"),)
 
 "Count work items in `dir`, ignoring the .meta.json sidecars stages 2/4 write."
 function count_files(dir::AbstractString)::Int
@@ -199,7 +200,7 @@ Find the running server process, or `nothing`.
 server: any shell launched with the command in its own argv (`sh -c 'julia …
 bin/server.jl > log'`, a `setsid`/`nohup` wrapper, even the terminal running the
 benchmark) matches the same pattern. Sampling one of those reports a few MiB of
-shell as the server's memory — a wrong answer that looks plausible, which is the
+shell as the server's memory: a wrong answer that looks plausible, which is the
 worst kind.
 
 So candidates are filtered by what each process *is* (`/proc/<pid>/comm`, the
@@ -308,7 +309,7 @@ function stage_deltas(before, after, peak_depth::Dict{Int,Int})
 end
 
 "JSON has no NaN. A stage that completed nothing has no service time, and `null`
-is the honest way to say that — writing NaN just makes JSON3 throw."
+is the honest way to say that; writing NaN just makes JSON3 throw."
 json_num(x::Real) = isfinite(x) ? x : nothing
 
 pad(s, n) = rpad(string(s), n)
@@ -320,7 +321,7 @@ Print the per-stage table and say which stage is the bottleneck.
 The verdict reads utilization, not throughput: in a pipeline every stage
 completes the same files, so at steady state they all report nearly the same
 files/s regardless of which one is the constraint. What separates them is how
-hard each pool had to work to keep up — the bottleneck is pinned near 1.0 while
+hard each pool had to work to keep up: the bottleneck is pinned near 1.0 while
 its neighbours idle.
 """
 function stage_report(deltas::Vector{StageDelta}, window::Float64)
@@ -355,14 +356,14 @@ function stage_report(deltas::Vector{StageDelta}, window::Float64)
             "$(fmt(utilization(top, window) * 100, 0))% utilization of " *
             "$(top.workers) worker(s)")
     if utilization(top, window) < 0.5
-        println("                  — but no stage is near saturated: the pipeline is " *
+        println("                  ...but no stage is near saturated: the pipeline is " *
                 "waiting on intake,\n                    not on itself. Raise --concurrency " *
                 "or --files to load it properly.")
     end
     for d in worked
         blocked_share(d) > 0.25 && println("  ! stage $(d.stage) ($(d.name)) spent " *
             "$(fmt(blocked_share(d) * 100, 0))% of its time parked on a full downstream " *
-            "queue —\n    it is being held up by the stage after it, not doing that work itself.")
+            "queue.\n    It is being held up by the stage after it, not doing that work itself.")
     end
     return nothing
 end
@@ -458,7 +459,7 @@ end
 Upload every path, at most `concurrency` in flight.
 
 A bounded set of worker tasks pulling from a shared index keeps exactly
-`concurrency` requests in flight for the whole run — unlike batching, where each
+`concurrency` requests in flight for the whole run, unlike batching, where each
 batch stalls on its slowest (largest) file and the real concurrency sags.
 """
 function upload_all(url::String, paths::Vector{String}, concurrency::Int)
@@ -503,7 +504,7 @@ function main(argv)
     try
         HTTP.get(string(rstrip(url, '/'), "/health"); retry = false, readtimeout = 5)
     catch e
-        println(stderr, "cannot reach $url/health — is the server running?")
+        println(stderr, "cannot reach $url/health. Is the server running?")
         println(stderr, "  start it with: julia --project=. -t auto bin/server.jl")
         return 1
     end
@@ -513,7 +514,7 @@ function main(argv)
     # Leftovers mid-pipeline would land in the sinks during our window and be
     # counted as our throughput, so say so up front rather than quietly skewing.
     pending = total(counts(stages))
-    pending > 0 && @warn "pipeline is not idle: $pending file(s) in the stage dirs; " *
+    pending > 0 && @warn "pipeline is not idle: $pending file(s) still in spool/; " *
                          "throughput will include their completions"
 
     # --- corpus
@@ -547,7 +548,7 @@ function main(argv)
                                     "reporting sampled RSS only"
                 # Baseline is read *after* the reset, not before. VmHWM restarts
                 # from whatever RSS is at the moment of the reset, so a baseline
-                # sampled earlier is measured against a different origin — and if
+                # sampled earlier is measured against a different origin, and if
                 # the GC hands memory back in between, the run reports negative
                 # growth, which is nonsense on its face.
                 r2 = read_rss(pid)
@@ -570,7 +571,7 @@ function main(argv)
                   "(the server predates src/stats.jl)"
         end
 
-        # --- sampler: RSS curve, stage dir depths, and queue depths.
+        # --- sampler: RSS curve, spool depth, and queue depths.
         stop = Threads.Atomic{Bool}(false)
         rss_samples = Float64[]
         depth_max = Dict(k => 0 for k in keys(stages))
@@ -586,8 +587,8 @@ function main(argv)
                     depth_max[k] = max(depth_max[k], getfield(d, k))
                 end
                 # Queue depth, unlike directory depth, can't be missed by a slow
-                # sample in the same way — a file's *reference* sits in the queue
+                # sample in the same way, since a file's *reference* sits in the
-                # for the whole time it waits — so this is the depth the stage
+                # queue for the whole time it waits, so this is the depth the stage
                 # table reports.
                 if stats_before !== nothing
                     s = scrape_stats(url)
@@ -672,7 +673,7 @@ function main(argv)
                 "$(human(corpusbytes / length(paths))) avg")
         println("concurrency       $(opts["concurrency"])")
         println()
-        println("INTAKE (HTTP 202 — bytes spooled, not processed)")
+        println("INTAKE (HTTP 202: bytes spooled, not processed)")
         println("  accepted        $accepted of $(length(paths))   [202: $n_202, 503: $n_503, error: $n_err]")
         println("  wall            $(fmt(intake_secs))s")
         println("  rate            $(fmt(accepted / max(intake_secs, 1e-9))) files/s, " *
@@ -699,7 +700,7 @@ function main(argv)
             println("SERVER MEMORY (pid $pid)")
             println("  baseline RSS    $(human(baseline_rss))")
             println("  peak RSS        $(human(peak_rss))" *
-                    (peak_reset ? "  (kernel VmHWM, reset at start)" : "  (sampled — may miss spikes)"))
+                    (peak_reset ? "  (kernel VmHWM, reset at start)" : "  (sampled; may miss spikes)"))
             growth = peak_rss - baseline_rss
             if growth <= 0
                 # RSS never got back to where it started, so the run's own cost
@@ -707,13 +708,13 @@ function main(argv)
                 # Printing a negative "growth" would invite reading a memory
                 # *saving* into what is really "too small to measure here".
                 println("  growth          none measurable (peak never exceeded the baseline)")
-                println("                  The baseline was still falling when we sampled it — give the")
+                println("                  The baseline was still falling when we sampled it. Give the")
                 println("                  server ~30s to settle after startup for a comparable figure.")
             else
                 println("  growth          $(human(growth))")
                 println("  per in-flight   $(human(growth / opts["concurrency"])) " *
                         "at $(human(corpusbytes / length(paths))) avg file size")
-                println("                  (should not grow with file size — intake streams to disk)")
+                println("                  (should not grow with file size: intake streams to disk)")
             end
             # Julia's GC returns memory to the OS lazily, so a second run on the
             # same process starts from an inflated baseline and under-reports
@@ -728,10 +729,10 @@ function main(argv)
         println("=" ^ 68)
 
         sink_delta.failed > 0 &&
-            println("\nnote: $(sink_delta.failed) file(s) landed in $(sinks.failed) — check the server log.")
+            println("\nnote: $(sink_delta.failed) file(s) landed in $(sinks.failed); check the server log.")
         timed_out &&
             println("\nnote: drain stalled with $(accepted - completed) file(s) outstanding. " *
-                    "Check the server log and the stage dirs; raise --timeout if the pipeline is just slow.")
+                    "Check the server log and spool/; raise --timeout if the pipeline is just slow.")
 
         if opts["json"] !== nothing
             result = (

bin/bench_model.jl

@@ -1,6 +1,6 @@
 #!/usr/bin/env julia
 #
-# bench_model.jl — microbenchmark the classifier in isolation, with no server,
+# bench_model.jl: microbenchmark the classifier in isolation, with no server,
 # no queue, and no disk in the way.
 #
 # bin/bench.jl measures the *pipeline*: it reports stage 1 as one number, the
@@ -12,7 +12,7 @@
 #
 #   read_features   open, read 16 bytes, seek, read 16 bytes, scale
 #   Lux.apply       the network itself, on a feature vector already in memory
-#   classify        both together — what stage 1 actually calls per file
+#   classify        both together: what stage 1 actually calls per file
 #
 # Three properties are worth checking beyond the raw per-file cost:
 #
@@ -21,7 +21,7 @@
 #     (This is the same claim bin/bench.jl makes about memory, on the CPU axis.)
 #   * Batching should be much cheaper per file. A 32x1 matmul wastes most of a
 #     BLAS call; if batch-64 inference is many times cheaper per file, that is
-#     the headroom a batching stage-1 would buy — worth knowing before building
+#     the headroom a batching stage-1 would buy, worth knowing before building
 #     one, since today the pipeline classifies strictly one file at a time.
 #   * Inference should scale across threads. `Classifier` is shared read-only by
 #     the whole stage-1 pool on the claim that Lux inference is pure. If per-file
@@ -48,7 +48,7 @@ using Statistics
 using Printf
 
 # The script is run directly, not as part of the package, so pull in exactly the
-# pieces the classifier needs. `Lux`/`JLD2` first — model.jl and classify.jl both
+# pieces the classifier needs. `Lux`/`JLD2` first, because model.jl and classify.jl both
 # assume the including scope already has them (see the note at the top of model.jl).
 using Lux
 using JLD2
@@ -156,7 +156,7 @@ fmt2(x::Real) = @sprintf("%.2f", x)
 """
     control_kernel(x) -> Float64
 
-Pure arithmetic, no allocation, no library call — deliberately dependent
+Pure arithmetic, no allocation, no library call, and deliberately dependent
 (each step needs the last) so the compiler can't vectorize it away, and sized to
 land in the same microsecond neighbourhood as one `Lux.apply`.
 """
@@ -209,7 +209,7 @@ end
 """
 Write a file of exactly `size` random bytes, in bounded chunks.
 
-Content is random rather than zeros so the classifier sees a realistic input —
+Content is random rather than zeros so the classifier sees a realistic input,
 and so the filesystem can't cheat with a sparse file, which would make the tail
 `seek` unrepresentatively fast.
 """
@@ -252,7 +252,7 @@ function main(argv)
         Lux.apply(clf.model, x1, clf.ps, clf.st)
     end
     println()
-    println("INFERENCE (Lux.apply, batch 1 — features already in memory)")
+    println("INFERENCE (Lux.apply, batch 1; features already in memory)")
     println("  per call        $(human_time(infer_ns))   $(human_rate(rate(infer_ns)))")
     println("  allocations     $(human_bytes(infer_bytes)) per call")
     results["inference_batch1"] = (; ns = infer_ns, bytes = infer_bytes, per_sec = rate(infer_ns))
@@ -302,14 +302,14 @@ function main(argv)
     results["read_features"] = read_rows
     flat = length(read_rows) > 1 ?
         maximum(r.ns for r in read_rows) / minimum(r.ns for r in read_rows) : 1.0
-    @printf("  spread across a %.0fx size range: %.1fx — %s\n",
+    @printf("  spread across a %.0fx size range: %.1fx, %s\n",
             maximum(sizes) / minimum(sizes), flat,
             flat < 3 ? "flat, as designed (it seeks to the tail)" :
                        "NOT flat: something is reading more than 32 bytes")
 
     # --- 4. classify(): what stage 1 calls, I/O and inference together.
     println()
-    println("CLASSIFY (read_features + Lux.apply — one whole stage-1 file)")
+    println("CLASSIFY (read_features + Lux.apply; one whole stage-1 file)")
     dir2 = mktempdir(; prefix = "fsmodel-")
     classify_ns = 0.0
     try
@@ -378,7 +378,7 @@ function main(argv)
         end
         results["thread_scaling"] = thread_rows
 
-        # A poor scaling curve has two possible authors — the model or the box —
+        # A poor scaling curve has two possible authors, the model or the box,
         # and the table alone can't tell them apart. So run the same sweep on a
         # kernel that is pure arithmetic with no allocation and no library
         # underneath: whatever *it* achieves is this machine's ceiling for

bin/bench_stage1.jl

@@ -1,6 +1,6 @@
 #!/usr/bin/env julia
 #
-# bench_stage1.jl — take stage 1 apart and find the slowest component.
+# bench_stage1.jl: take stage 1 apart and find the slowest component.
 #
 # bin/bench.jl reports stage 1 as a single number (the wall time of
 # `handle_classify_job` under whole-pipeline contention) and bin/bench_model.jl
@@ -10,7 +10,6 @@
 #   classify        filesize + two 16-byte reads + a 32x1 forward pass
 #     read_features   open, read head, seek, read tail, scale to Float32
 #     Lux.apply       the network on a feature vector already in memory
-#   move_to         rename spool/<f> -> known/<f> or unknown/<f>
 #   enqueue         push a Job reference onto the downstream bounded queue
 #   logging         two @info lines ("classified file", "routed to ...")
 #
@@ -18,7 +17,7 @@
 # `handle_classify_job` end to end so the parts can be checked against the whole.
 # The two knobs that most change the answer get their own sweeps:
 #
-#   * Logger. The server runs `FlushLogger(ConsoleLogger(stderr))` — it formats
+#   * Logger. The server runs `FlushLogger(ConsoleLogger(stderr))`, which formats
 #     and flushes every message. Under redirect (a log file, journald) that is a
 #     syscall per line, two lines per file, on the hot path. We time the handler
 #     under a null logger, a formatting-but-discarding logger, and the real
@@ -26,7 +25,7 @@
 #     guess. This sweep is what demoted stage 1's per-file lines to `@debug`
 #     (see the note in src/worker.jl); the standalone `logging (...)` rows below
 #     still price a *formatted* line, i.e. what those lines cost when switched
-#     back on with `JULIA_DEBUG=FileServer`, while the handler rows show what the
+#     back on with `JULIA_DEBUG=DarkStruct`, while the handler rows show what the
 #     stage pays with them off.
 #   * Concurrency. Components that own a lock (the queue's condition, the
 #     logger's stream) don't scale, and the ranking at one worker need not be the
@@ -36,8 +35,8 @@
 # Usage:
 #   julia --project=. -t auto bin/bench_stage1.jl [options]
 #
-#   --files N        files per timed pass for consuming benchmarks (default: 2000)
+#   --files N        files per timed pass for whole-corpus benchmarks (default: 2000)
-#   --reps N         calls per timed pass for non-consuming benchmarks (default: 20000)
+#   --reps N         calls per timed pass for per-call benchmarks (default: 20000)
 #   --trials N       timed passes; the minimum is reported (default: 5)
 #   --size SPEC      corpus file size (default: 64k)
 #   --dir PATH       working directory for the corpus (default: a temp dir under data/)
@@ -49,14 +48,14 @@
 # Reported times are the *minimum* over trials: the floor is the signal and
 # everything above it is scheduler, page-cache and GC noise.
 
-using FileServer
+using DarkStruct
 using Lux
 using JSON3
 using Logging
 using Printf
 using Random
 
-const FS = FileServer
+const FS = DarkStruct
 
 # ---------------------------------------------------------------- option parsing
 
@@ -113,8 +112,8 @@ const SINK = Ref{Any}(nothing)
 
 Run `pass()` `trials` times and report the fastest, in nanoseconds per operation
 (`pass` returns the number of operations it performed). `prepare()` runs before
-each pass and is *not* timed — that is where a consuming benchmark puts the file
+each pass and is *not* timed: that is where a benchmark resets whatever its
-back where it started. `pass` comes first so callers can pass it as a `do` block.
+last pass consumed (today: the queues). `pass` comes first so callers can pass it as a `do` block.
 
 The first pass is thrown away: it pays Julia's JIT compilation, which on calls
 this small is orders of magnitude more than the thing being measured.
@@ -165,7 +164,7 @@ end
 """
 Write a file of exactly `size` random bytes, in bounded chunks.
 
-Content is random rather than zeros so the classifier sees a realistic input —
+Content is random rather than zeros so the classifier sees a realistic input,
 and so the filesystem can't cheat with a sparse file, which would make the tail
 `seek` unrepresentatively fast.
 """
@@ -186,7 +185,7 @@ end
     make_corpus(cfg, n, size, rng) -> Vector{Job}
 
 Create `n` spooled files and the `Job` references a stage-1 worker would dequeue
-for them — the exact input `handle_classify_job` sees.
+for them: the exact input `handle_classify_job` sees.
 """
 function make_corpus(cfg::FS.Config, n::Int, size::Int, rng)
     jobs = FS.Job[]
@@ -198,26 +197,11 @@ function make_corpus(cfg::FS.Config, n::Int, size::Int, rng)
     return jobs
 end
 
-"""
+# There is no `respool!` step any more. Stage 1 used to consume its input by
-    respool!(cfg, jobs)
+# renaming each file into known/ or unknown/, so every repeated pass had to put
-
+# the corpus back first. It now routes by enqueueing alone and leaves the file in
-Put every corpus file back in `spool/`, wherever the last pass left it (known/,
+# spool/, so the corpus is reusable as-is and the only per-pass reset is draining
-unknown/, or already home). This is the untimed `prepare` step for benchmarks
+# the queues.
-that consume their input by moving it.
-"""
-function respool!(cfg::FS.Config, jobs::Vector{FS.Job})
-    for job in jobs
-        isfile(job.path) && continue
-        for dir in (cfg.known_dir, cfg.unknown_dir, cfg.failed_dir)
-            candidate = joinpath(dir, basename(job.path))
-            if isfile(candidate)
-                mv(candidate, job.path; force = true)
-                break
-            end
-        end
-    end
-    return nothing
-end
 
 "Drain a queue without blocking, so the next pass starts from empty."
 function drain!(q::FS.ChannelQueue)
@@ -234,15 +218,15 @@ end
 
 Run `f` under one of the three loggers the cost of logging is bracketed by:
 
-  * `:null`    — `NullLogger`: the `@info` macro's own overhead, nothing else.
+  * `:null`    `NullLogger`: the `@info` macro's own overhead, nothing else.
-  * `:format`  — `ConsoleLogger` to `devnull`: message formatting and key/value
+  * `:format`  `ConsoleLogger` to `devnull`: message formatting and key/value
-                 interpolation, but no I/O.
+               interpolation, but no I/O.
-  * `:flush`   — `FlushLogger(ConsoleLogger(io))` to a real file: what
+  * `:flush`   `FlushLogger(ConsoleLogger(io))` to a real file: what
-                 `FileServer.run` installs, under the redirect it was written for.
+               `DarkStruct.run` installs, under the redirect it was written for.
-  * `:debug`   — the same, at `Debug` level: the stage's per-file lines are
+  * `:debug`   the same, at `Debug` level: the stage's per-file lines are
-                 `@debug`, so this is the equivalent of running the server with
+               `@debug`, so this is the equivalent of running the server with
-                 `JULIA_DEBUG=FileServer` and the only setting under which they
+               `JULIA_DEBUG=DarkStruct` and the only setting under which they
-                 are emitted at all.
+               are emitted at all.
 """
 function with_logger_named(f, which::Symbol, path::AbstractString)
     if which === :null
@@ -263,10 +247,10 @@ end
 """
     component_rows(cfg, clf, jobs, opts) -> Vector
 
-Time each piece of stage 1 on its own. Non-consuming pieces (`filesize`,
+Time each piece of stage 1 on its own. Per-call pieces (`filesize`,
 `read_features`, `Lux.apply`, `classify`, the log lines) run `reps` times over
-the corpus; consuming pieces (`move_to`, the full handler) run once per corpus
+the corpus; whole-corpus pieces (`enqueue_blocking!`, the full handler) run once
-file with an untimed reset between passes.
+per corpus file, with an untimed queue drain between passes.
 """
 function component_rows(cfg::FS.Config, clf::FS.Classifier, jobs::Vector{FS.Job}, opts)
     reps, trials = opts["reps"], opts["trials"]
@@ -317,14 +301,11 @@ function component_rows(cfg::FS.Config, clf::FS.Classifier, jobs::Vector{FS.Job}
             reps
         end))
 
-    # --- move_to: the rename out of spool/. Consuming: reset before each pass.
+    # Stage 1 used to rename each file into known/ or unknown/ before enqueueing
-    push!(rows, (; name = "move_to (rename)", part = "route",
+    # it, and that rename was measured here as its own line item. It is gone:
-        ns = best_of(() -> respool!(cfg, jobs); trials) do
+    # routing is the enqueue alone, and a file does not move until it is
-            @inbounds for job in jobs
+    # committed to a terminal sink (src/worker.jl header). So `route` below is
-                SINK[] = FS.move_to(cfg.unknown_dir, job)
+    # now just the queue handoff.
-            end
-            nfiles
-        end))
 
     # --- enqueue: lock, push, notify on an uncontended, non-full queue
     q = FS.ChannelQueue(nfiles + 1)
@@ -381,7 +362,7 @@ function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
                            (:flush, "handle_classify_job (flush→file)"),
                            (:debug, "handle_classify_job (JULIA_DEBUG)"))
         ns = with_logger_named(which, logfile) do
-            best_of(() -> (respool!(cfg, jobs); drain!(known); drain!(unknown)); trials) do
+            best_of(() -> (drain!(known); drain!(unknown)); trials) do
                 @inbounds for job in jobs
                     FS.handle_classify_job(job, cfg, 1, known, unknown, stats)
                 end
@@ -390,7 +371,7 @@ function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
         end
         push!(rows, (; name = label, part = "total", ns))
     end
-    respool!(cfg, jobs); drain!(known); drain!(unknown)
+    drain!(known); drain!(unknown)
     rm(logfile; force = true)
     return rows
 end
@@ -399,8 +380,8 @@ end
     thread_rows(cfg, jobs, opts) -> Vector
 
 Run the full handler across worker counts, under the server's real logger. A
-component that owns a lock — the queue's condition variable, the logger's
+component that owns a lock (the queue's condition variable, the logger's
-stream — stops scaling here even though it looked cheap single-threaded, so this
+stream) stops scaling here even though it looked cheap single-threaded, so this
 is where the single-thread ranking gets checked against the deployed one.
 """
 function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
@@ -419,9 +400,9 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
     base = 0.0
     for k in counts
         ns = with_logger_named(:flush, logfile) do
-            best_of(() -> (respool!(cfg, jobs); drain!(known); drain!(unknown)); trials) do
+            best_of(() -> (drain!(known); drain!(unknown)); trials) do
                 # Static split: each task takes a contiguous slice, so the only
-                # sharing between workers is the state the server also shares —
+                # sharing between workers is the state the server also shares:
                 # the classifier, the queues, the logger, the filesystem.
                 chunk = cld(nfiles, k)
                 @sync for t in 1:k
@@ -441,7 +422,7 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
         k == counts[1] && (base = r)
         push!(rows, (; workers = k, ns, files_per_sec = r, speedup = r / base))
     end
-    respool!(cfg, jobs); drain!(known); drain!(unknown)
+    drain!(known); drain!(unknown)
     rm(logfile; force = true)
     return rows
 end
@@ -481,12 +462,10 @@ function main(argv)
     owned = opts["dir"] === nothing
     cfg = FS.Config(
         spool_dir   = joinpath(root, "spool"),
-        known_dir   = joinpath(root, "known"),
-        unknown_dir = joinpath(root, "unknown"),
         failed_dir  = joinpath(root, "failed"),
         model_path  = modelpath,
     )
-    for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.failed_dir)
+    for d in (cfg.spool_dir, cfg.failed_dir)
         mkpath(d)
     end
 
@@ -511,7 +490,7 @@ function main(argv)
         # flushing logger at Info level, one worker. Percentages are shares of
         # that, so they are directly comparable and the parts can be checked
         # against the whole. The JULIA_DEBUG row is deliberately *not* the
-        # baseline — it is the opt-in configuration, and letting it set the scale
+        # baseline. It is the opt-in configuration, and letting it set the scale
         # would make every other component look free.
         total = only(r.ns for r in handlers if r.name == "handle_classify_job (flush→file)")
 
@@ -524,7 +503,7 @@ function main(argv)
         # Stage 1's own log lines are `@debug`, so what the deployed handler pays
         # for them is the disabled-macro cost, not a formatted line.
         accounted = sum(r.ns for r in comps if r.name in
-            ("classify (total)", "move_to (rename)", "enqueue_blocking!", "logging (NullLogger)"))
+            ("classify (total)", "enqueue_blocking!", "logging (NullLogger)"))
         println()
         @printf("accounted: %s of %s (%.0f%%); unaccounted overhead %s\n",
                 human_time(accounted), human_time(total), 100 * accounted / total,

bin/bench_stage2.jl

@@ -1,6 +1,6 @@
 #!/usr/bin/env julia
 #
-# bench_stage2.jl — take stage 2 apart and find the slowest component.
+# bench_stage2.jl: take stage 2 apart and find the slowest component.
 #
 # bin/bench.jl reports stage 2 as a single number (its throughput and worker
 # utilization under whole-pipeline contention). It doesn't say *which part* of
@@ -17,7 +17,7 @@
 #     JSON3.write         serialize the sidecar payload
 #     write + fsync       durably persist the sidecar bytes to a temp name
 #     mv + fsync_dir      commit the sidecar, then persist the rename itself
-#     move_to             rename known/<f> -> done/<f>, the commit point
+#     move_to             rename spool/<f> -> done/<f>, the commit point
 #   logging             one @info line ("enriched")
 #
 # This script times each of those in isolation, then times the real
@@ -27,13 +27,13 @@
 #
 #   * The corpus must be real files. exiftool's cost depends on what it finds;
 #     random bytes exit early and would understate the stage by a lot. The
-#     default corpus is `data/done` — files that already went through stage 2 on
+#     default corpus is `data/done`, files that already went through stage 2 on
-#     this machine — copied back into a scratch known/ dir.
+#     this machine, copied back into a scratch spool/ dir.
 #   * Two rows price the *alternatives* to one-fork-per-file, because if the
 #     fork dominates then the only fixes are to stop paying it per file:
 #     `exiftool (batched Nx)` runs the whole corpus through one process, and
 #     `exiftool (-stay_open)` keeps a single process alive and feeds it one file
-#     at a time over a pipe — the shape a streaming pipeline could actually use.
+#     at a time over a pipe: the shape a streaming pipeline could actually use.
 #     Both are measured, not assumed.
 #   * `run_with_timeout` gets its own row *next to* a bare `Base.run` of the same
 #     command. The difference is what the watchdog costs, and its polling loop
@@ -48,7 +48,7 @@
 #   julia --project=. -t auto bin/bench_stage2.jl [options]
 #
 #   --files N        corpus files per timed pass (default: 48). The concurrency
-#                    sweep wants more than the component rows do — with a single
+#                    sweep wants more than the component rows do. With a single
 #                    2 s file in the corpus, 48 files can't show more than ~4x no
 #                    matter how many workers run, so pass --files 150 when the
 #                    question is scaling.
@@ -65,13 +65,13 @@
 # Reported times are the *minimum* over trials: the floor is the signal and
 # everything above it is scheduler, page-cache and GC noise.
 
-using FileServer
+using DarkStruct
 using JSON3
 using Logging
 using Printf
 using Random
 
-const FS = FileServer
+const FS = DarkStruct
 
 # ---------------------------------------------------------------- option parsing
 
@@ -121,7 +121,7 @@ const SINK = Ref{Any}(nothing)
 
 Run `pass()` `trials` times and report the fastest, in nanoseconds per operation
 (`pass` returns the number of operations it performed). `prepare()` runs before
-each pass and is *not* timed — that is where a consuming benchmark puts the file
+each pass and is *not* timed: that is where a consuming benchmark puts the file
 back where it started. `pass` comes first so callers can pass it as a `do` block.
 
 The first pass is thrown away: it pays Julia's JIT compilation, which on calls
@@ -173,13 +173,13 @@ end
 """
     make_corpus(cfg, corpus_dir, n) -> Vector{Job}
 
-Copy up to `n` real files from `corpus_dir` into `known/` and build the `Job`
+Copy up to `n` real files from `corpus_dir` into `spool/` and build the `Job`
-references a stage-2 worker would dequeue for them — the exact input
+references a stage-2 worker would dequeue for them: the exact input
 `handle_known_job` sees.
 
 Real files, not generated ones: exiftool's cost is a function of what it can
 parse, and a file of random bytes bails out early enough to understate the stage
-by an order of magnitude. `.meta.json` sidecars are skipped — they are stage-2
+by an order of magnitude. `.meta.json` sidecars are skipped, since they are stage-2
 *output*, and enriching them would measure the wrong population.
 """
 function make_corpus(cfg::FS.Config, corpus_dir::AbstractString, n::Int)
@@ -198,21 +198,23 @@ function make_corpus(cfg::FS.Config, corpus_dir::AbstractString, n::Int)
         # corpus the file came from.
         id, spooled = FS.spool_path(cfg, @sprintf("s2-%04d-%s", i, basename(name)))
         cp(src, spooled; force = true)
-        dest = joinpath(cfg.known_dir, basename(spooled))
+        # Staged in spool/, not a known/ dir: stage 1 routes by enqueueing and
-        mv(spooled, dest; force = true)
+        # leaves the bytes where intake put them, so this is the exact on-disk
-        push!(jobs, FS.Job(id, basename(name), dest, filesize(dest), time()))
+        # state a stage-2 worker dequeues into (src/worker.jl header).
+        push!(jobs, FS.Job(id, basename(name), spooled, filesize(spooled), time()))
     end
     return jobs
 end
 
 """
-    reknown!(cfg, jobs)
+    respool!(cfg, jobs)
 
-Put every corpus file back in `known/`, wherever the last pass left it (done/ or
+Put every corpus file back in `spool/`, wherever the last pass left it (done/ or
 already home), and delete any sidecar it produced. This is the untimed `prepare`
-step for benchmarks that consume their input by committing it.
+step for benchmarks that consume their input by committing it. Stage 2 still
+moves, because its move is the terminal commit, not an inter-stage hop.
 """
-function reknown!(cfg::FS.Config, jobs::Vector{FS.Job})
+function respool!(cfg::FS.Config, jobs::Vector{FS.Job})
     for job in jobs
         base = basename(job.path)
         for dir in (cfg.done_dir, cfg.failed_dir)
@@ -239,13 +241,13 @@ end
 
 Run `f` under one of the loggers the cost of logging is bracketed by:
 
-  * `:null`    — `NullLogger`: the `@info` macro's own overhead, nothing else.
+  * `:null`    `NullLogger`: the `@info` macro's own overhead, nothing else.
-  * `:format`  — `ConsoleLogger` to `devnull`: message formatting and key/value
+  * `:format`  `ConsoleLogger` to `devnull`: message formatting and key/value
-                 interpolation, but no I/O.
+               interpolation, but no I/O.
-  * `:flush`   — `FlushLogger(ConsoleLogger(io))` to a real file: what
+  * `:flush`   `FlushLogger(ConsoleLogger(io))` to a real file: what
-                 `FileServer.run` installs, under the redirect it was written
+               `DarkStruct.run` installs, under the redirect it was written
-                 for. Stage 2's per-file line is `@info`, not `@debug`, so this
+               for. Stage 2's per-file line is `@info`, not `@debug`, so this
-                 row is what the deployed server actually pays.
+               row is what the deployed server actually pays.
 """
 function with_logger_named(f, which::Symbol, path::AbstractString)
     if which === :null
@@ -284,7 +286,7 @@ end
 
 Run the whole corpus through *one* `exiftool` process and divide by the file
 count. This is the floor for "what does exiftool cost if you stop paying the
-interpreter startup per file" — the fork, the Perl boot and the module loads are
+interpreter startup per file": the fork, the Perl boot and the module loads are
 paid once for the batch instead of once per file.
 """
 function batched_ns(paths::Vector{String}, trials::Int)
@@ -299,8 +301,8 @@ end
 
 Feed files one at a time to a single long-lived `exiftool -stay_open True -@ -`
 process over a pipe, reading its `{ready}` sentinel after each. Unlike the
-batched row this preserves the pipeline's actual shape — one file in, one result
+batched row this preserves the pipeline's actual shape (one file in, one result
-out, arriving whenever it arrives — so it prices the realistic fix rather than
+out, arriving whenever it arrives) so it prices the realistic fix rather than
 an unrealistic one.
 """
 function stay_open_ns(paths::Vector{String}, trials::Int)
@@ -437,7 +439,7 @@ function component_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
     # One pass over the real sidecar population, not `reps` of them. Two reasons,
     # and the first is a correctness trap: thousands of back-to-back fsyncs
     # saturate the device's write cache and each one starts waiting on the
-    # queue, which reported this row at 16 ms/file — eight times the whole
+    # queue, which reported this row at 16 ms/file: eight times the whole
     # `commit_enriched!` that contains it. The real stage fsyncs once per file
     # with ~160 ms of exiftool between, and never queues that way. Second, real
     # sidecars vary hugely in size (a zip's raw dump dwarfs a jpeg's), so the
@@ -463,8 +465,8 @@ function component_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
         nio
     end)
 
-    # --- move_to: the rename known/<f> -> done/<f>. Consuming: reset each pass.
+    # --- move_to: the rename spool/<f> -> done/<f>. Consuming: reset each pass.
-    add!("move_to (rename)", "commit", best_of(() -> reknown!(cfg, jobs); trials) do
+    add!("move_to (rename)", "commit", best_of(() -> respool!(cfg, jobs); trials) do
         @inbounds for job in jobs
             SINK[] = FS.move_to(cfg.done_dir, job)
         end
@@ -474,17 +476,17 @@ function component_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
     # --- commit_enriched!: the whole sidecar-first commit, extraction excluded.
     committable = [j for (j, _) in good]
     add!("commit_enriched! (total)", "commit",
-        best_of(() -> reknown!(cfg, jobs); trials) do
+        best_of(() -> respool!(cfg, jobs); trials) do
             @inbounds for (k, job) in enumerate(committable)
                 SINK[] = FS.commit_enriched!(cfg.done_dir, job, metas[k])
             end
             length(committable)
         end)
-    reknown!(cfg, jobs)
+    respool!(cfg, jobs)
 
     # --- the one @info line, under each logger.
     job1, meta1 = good[1][1], metas[1]
-    logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
+    logfile = joinpath(dirname(cfg.spool_dir), "bench_stage2.log")
     for (which, label) in ((:null, "logging (NullLogger)"),
                            (:format, "logging (format only)"),
                            (:flush, "logging (flush→file)"))
@@ -513,13 +515,13 @@ running server actually pays.
 function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
     trials = opts["trials"]
     nfiles = length(jobs)
-    logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
+    logfile = joinpath(dirname(cfg.spool_dir), "bench_stage2.log")
     rows = []
     for (which, label) in ((:null, "handle_known_job (NullLogger)"),
                            (:format, "handle_known_job (format only)"),
                            (:flush, "handle_known_job (flush→file)"))
         ns = with_logger_named(which, logfile) do
-            best_of(() -> reknown!(cfg, jobs); trials) do
+            best_of(() -> respool!(cfg, jobs); trials) do
                 @inbounds for job in jobs
                     FS.handle_known_job(job, cfg, 1)
                 end
@@ -528,7 +530,7 @@ function handler_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
         end
         push!(rows, (; name = label, part = "total", ns))
     end
-    reknown!(cfg, jobs)
+    respool!(cfg, jobs)
     rm(logfile; force = true)
     return rows
 end
@@ -544,7 +546,7 @@ machine's cores, not about Julia.
 Workers pull from a shared atomic counter rather than taking a contiguous slice.
 That matches the server (its pool pulls from one queue), and it matters here in a
 way it doesn't for stage 1: per-file exiftool time spans two orders of magnitude
-on a real corpus — a single 2 s archive among 48 files — so a static split leaves
+on a real corpus (a single 2 s archive among 48 files), so a static split leaves
 whichever worker drew it running alone while the rest idle, and the sweep would
 report a scaling ceiling that is really just load imbalance.
 """
@@ -556,13 +558,13 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
         [parse(Int, s) for s in split(String(opts["threads"]), ",")]
     counts = sort(unique(filter(k -> 1 <= k <= Threads.nthreads(), counts)))
 
-    logfile = joinpath(dirname(cfg.known_dir), "bench_stage2.log")
+    logfile = joinpath(dirname(cfg.spool_dir), "bench_stage2.log")
     rows = []
     base = 0.0
     for k in counts
         ns = with_logger_named(:flush, logfile) do
             next = Threads.Atomic{Int}(1)
-            best_of(() -> (reknown!(cfg, jobs); next[] = 1); trials) do
+            best_of(() -> (respool!(cfg, jobs); next[] = 1); trials) do
                 # Shared counter, not a contiguous slice: every worker takes the
                 # next unclaimed file the moment it frees up, exactly as the
                 # server's pool takes the next job off the known queue.
@@ -582,7 +584,7 @@ function thread_rows(cfg::FS.Config, jobs::Vector{FS.Job}, opts)
         k == counts[1] && (base = r)
         push!(rows, (; workers = k, ns, files_per_sec = r, speedup = r / base))
     end
-    reknown!(cfg, jobs)
+    respool!(cfg, jobs)
     rm(logfile; force = true)
     return rows
 end
@@ -622,12 +624,11 @@ function main(argv)
     owned = opts["dir"] === nothing
     cfg = FS.Config(
         spool_dir        = joinpath(root, "spool"),
-        known_dir        = joinpath(root, "known"),
         done_dir         = joinpath(root, "done"),
         failed_dir       = joinpath(root, "failed"),
         exiftool_timeout = opts["timeout"],
     )
-    for d in (cfg.spool_dir, cfg.known_dir, cfg.done_dir, cfg.failed_dir)
+    for d in (cfg.spool_dir, cfg.done_dir, cfg.failed_dir)
         mkpath(d)
     end
 

bin/cluster_calibrate.jl

@@ -5,7 +5,7 @@
 # recovered partition against magic-collapsed ground truth with ARI / V-measure,
 # grid-tunes (α, β, bg_mass, n), and cross-checks the winning config against a
 # model-free NCD (gzip) baseline (§8). The settings printed here are the ones the
-# machine rediscovers known formats at — copy the winner into config.jl.
+# machine rediscovers known formats at; copy the winner into config.jl.
 #
 #   julia --project=. bin/cluster_calibrate.jl [training_set_dir]
 #
@@ -23,7 +23,7 @@ include(joinpath(@__DIR__, "..", "src", "cluster.jl"))
 
 The magic-collapsed format class of a file, read from its actual bytes (so
 docx≡zip and the whole ELF family merge, exactly the answer we want the
-clustering to reproduce). `tar` is detected by the `ustar` magic at offset 257 —
+clustering to reproduce). `tar` is detected by the `ustar` magic at offset 257,
 outside the model's front window, so tars are the accepted blind spot that
 scatters to background.
 """
@@ -53,7 +53,7 @@ function gz_size(bytes::Vector{UInt8})
     return length(take!(out))
 end
 
-"NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)) — 0 = identical, ~1 = unrelated."
+"NCD(x,y) = (C(xy) - min(C(x),C(y))) / max(C(x),C(y)); 0 = identical, ~1 = unrelated."
 function ncd(xb, yb, cx, cy)
     cxy = gz_size(vcat(xb, yb))
     return (cxy - min(cx, cy)) / max(cx, cy)
@@ -62,7 +62,7 @@ end
 """
     ncd_1nn_purity(paths, truth; head_bytes) -> Float64
 
-Fraction of files whose NCD-nearest neighbour shares its true label — a cheap,
+Fraction of files whose NCD-nearest neighbour shares its true label: a cheap,
 O(N²) sanity read on how well raw gzip-similarity alone separates formats on the
 same input. The Bayesian clusters should broadly agree; a big gap is a red flag
 (DESIGN §10.3). Uses each file's first `head_bytes` so the giant files don't
@@ -173,7 +173,7 @@ function main()
     end
 
     # Rank by ARI-excluding-tar (tar is the accepted blind spot; scoring it would
-    # penalise the correct answer of scattering tars to background — DESIGN §7.2).
+    # penalise the correct answer of scattering tars to background; DESIGN §7.2).
     sort!(results; by=r -> r.e.ari_notar, rev=true)
     best = results[1]
     println()
@@ -184,7 +184,7 @@ function main()
 
     # Per-cluster composition of the winning partition, and promotion nominations.
     pred = best.e.result.assignments
-    println("\nwinning partition — cluster composition (truth breakdown):")
+    println("\nwinning partition, cluster composition (truth breakdown):")
     for (id, c) in sort(collect(best.e.result.clusters); by=x -> -x[2].members)
         members = [truth[i] for i in eachindex(pred) if pred[i] == id]
         comp = sort([(l, count(==(l), members)) for l in unique(members)]; by=x -> -x[2])
@@ -211,7 +211,7 @@ function main()
     bay_pur = cluster_1nn_purity(rsub.assignments, subtruth)
     @printf("  subsample=%d  NCD 1-NN label purity=%.3f   Bayesian same-cluster purity=%.3f\n",
             subn, ncd_pur, bay_pur)
-    println("  (both high ⇒ header-byte signal agrees with model-free gzip similarity — DESIGN §10.3)")
+    println("  (both high ⇒ header-byte signal agrees with model-free gzip similarity; DESIGN §10.3)")
 end
 
 main()

bin/cluster_sweep.jl

@@ -3,17 +3,17 @@
 # Stage-5 phase-B runner (model/DESIGN_clustering.md §9): the single-owner,
 # periodic/cron process that sweeps `binary/`, folds new files into the durable
 # format catalog by sequential CRP-predictive assignment, and (re)writes
-# promotion nominations. Run it single-threaded on a schedule — it is the ONLY
+# promotion nominations. Run it single-threaded on a schedule: it is the ONLY
 # writer of the catalog, so no locking is needed.
 #
 #   julia --project=. bin/cluster_sweep.jl            # incremental live sweep
 #   julia --project=. bin/cluster_sweep.jl --compact  # offline Gibbs (seed / recompact)
 #
 # On a fresh catalog (nothing processed yet) the incremental sweep would send
-# every file to background — there are no clusters to match — so the first run
+# every file to background (there are no clusters to match), so the first run
 # auto-promotes to a compaction pass to seed the catalog. Configure via the
 # FS_CLUSTER_* / FS_NOMINATED_DIR env vars (see src/config.jl).
 
-using FileServer
+using DarkStruct
 
-FileServer.cluster_sweep_cli(ARGS)
+DarkStruct.cluster_sweep_cli(ARGS)

bin/send_dir.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# send_dir.sh — upload every file in a directory to a running FileServer.
+# send_dir.sh — upload every file in a directory to a running DarkStruct.
 #
 # Each file is POSTed to /upload as multipart/form-data. By default one file per
 # request (so you get per-file feedback); use -b to batch all files into a

bin/server.jl

@@ -1,4 +1,4 @@
 #!/usr/bin/env julia
 # Entry point. Run with:  julia --project -t auto bin/server.jl
-using FileServer
+using DarkStruct
-FileServer.run()
+DarkStruct.run()

docker-compose.rabbitmq.yml

@@ -0,0 +1,55 @@
+# Overlay that swaps the in-process queues for durable RabbitMQ ones.
+#
+#   docker compose up                                                  # channels
+#   docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up
+#
+# The base file is left alone deliberately: `docker compose up` still brings up
+# exactly what it always did, so neither backend is the awkward one to run.
+#
+# What the broker changes: a job stays unacked until its handler commits, so a
+# restart resumes each file at the stage it had reached instead of re-driving
+# everything in spool/ from stage 1. See "Queue backends" in the README for the
+# guarantees, and for the deliberate holes.
+
+services:
+  rabbitmq:
+    image: rabbitmq:4-management
+    environment:
+      # The built-in guest/guest only authenticates over loopback, so it cannot
+      # work across the compose network. Override these for anything real; they
+      # are here so the stack comes up with one command, not as a credential.
+      RABBITMQ_DEFAULT_USER: "darkstruct"
+      RABBITMQ_DEFAULT_PASS: "darkstruct"
+    ports:
+      # 5672 is published so you can run the server on the host
+      # (`julia --project=. -t auto bin/server.jl`) against this broker; 15672 is
+      # the management UI, which is most of the reason to run a broker you can
+      # look at. Neither needs publishing for the composed server to work.
+      - "5672:5672"
+      - "15672:15672"
+    volumes:
+      # Durable queues and persistent messages are only as durable as what they
+      # are written to: without this, `docker compose down` discards the backlog
+      # the acks exist to protect.
+      - rabbitmq-data:/var/lib/rabbitmq
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+    restart: unless-stopped
+
+  darkstruct:
+    environment:
+      FS_QUEUE_BACKEND: "rabbitmq"
+      FS_AMQP_URL: "amqp://darkstruct:darkstruct@rabbitmq:5672/"
+    depends_on:
+      rabbitmq:
+        # The server retries a broker that isn't up yet (AMQP_CONNECT_RETRY_SECONDS),
+        # so this gate is belt-and-braces rather than load-bearing — but it keeps
+        # a cold start quiet instead of a screenful of retry lines.
+        condition: service_healthy
+
+volumes:
+  rabbitmq-data:

docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+  darkstruct:
+    build:
+      context: .
+      args:
+        # false drops the Ruby toolchain and github-linguist (~160MB smaller).
+        # Stage-4 text files then get natural-language enrichment but no
+        # programming language; the server warns at startup and carries on.
+        WITH_LINGUIST: "true"
+    image: darkstruct:latest
+    ports:
+      - "8080:8080"
+    volumes:
+      # Every pipeline stage (spool, done, text_done, binary, failed, nominated)
+      # lives under /data, so one volume holds the whole in-flight working set.
+      - fs-data:/data
+    environment:
+      # The four pools each default to nthreads(), which oversubscribes the
+      # threads Julia actually has. Size them for the host instead.
+      FS_WORKERS: "4"
+      FS_KNOWN_WORKERS: "4"
+      FS_UNKNOWN_WORKERS: "2"
+      FS_TEXT_WORKERS: "2"
+    restart: unless-stopped
+
+volumes:
+  fs-data:

model/DESIGN_clustering.md

@@ -1,20 +1,20 @@
 # Stage-5: Unknown-format discovery by Bayesian header clustering
 
-Status: **phases A and B implemented and calibrated** (`src/cluster.jl` +
+Status: phases A and B implemented and calibrated (`src/cluster.jl` +
 `src/catalog.jl`, `bin/cluster_calibrate.jl` + `bin/cluster_sweep.jl`, tests in
 `test/runtests.jl`). Phase A (offline Gibbs) is calibrated; phase B's durable
 single-owner catalog, incremental sweep, and nomination writer are now built on
-top of the `assign_file` scoring core. Product
+top of the `assign_file` scoring core. This is the product of a design
-of a design interview; captures the decisions and — as important — the
+interview. It captures the decisions and, just as importantly, the assumptions
-assumptions we *rejected* so they don't get silently reintroduced. §11 records
+we *rejected*, so they don't get silently reintroduced. §11 records
 what building it actually taught us, including three assumptions in this document
 that the data corrected.
 
 ## 1. Goal
 
-Discover **recurring new file formats** hiding in the `binary/` bucket (the
+Discover recurring new file formats hiding in the `binary/` bucket (the
 `:unknown` sink from `classify.jl` → stage-3 triage). A genuinely novel format is
-a plausible proxy for a genuinely novel producing application, but we do **not**
+a plausible proxy for a genuinely novel producing application, but we do not
 try to identify producers directly (see §3). The output is a **growing catalog of
 discovered formats**, each with a magic-byte signature that can be promoted into
 the classifier's fast path.
@@ -23,42 +23,42 @@ Task shape (settled): **unsupervised clustering with an unknown number of
 clusters.** Not pairwise "same producer" scoring, not classification against a
 fixed label set.
 
-## 2. Two phases — build (A) then run (B)
+## 2. Two phases: build (A) then run (B)
 
-**(A) Batch, offline — the science.** Cluster the accumulated pile from scratch.
+**(A) Batch, offline, the science.** Cluster the accumulated pile from scratch.
 Its job is *not* to be the catalog; it is to (i) prove the header-byte signal
 actually separates formats, cross-checked against an NCD baseline (§8), and
-(ii) **calibrate hyperparameters** against known formats (§7). Ship this first —
+(ii) calibrate hyperparameters against known formats (§7). Ship this first,
-it de-risks (B). If (A)'s clusters are garbage, (B)'s machinery is wasted.
+because it de-risks (B). If (A)'s clusters are garbage, (B)'s machinery is wasted.
 
-**(B) Online, live — the catalog.** The target deliverable. A persistent catalog
+**(B) Online, live, the catalog.** The target deliverable. A persistent catalog
-where each discovered format has a **durable, frozen ID** and stored sufficient
+where each discovered format has a durable, frozen ID and stored sufficient
 statistics. New unknown files are scored against existing clusters; only
 genuinely novel ones spawn a new entry. Clusters that accumulate enough evidence
-are **nominated for promotion** into the classifier (§6).
+are nominated for promotion into the classifier (§6).
 
 ## 3. What we are and are NOT clustering
 
-We cluster by **file format**, not by producer. The first-*n* header bytes are
+We cluster by file format, not by producer. The first-*n* header bytes are
 format-mandated and producer-invariant: every valid PNG shares the same magic
 regardless of which program wrote it; a PDF's producer string lives deep inside
 the file, not in the header. Producer identity, where recoverable at all, is
 `exiftool`'s job (stage 2), not this stage's.
 
-Corollary already visible in `../training_set`: extension labels are **not**
+Corollary already visible in `../training_set`: extension labels are not
 header-format labels. `docx` *is* a PK zip; `so`/`o`/`elf`/`out` are all ELF.
-Merging those is **correct**, not error (see §7).
+Merging those is correct, not error (see §7).
 
 ## 4. Model: DP mixture of per-position categoricals
 
-A cluster is a **product of independent per-position categorical distributions**
+A cluster is a product of independent per-position categorical distributions
 over the first *n* header bytes. Position *i* carries a distribution `θᵢ` over a
 **257-symbol alphabet**: byte values `0–255`, plus symbol `256 = "past EOF"`.
 
 - Invariant positions (magic bytes) learn a spiked `θᵢ`; variable positions
   (lengths, timestamps) learn a flat one. A cluster's signature = the vector of
   modal symbols + per-position peakedness. That signature **is a magic-number
-  template** — this is the entire reason for the categorical choice.
+  template**, which is the entire reason for the categorical choice.
 - `257` alphabet handles short files honestly: a format that is always 20 bytes
   produces a spiked "past-EOF" at positions 20–31, which is real, discriminative
   signal. No zero-padding (would collide `0x00` padding with real `0x00` bytes).
@@ -69,13 +69,13 @@ process (CRP)** over cluster assignments → unknown *k* falls out natively.
 **Why categorical, not Euclidean.** Bytes are categorical, not ordinal: `0x89`
 and `0x88` are not "close," `0x00` and `0xFF` are not "far." k-means / Gaussian
 mixtures over scaled bytes assert a metric that does not exist in header space.
-**Do not reuse `model.jl`'s `[0,1]` byte scaling here** — that scaling is correct
+**Do not reuse `model.jl`'s `[0,1]` byte scaling here.** That scaling is correct
 for the Lux net and wrong for this model. We need the raw `0–255` byte as a
 categorical index.
 
 ### 4a. Background component (high-entropy handling)
 
-Add a fixed, **non-adaptive uniform component** (each position uniform over 257)
+Add a fixed, non-adaptive uniform component (each position uniform over 257)
 as the "junk drawer." Compressed/encrypted/structureless blobs are ~uniform after
 any magic and would otherwise either (i) mint a singleton per file or (ii)
 collapse into one flat cluster that then matches everything. The background
@@ -84,73 +84,73 @@ absorbs them cleanly.
 Two populations, to be precise:
 - **Structured prefix + random tail** (gzip `1f 8b`, PK zip, zstd, most encrypted
   *containers*): peaked at positions 0–3, flat after. These form **real clusters
-  for free** — genuine discoveries, no special handling.
+  for free**: genuine discoveries, no special handling.
 - **Uniform from byte 0** (raw encrypted streams, key material): nothing in the
   header to cluster on → absorbed by background.
 
-The background is **never promotable**. But it is **not a silent sink**: its
+The background is never promotable. But it is not a silent sink: its
 size / growth / entropy histogram is surfaced as a first-class signal ("12% of
 this week's unknowns are structureless"). If sub-clustering the structureless
 residue ever matters, that needs a *different* feature (byte histogram / entropy),
-a separate v3 model — header bytes genuinely cannot do it.
+a separate v3 model. Header bytes genuinely cannot do it.
 
 ### 4b. Feature window
 
 **Front-only, `n = 32`** (config knob; try 64 if under-resolved). Magic lives at
-offset 0. Tail window **deferred to v2** — a minority of formats have trailers
+offset 0. Tail window deferred to v2: a minority of formats have trailers
 (ZIP EOCD, ID3v1, PDF `%%EOF`); add as an independent *second block* of positions
 only if real trailer-formats show up in the residue.
 
-**Known blind spot: tar.** `ustar` magic is at **offset 257**, outside the
+**Known blind spot: tar.** `ustar` magic is at offset 257, outside the window,
-window, so all 100 training tars scatter to background. Accepted for v1 — tar is
+so all 100 training tars scatter to background. Accepted for v1, since tar is
-already a *known* format, so discovery doesn't need it. General lesson: a minority
+already a *known* format and discovery doesn't need it. General lesson: a
-of formats put magic at a fixed deeper offset; the fix (if ever needed) is a
+minority of formats put magic at a fixed deeper offset; the fix (if ever needed)
-**sparse probe window** at that offset (e.g. bytes 257–262 as a third block), not
+is a sparse probe window at that offset (e.g. bytes 257–262 as a third block),
-densely modeling 257 front bytes — that would 8× every cluster's `n×257`
+rather than densely modeling 257 front bytes, which would 8× every cluster's
-sufficient-stat table to catch one format.
+`n×257` sufficient-stat table to catch one format.
 
 ## 5. Inference: different mode per phase (resolves the Bayesian-vs-catalog tension)
 
 A sampler yields a *posterior over partitions*; a catalog needs *one partition
-with durable IDs*. Two MCMC gotchas: **label switching** (cluster #3 is not a
+with durable IDs*. Two MCMC gotchas: label switching (cluster #3 is not a
-stable identity across iterations/runs) and **distribution-not-answer** (1000
+stable identity across iterations/runs) and distribution-not-answer (1000
 partitions, not one). We sidestep both by using two inference modes:
 
-- **Phase (A), offline:** full **collapsed Gibbs** sampler over the
+- **Phase (A), offline:** full collapsed Gibbs sampler over the
   Dirichlet-Categorical (conjugacy → ~100 lines, no continuous approximation,
   unknown *k* native). Used to validate signal, tune `α` + Dirichlet strength,
-  and seed the initial catalog (summarize to a point partition **once**, via a
+  and seed the initial catalog (summarize to a point partition once, via a
-  VI/Binder loss over the posterior similarity matrix — tolerated because it is
+  VI/Binder loss over the posterior similarity matrix, tolerated because it is
   offline, never in the hot path).
-- **Phase (B), live:** **deterministic sequential CRP-predictive assignment.**
+- **Phase (B), live:** deterministic sequential CRP-predictive assignment.
   Each catalog cluster stores per-position 257-count vectors (sufficient stats).
   A new file's CRP predictive probability of joining each existing cluster vs.
   the background vs. spawning a new cluster is computed; assign to the argmax.
   A new cluster is minted only if the new-cluster evidence beats the background
-  by a margin. **IDs are frozen at birth → no label switching.** This is exactly
+  by a margin. IDs are frozen at birth → no label switching. This is exactly
-  the Gibbs predictive rule with existing assignments held fixed — same math, not
+  the Gibbs predictive rule with existing assignments held fixed: the same math,
-  an ad-hoc hack.
+  not an ad-hoc hack.
 - **Periodic compaction, offline:** re-run Gibbs seeded from the current catalog
   to merge drifted clusters / split bloated ones.
 
 ## 6. Promotion (closing the loop to the classifier)
 
 **Layered known-check at ingest** becomes:
-1. Match against **promoted signatures** (exact, fast) — runs *before* the net.
+1. Match against promoted signatures (exact, fast), which runs *before* the net.
 2. Else the Lux `:known` / `:unknown` classifier.
 3. Else route to `binary/` for this stage.
 
 **Promotion = append a magic-byte signature to a registry.** A cluster's spiked
 positions (posterior max-prob `> ~0.9`) become required bytes; flat positions
-become wildcards — a libmagic-style signature. This is a **data change, not a
+become wildcards, giving a libmagic-style signature. This is a **data change, not a
 retrain**; interpretable, auditable, reversible. Retraining the Lux net is a
 separate, *optional periodic* activity using accumulated signature-labeled files,
 never the promotion mechanism itself.
 
-**Nominate automatically, activate by hand.** A cluster crossing thresholds —
+**Nominate automatically, activate by hand.** A cluster crossing the thresholds
-`≥ N` members (start `N ≈ 20–50`, loose dial since a human is the backstop) **and**
+(`≥ N` members, starting at `N ≈ 20–50`, a loose dial since a human is the
-`≥ ~3` magic positions **and** not the background — is written to a `nominated/`
+backstop; `≥ ~3` magic positions; and not the background) is written to a
-registry with its signature, member count, and example files. A human glance
+`nominated/` registry with its signature, member count, and example files. A human glance
 promotes it into the active set. Human gate guards the one hard-to-reverse action
 (redefining "known"); everything upstream stays automatic.
 
@@ -159,52 +159,52 @@ promotes it into the active set. Human gate guards the one hard-to-reverse actio
 Do not pick priors blind. We have ground truth: `../training_set` (100 each of
 tgz/tar/pdf/docx, 98 zip, 93 jpg, ELF family) and the `data/done` corpus.
 
-1. Run **labeled known files** through the exact clustering pipeline.
+1. Run labeled known files through the exact clustering pipeline.
-2. Ground truth = **magic-collapsed classes**, *not* extensions:
+2. Ground truth = magic-collapsed classes, *not* extensions:
    `{gzip (tgz), PKzip (docx≡zip), ELF (so/o/elf/out/x86_64), JPEG, PDF, tar}`.
-   Merging docx+zip and the ELF family is the **correct** answer — scoring
+   Merging docx+zip and the ELF family is the correct answer; scoring
    against raw extensions would penalize correctness and mistune `α`.
-3. Measure recovered-vs-truth agreement with **Adjusted Rand Index / V-measure**.
+3. Measure recovered-vs-truth agreement with Adjusted Rand Index / V-measure.
-4. **Grid-tune `α` and the Dirichlet pseudocount to maximize agreement** — the
+4. **Grid-tune `α` and the Dirichlet pseudocount to maximize agreement**, which
-   settings at which the machine rediscovers formats we already know.
+   finds the settings at which the machine rediscovers formats we already know.
 5. Freeze, deploy on the `:unknown` pile.
 
-Splitting docx from zip is a **later tier**: the discriminating info
+Splitting docx from zip is a later tier: the discriminating info
 (central-directory filenames like `word/document.xml`) sits at a *variable
-offset*, not a fixed position — a different feature problem, deferred.
+offset*, not a fixed position. That is a different feature problem, deferred.
 
 ## 8. Julia package surface
 
-- **Hand-rolled collapsed Gibbs** over Dirichlet-Categorical — recommended. The
+- **Hand-rolled collapsed Gibbs** over Dirichlet-Categorical. Recommended: the
   conjugacy makes it short/fast; we own the online + promotion logic; no library
   impedance. `Distributions.jl` for `Dirichlet`/`Categorical` primitives.
-- **`CodecZlib`** for the **NCD (Normalized Compression Distance)** baseline —
+- **`CodecZlib`** for the NCD (Normalized Compression Distance) baseline:
   model-free gzip-similarity clustering. Excellent at format grouping and a great
   step-(A) sanity check, but O(N²), non-generative (no signature, no online
-  scoring, no promotion) → **baseline only, cannot be the catalog.**
+  scoring, no promotion) → baseline only, cannot be the catalog.
-- **`Clustering.jl`** — `randindex` / `vmeasure` for the §7 calibration metric,
+- **`Clustering.jl`** for `randindex` / `vmeasure`, the §7 calibration metric,
-  plus a throwaway k-modes-ish baseline. **Not** the real model (its k-means is
+  plus a throwaway k-modes-ish baseline. Not the real model (its k-means is
   the Euclidean trap of §4).
-- **`Turing.jl`** — considered and rejected: discrete assignment latents + DP are
+- **`Turing.jl`**, considered and rejected: discrete assignment latents + DP are
   awkward, and we'd still hand-roll the online path. Overkill.
 
 ## 9. Architecture: single-owner batch stage, NOT inline inference
 
 The classifier is stateless, immutable, shared read-only across worker threads
-(see `classify.jl`). **The catalog is the opposite: mutable, learned, shared** —
+(see `classify.jl`). The catalog is the opposite: mutable, learned, shared, since
-every assigned file updates a cluster's counts. It therefore must **not** copy the
+every assigned file updates a cluster's counts. It therefore must not copy the
 classifier's concurrency model (concurrent workers → lock contention, torn reads
 of sufficient stats, CRP assignment against stale mass).
 
-Because **promotion is human-gated, nothing here is latency-sensitive.** So:
+Because promotion is human-gated, nothing here is latency-sensitive. So:
 
-- Workers stay stateless — they deposit `:unknown` files into `binary/` exactly as
+- Workers stay stateless. They deposit `:unknown` files into `binary/` exactly as
-  today. **No catalog access on the hot path.**
+  today. No catalog access on the hot path.
-- A **separate stage-5 process** (periodic / cron, single-threaded) owns the
+- A separate stage-5 process (periodic / cron, single-threaded) owns the
-  catalog **exclusively**: sweeps newly-arrived `binary/` files, runs sequential
+  catalog exclusively: sweeps newly-arrived `binary/` files, runs sequential
   CRP-predictive assignment, updates sufficient stats, writes nominations.
   **One writer, zero locks, no cross-thread shared mutable state.**
-- The catalog is a **durable file** mutated by one process — reuse the stage-2
+- The catalog is a durable file mutated by one process, so reuse the stage-2
   **sidecar-first durable-commit** discipline (`commit_enriched!`: temp write →
   fsync bytes → rename → fsync dir) so a crash can't corrupt it or lose a rename.
 
@@ -216,8 +216,8 @@ This slots in as a batch stage, matching how stages 2/3/4 already work. New
 ## 10. Concrete test assertions (write these first)
 
 1. **Discovers nothing from noise.** Current `data/binary` = 20 small random
-   blobs + 1 giant PDF. Correct output: PDF is a singleton that **never promotes**
+   blobs + 1 giant PDF. Correct output: PDF is a singleton that never promotes
-   (N=1), 20 blobs absorbed by background, **zero promoted clusters.** Any
+   (N=1), 20 blobs absorbed by background, zero promoted clusters. Any
    promoted cluster from this pile = broken.
 2. **Recovers known formats.** On a `../training_set` sample, calibrated settings
    cluster into the ~6 magic-collapsed classes with high ARI (docx+zip merged,
@@ -228,23 +228,23 @@ This slots in as a batch stage, matching how stages 2/3/4 already work. New
 
 ## 11. Implementation status & calibration results (v1)
 
-**Shipped.** `src/cluster.jl` — feature extraction (`header_symbols`, 257-symbol
+**Shipped.** `src/cluster.jl` holds feature extraction (`header_symbols`, 257-symbol
 alphabet), collapsed Gibbs (`gibbs_cluster`, phase A), the sequential
 CRP-predictive rule (`assign_file`, phase B core), signatures/promotion
 (`signature`, `is_promotable`), and calibration metrics (`adjusted_rand_index`,
-`v_measure`). All base-Julia — a base-only Lanczos `loggamma` keeps the
+`v_measure`). All base-Julia: a base-only Lanczos `loggamma` keeps the
 Dirichlet-multinomial marginal dependency-free (no Manifest churn). Config knobs
 `FS_CLUSTER_*` (§9) added. `bin/cluster_calibrate.jl` runs the §7 grid and the §8
 NCD baseline. Concrete §10 assertions are in the test suite (hermetic synthetic
 corpora, so they need neither `../training_set` nor gzip).
 
 **Calibrated defaults** (grid over the 700-file `training_set`, ranked by ARI
-excluding tar): **n=32, α=1.0, β=0.1, bg_mass=5.0** → ARI **0.77** (0.885 excl.
+excluding tar): n=32, α=1.0, β=0.1, bg_mass=5.0 → ARI 0.77 (0.885 excl.
 tar), V-measure 0.83, homogeneity 0.87. Clusters are clean and promotable:
 `pkzip:197` (docx+zip correctly merged, §7.2 ✓), `gzip:100`, `jpeg`, and several
-`pdf` clusters all self-nominate. The **NCD baseline agrees** (§10.3): on a
+`pdf` clusters all self-nominate. The NCD baseline agrees (§10.3): on a
 150-file subsample, NCD 1-NN label purity 0.90 vs. the model's same-cluster
-purity 0.987 — the generative header model separates formats at least as well as
+purity 0.987. The generative header model separates formats at least as well as
 model-free gzip similarity.
 
 ### Three assumptions the data corrected
@@ -253,34 +253,34 @@ model-free gzip similarity.
    tar's `ustar`-at-257 magic is out of window so tars scatter to background. But
    98/100 tars in the corpus are Hex/Elixir package tarballs whose *first
    archived file is named `VERSION`* → a constant, strongly-peaked `VERSION\0`
-   prefix at offset 0. They do form a peaked cluster — but it **merges with ELF**,
+   prefix at offset 0. They do form a peaked cluster, but it merges with ELF,
    because ELF's ident padding and tar's name-field zero-padding give the two a
    long shared run of `0x00` in bytes 5–31; they differ in only ~3 magic bytes,
    and 32 equally-weighted positions let ~20 shared zeros outvote 3 real ones. No
    β both separates ELF/tar and keeps the other formats whole. The honest v1
    position: this is the *same* "tar is hard" reality §4b flagged, just wearing a
-   different mask. **Fix (v2):** weight positions by inverse entropy so a
+   different mask. Fix (v2): weight positions by inverse entropy so a
    low-information shared-zero run stops dominating a few high-information magic
-   bytes — this generalizes beyond tar and is the highest-value next lever.
+   bytes. This generalizes beyond tar and is the highest-value next lever.
 
 2. **You cannot cold-start every point in the background.** A natural reading of
    §4a/§5 is "everything starts in the junk drawer, real clusters condense out."
-   That **deadlocks**: at a format's first file a fresh cluster and the background
+   That deadlocks: at a format's first file a fresh cluster and the background
    are *both* uniform, so with the `bg_mass ≥ α` that §4a needs for absorption,
    the background always wins and no cluster is ever seeded. Fix: **initialize
    every file in its own singleton**; same-format singletons merge and snowball,
    while a lone random-blob singleton dissolves on resample and is reclaimed by
-   the (stickier) background. Absorption still works — just not as the *initial*
+   the (stickier) background. Absorption still works, just not as the *initial*
    state.
 
 3. **Two pieces of math that look optional but aren't.** (a) Signature peakedness
-   is a **Bernoulli** question ("is this position fixed to byte v?"), so it uses a
+   is a Bernoulli question ("is this position fixed to byte v?"), so it uses a
-   2-way posterior `(count+β)/(members+2β)`, **not** the 257-way mixture
+   2-way posterior `(count+β)/(members+2β)`, not the 257-way mixture
-   predictive — the alphabet-wide denominator drags even a unanimous position
+   predictive, because the alphabet-wide denominator drags even a unanimous position
    below 0.9 once β<1, which would make promotion *impossible*. (b) Ranking Gibbs
-   restarts needs the **collapsed Dirichlet-multinomial marginal** (with its
+   restarts needs the collapsed Dirichlet-multinomial marginal (with its
    `loggamma` normalizer / Occam penalty); a plain product-of-predictives score
-   omits the penalty and actively **rewards merging** everything into one blob
+   omits the penalty and actively rewards merging everything into one blob
    (observed, then fixed).
 
 ### Known v1 limitations (accepted)
@@ -289,23 +289,23 @@ model-free gzip similarity.
   version byte). This costs completeness/ARI but not the mission: each sub-cluster
   still carries valid magic and promotes independently, and a human dedupes
   overlapping `%PDF-1.x` nominations at the gate.
-- The point partition is the **best of N Gibbs restarts by marginal likelihood**,
+- The point partition is the best of N Gibbs restarts by marginal likelihood,
-  a MAP-style stand-in for the VI/Binder posterior summary §5 defers — adequate
+  a MAP-style stand-in for the VI/Binder posterior summary §5 defers. Adequate,
   because the formats are strongly separated; revisit if compaction (§5) needs it.
-- Phase B's **live single-owner batch process** (§9) and the durable catalog file
+- Phase B's live single-owner batch process (§9) and the durable catalog file
   are implemented in `src/catalog.jl` (the `Catalog` durable state, the
   incremental `catalog_sweep!`, the offline `compact!` seed/recompaction, and
   `write_nominations!`), driven by `bin/cluster_sweep.jl` (cron/periodic; the
   first run auto-compacts to seed, subsequent runs sweep incrementally). The
   catalog is persisted with the stage-2 sidecar-first temp→fsync→rename→fsync-dir
   discipline. As §5B predicts, under the calibrated `bg_mass > α` the live sweep
-  never mints single-file clusters — new formats are discovered by the offline
+  never mints single-file clusters: new formats are discovered by the offline
   `compact!` re-clustering the background residue, not by the live path.
 
 ## Open items (deferred, intentionally)
 
 - **v2, now top priority: inverse-entropy position weighting** (unblocks ELF/tar
-  and any format pair that shares a long constant run — see §11).
+  and any format pair that shares a long constant run; see §11).
 
 - v2: tail-window block; sparse deep-offset probe (tar-class).
 - v3: sub-clustering structureless high-entropy residue (needs entropy/histogram

src/DarkStruct.jl

@@ -1,4 +1,4 @@
-module FileServer
+module DarkStruct
 
 using Logging
 using Random
@@ -15,6 +15,7 @@ include("config.jl")
 include("job.jl")
 include("queue.jl")
 include("stats.jl")     # per-stage counters behind GET /stats (needs Config/Job/JobQueue)
+include("rabbit.jl")    # RabbitMQ-backed JobQueue (needs Config/Job/JobQueue/STAGE_KEYS)
 include("spool.jl")
 include("model.jl")     # build_model() + read_features(); shared with bin/train.jl
 include("classify.jl")  # Classifier + load_classifier/classify (needs model.jl)
@@ -27,12 +28,17 @@ include("worker.jl")
 
 # Globals the HTTP handlers read at request time. Set once in `run`, before the
 # server starts accepting connections. Declared after the includes above so the
-# `Config`/`ChannelQueue` types exist.
+# `Config`/`JobQueue` types exist.
+#
+# Typed as the abstract `JobQueue` because which implementation these hold is a
+# runtime choice (`FS_QUEUE_BACKEND`). That makes `enqueue!` a dynamic dispatch
+# on the intake path — nanoseconds against a handler that writes a file to disk,
+# and the price of the backend being swappable at all.
 const CONFIG = Ref{Config}()
-const QUEUE  = Ref{ChannelQueue}()          # stage-1 (classification) queue; HTTP intake enqueues here
+const QUEUE  = Ref{JobQueue}()          # stage-1 (classification) queue; HTTP intake enqueues here
-const KNOWN_QUEUE = Ref{ChannelQueue}()     # stage-2 (enrichment) queue; stage-1 workers enqueue here
+const KNOWN_QUEUE = Ref{JobQueue}()     # stage-2 (enrichment) queue; stage-1 workers enqueue here
-const UNKNOWN_QUEUE = Ref{ChannelQueue}()   # stage-3 (content triage) queue; stage-1 workers enqueue here
+const UNKNOWN_QUEUE = Ref{JobQueue}()   # stage-3 (content triage) queue; stage-1 workers enqueue here
-const TEXT_QUEUE = Ref{ChannelQueue}()      # stage-4 (language enrichment) queue; stage-3 workers enqueue here
+const TEXT_QUEUE = Ref{JobQueue}()      # stage-4 (language enrichment) queue; stage-3 workers enqueue here
 const CLASSIFIER = Ref{Classifier}()  # loaded once at startup, shared read-only across workers
 const DETECTOR = Ref{LanguageDetector}()  # natural-language detector; built once at startup, shared read-only
 
@@ -42,7 +48,7 @@ export run
 
 # When stderr is a live terminal Julia flushes each write, but when it's
 # redirected to a file or pipe (a log file, `tee`, journald, a container log
-# driver) Julia block-buffers it — so a long-running server's logs sit unseen in
+# driver) Julia block-buffers it, so a long-running server's logs sit unseen in
 # the buffer until it fills or the process exits, making it look like nothing is
 # happening. This wrapper delegates to a normal logger and flushes after every
 # message so output appears immediately wherever stderr is pointed.
@@ -61,7 +67,7 @@ end
 """
     run(; overrides...)
 
-Start the file server: build config, ensure directories, recover any leftover
+Start DarkStruct: build config, ensure directories, recover any leftover
 spooled files, spawn the worker pool, then serve HTTP until interrupted
 (Ctrl-C / SIGINT or SIGTERM). On shutdown it stops accepting uploads, drains the
 queue, waits for workers to finish in-flight files, and exits cleanly.
@@ -94,13 +100,24 @@ function run(; overrides...)
 
     # github-linguist powers stage-4 *programming*-language detection, but it's
     # best-effort (natural-language enrichment stands on its own), so a missing
-    # binary is a warning, not a fatal error — per-file lookups degrade to none.
+    # binary is a warning, not a fatal error; per-file lookups degrade to none.
     linguist_available() || @warn "github-linguist not found on PATH; stage-4 text files will have no programming language (install it to enable)"
 
-    queue         = ChannelQueue(cfg.queue_capacity)
+    # The one place the backend choice is made. Everything downstream — handlers,
-    known_queue   = ChannelQueue(cfg.known_queue_capacity)
+    # workers, /stats — sees only `JobQueue` (src/queue.jl).
-    unknown_queue = ChannelQueue(cfg.unknown_queue_capacity)
+    # Set if the broker connection dies under us. This backend does not
-    text_queue    = ChannelQueue(cfg.text_queue_capacity)
+    # reconnect — see `poll_loop!` — so the honest response is to drain and let
+    # the restart policy bring us back to redelivered messages.
+    connection_lost = Threads.Atomic{Bool}(false)
+    backend = cfg.queue_backend === :rabbitmq ? connect_backend(cfg) : nothing
+    queues  = backend === nothing ?
+        (classify = ChannelQueue(cfg.queue_capacity),
+         enrich   = ChannelQueue(cfg.known_queue_capacity),
+         triage   = ChannelQueue(cfg.unknown_queue_capacity),
+         language = ChannelQueue(cfg.text_queue_capacity)) :
+        open_queues(backend, cfg; on_lost = () -> (connection_lost[] = true))
+    queue, known_queue = queues.classify, queues.enrich
+    unknown_queue, text_queue = queues.triage, queues.language
     CONFIG[]        = cfg
     QUEUE[]         = queue
     KNOWN_QUEUE[]   = known_queue
@@ -117,18 +134,13 @@ function run(; overrides...)
     DETECTOR[] = LanguageDetector()
     @info "loaded language detector"
 
-    # Stage-aware recovery: re-drive each stage's leftovers onto its own queue so
+    @info "starting darkstruct" backend=cfg.queue_backend host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity
-    # files resume where they were, not from scratch. spool/ → stage-1,
-    # known/ → stage-2, unknown/ → stage-3.
-    recovered         = recover_dir!(cfg.spool_dir, queue)
-    recovered_known   = recover_dir!(cfg.known_dir, known_queue)
-    recovered_unknown = recover_dir!(cfg.unknown_dir, unknown_queue)
-    recovered_text    = recover_dir!(cfg.text_dir, text_queue)
-    @info "starting file-server" host=cfg.host port=cfg.port workers=cfg.worker_count known_workers=cfg.known_worker_count unknown_workers=cfg.unknown_worker_count text_workers=cfg.text_worker_count capacity=cfg.queue_capacity known_capacity=cfg.known_queue_capacity unknown_capacity=cfg.unknown_queue_capacity text_capacity=cfg.text_queue_capacity recovered=recovered recovered_known=recovered_known recovered_unknown=recovered_unknown recovered_text=recovered_text
 
     # Zero the counters here, not at module load: `since` should mean "serving
     # since", so a scrape's totals cover the run, not the minutes spent loading
-    # the classifier. Nothing has been processed yet — recovery only enqueues.
+    # the classifier. It must also come *before* the pools start, since recovery
+    # below runs against live workers. The replayed files are real work and
+    # belong in the totals, but only from this point on.
     reset_metrics!()
 
     # Each pool gets its stage's counters (src/stats.jl); `worker_loop` records
@@ -151,6 +163,33 @@ function run(; overrides...)
                    st.language)
                     for i in 1:cfg.text_worker_count]
 
+    # Recovery runs *after* the pools are live, and blocks rather than dropping
+    # when the queue is full. `spool/` holds every in-flight file now, so a crash
+    # under load can leave far more leftovers than `queue_capacity`; with the
+    # consumers already running the queue drains as we fill it, so a big backlog
+    # takes longer to re-drive but none of it is abandoned. Serving hasn't
+    # started yet, so intake cannot race us for the directory.
+    #
+    # Everything re-enters at stage 1 and replays: in-process queues die with the
+    # process, so a crash loses the only record of how far each file had got.
+    # Safe, because every stage is a pure function of the file's bytes and every
+    # commit is idempotent; see the header of src/worker.jl.
+    #
+    # Broker-backed, this is exactly the wrong thing to do: the broker still
+    # holds every unacked job at the stage it reached, so re-driving `spool/`
+    # would duplicate the entire in-flight backlog on every restart and drag
+    # finished stages back to the start. `FS_RECOVER_SPOOL` forces it anyway, for
+    # the one case the broker cannot cover: it was purged or recreated, and the
+    # spooled files are the only surviving record of the work.
+    if cfg.queue_backend === :channel || cfg.recover_spool
+        recovered = recover_dir!(cfg.spool_dir, queue)
+        recovered > 0 && @info "recovered leftover files; replaying from stage 1" recovered=recovered
+    else
+        in_flight = count(p -> isfile(p) && !endswith(p, ".meta.json"),
+                          readdir(cfg.spool_dir; join = true))
+        @info "broker holds the in-flight record; not re-driving spool/ (set FS_RECOVER_SPOOL=true if the broker was purged)" spooled=in_flight
+    end
+
     register_routes()
     # `handler` replaces Oxygen's root stream handler so POST /upload can read its
     # body incrementally instead of having it buffered into memory first; every
@@ -162,7 +201,7 @@ function run(; overrides...)
     # buffered jobs, then exit. Called from two places:
     #   * the `finally` below, for SIGINT (Ctrl-C) and normal return, and
     #   * an `atexit` hook, for SIGTERM (systemd/Docker/k8s `stop`).
-    # We can't intercept SIGTERM directly — Julia blocks it on worker threads and
+    # We can't intercept SIGTERM directly: Julia blocks it on worker threads and
     # handles it in its own runtime, so a user signal() handler never fires. But
     # Julia's SIGTERM path runs `atexit` hooks, which gives us a reliable seam.
     drained = Threads.Atomic{Bool}(false)
@@ -171,22 +210,34 @@ function run(; overrides...)
         @info "draining queues and stopping workers"
         terminate()                    # 1. stop accepting new HTTP requests
         close!(queue)                  # 2. no new classify jobs; stage-1 drains buffered
-        foreach(wait, workers)         # 3. wait out stage-1 — the ONLY producer of BOTH the
+        foreach(wait, workers)         # 3. wait out stage-1, the ONLY producer of BOTH the
-                                       #    known and unknown queues — so nothing else enqueues
+                                       #    known and unknown queues, so nothing else enqueues
         close!(known_queue)            # 4. now safe to close the queues stage-1 fed
         close!(unknown_queue)
-        foreach(wait, known_workers)   # 5. wait out stage-2 (terminal) and stage-3 — stage-3 is
+        foreach(wait, known_workers)   # 5. wait out stage-2 (terminal) and stage-3; stage-3 is
         foreach(wait, unknown_workers) #    the ONLY producer of the text queue
         close!(text_queue)             # 6. now safe to close the queue stage-3 fed
         foreach(wait, text_workers)    # 7. wait out stage-4
+        backend === nothing || close_backend!(backend)   # 8. release the broker connection
         @info "shutdown complete"
     end
     atexit(drain)
 
+    # Two ways out of here: an interrupt, or the broker connection dying.
+    #
+    # SIGINT is delivered to whichever task happens to be running on thread 1,
+    # which is the main task in the common case but is not guaranteed to be:
+    # AMQPClient runs its own receiver tasks, and an interrupt that lands in one
+    # of those kills the connection instead of reaching this loop. That is not a
+    # hole so much as the same door: the poller sees the dead connection within
+    # a second and sets `connection_lost`, so the drain happens either way. Under
+    # a process manager, prefer SIGTERM for the broker backend — it goes through
+    # Julia's `atexit`, which has no such lottery (see "Shutdown" in the README).
     try
-        while true
+        while !connection_lost[]
             sleep(0.5)            # interruptible; SIGINT throws in here
         end
+        @error "stopping: broker connection closed"
     catch e
         e isa InterruptException || rethrow(e)
         @info "shutdown requested (SIGINT)"
@@ -194,6 +245,9 @@ function run(; overrides...)
         drain()
     end
 
+    # A lost broker is a failure, and saying so in the exit status is what lets a
+    # supervisor tell "asked to stop" from "fell over".
+    connection_lost[] && exit(1)
     return nothing
 end
 

src/catalog.jl

@@ -1,12 +1,12 @@
 # Stage-5 phase B: the live, single-owner format catalog (DESIGN §5B/§9).
 #
-# `cluster.jl` is the *science* — feature extraction, the collapsed Gibbs sampler
+# `cluster.jl` is the *science*: feature extraction, the collapsed Gibbs sampler
 # (phase A, offline), and `assign_file` (the phase-B scoring core). This file is
 # the *plumbing* that turns that core into a durable, growing catalog:
 #
 #   * a `Catalog` = the surviving clusters' sufficient statistics + a record of
 #     which files have already been folded in + a few example filenames each;
-#   * `catalog_sweep!` = the phase-B loop — for every *new* file in `binary/`,
+#   * `catalog_sweep!` = the phase-B loop. For every *new* file in `binary/`,
 #     run the deterministic CRP-predictive `assign_file` and fold it into the
 #     chosen cluster's stats (DESIGN §5B);
 #   * `compact!` = the offline Gibbs pass that *seeds* the catalog on first run
@@ -32,19 +32,19 @@ const CATALOG_EXAMPLE_CAP = 8
 
 The mutable phase-B state owned by the single sweep process:
 
-  * `n`         — header window these clusters were built at (must match the
+  * `n`         : header window these clusters were built at (must match the
                   feature window used to score new files; frozen once seeded).
-  * `clusters`  — id → `ClusterStats` (per-position 257-counts + member count).
+  * `clusters`  : id → `ClusterStats` (per-position 257-counts + member count).
-                  Ids are **frozen at birth** — never renumbered — so there is no
+                  Ids are frozen at birth and never renumbered, so there is no
                   label switching across sweeps (DESIGN §5).
-  * `examples`  — id → up to `CATALOG_EXAMPLE_CAP` member filenames, for the
+  * `examples`  : id → up to `CATALOG_EXAMPLE_CAP` member filenames, for the
                   human nomination glance.
-  * `next_id`   — the next fresh cluster id to hand out (monotone; retired ids are
+  * `next_id`   : the next fresh cluster id to hand out (monotone; retired ids are
                   never reused, keeping ids globally unique over the catalog's life).
-  * `processed` — basenames of every `binary/` file already folded in, so a sweep
+  * `processed` : basenames of every `binary/` file already folded in, so a sweep
                   is incremental: it touches only files it has not seen. This set
-                  grows with the `binary/` pile it mirrors — the same population,
+                  grows with the `binary/` pile it mirrors, at the same rate and
-                  no faster — which is acceptable for v1 (DESIGN §9).
+                  no faster, which is acceptable for v1 (DESIGN §9).
 """
 mutable struct Catalog
     n::Int
@@ -69,8 +69,9 @@ end
 # Durable persistence (reuse the stage-2 sidecar-first commit discipline)
 # ---------------------------------------------------------------------------
 
-# Count tables are stored sparsely — only the non-zero `(position, symbol, count)`
+# Count tables are stored sparsely, as only the non-zero
-# triples — because a cluster's n×257 table is overwhelmingly zero (a peaked
+# `(position, symbol, count)` triples, because a cluster's n×257 table is
+# overwhelmingly zero (a peaked
 # magic byte touches one of 257 symbols per position). Sparse keeps the catalog
 # file small and its load O(non-zeros), not O(n·257·K).
 function _sparse_counts(counts::Matrix{Int})
@@ -191,15 +192,15 @@ repeated sweeps are incremental and idempotent over the pile.
 
 Per file, `assign_file` returns the argmax component:
   * an existing cluster id → the file joins it (`add!`);
-  * `0` (background) → counted, not clustered — a novel-but-unmatched file parks
+  * `0` (background) → counted, not clustered. A novel-but-unmatched file parks
     here by design; genuinely new formats are discovered by the offline
     `compact!` re-clustering the residue, not by single-file minting;
   * `-1` (mint) → a fresh cluster is seeded with a frozen `next_id`. Under the
     calibrated `bg_mass > α` this never fires on the live path (fresh and
-    background share the one-file likelihood, so background always wins) — the
+    background share the one-file likelihood, so background always wins); the
     branch exists for correctness, not as a routine outcome (DESIGN §5B).
 
-Mutates `cat` but does NOT persist it — the caller commits once, after the sweep.
+Mutates `cat` but does NOT persist it; the caller commits once, after the sweep.
 Returns a summary of what happened this sweep.
 """
 function catalog_sweep!(cat::Catalog, cfg::Config)
@@ -281,7 +282,7 @@ function compact!(cat::Catalog, cfg::Config;
 end
 
 # ---------------------------------------------------------------------------
-# Nominations (closing the loop to the classifier — DESIGN §6)
+# Nominations (closing the loop to the classifier; DESIGN §6)
 # ---------------------------------------------------------------------------
 
 "Render a signature (from `signature`) into a human-readable hex template:
@@ -312,13 +313,13 @@ end
 
 Write one JSON nomination per promotable cluster (`is_promotable`, DESIGN §6)
 into `cfg.nominated_dir`, each carrying the cluster's signature (hex template +
-required magic positions), member count, and example filenames — everything a
+required magic positions), member count, and example filenames: everything a
 human needs to glance and promote. The background (id 0) is never a cluster here,
 so it can never be nominated, by construction.
 
 Nominations are rewritten every sweep (membership only grows), so each file is
 durably replaced via the same temp→fsync→rename→fsync-dir commit as the catalog.
-Returns the paths written. Non-promotable clusters are left alone — a cluster
+Returns the paths written. Non-promotable clusters are left alone; a cluster
 that *was* nominated and later fell below threshold cannot happen (members only
 grow), so there is nothing to retract.
 """
@@ -326,7 +327,7 @@ function write_nominations!(cat::Catalog, cfg::Config)
     mkpath(cfg.nominated_dir)
     written = String[]
     for (id, c) in sort(collect(cat.clusters); by=first)
-        sig = signature(c)                       # default β — signature β is
+        sig = signature(c)                       # default β; signature β is
                                                  # decoupled from clustering β (DESIGN §11.3a)
         is_promotable(c, sig; min_members=cfg.promote_min_members,
                       min_magic=cfg.promote_min_magic) || continue
@@ -353,14 +354,14 @@ function write_nominations!(cat::Catalog, cfg::Config)
 end
 
 # ---------------------------------------------------------------------------
-# Orchestration + CLI (the periodic single-owner process — DESIGN §9)
+# Orchestration + CLI (the periodic single-owner process; DESIGN §9)
 # ---------------------------------------------------------------------------
 
 """
     run_cluster_sweep(cfg; compact=false, rng) -> NamedTuple
 
 One end-to-end pass of the single-owner stage-5 process: load the durable
-catalog, either `compact!` (offline Gibbs — used to seed on first run or to
+catalog, either `compact!` (offline Gibbs, used to seed on first run or to
 recompact) or `catalog_sweep!` (the incremental live path), then durably persist
 the catalog and (re)write nominations. This is the whole job the cron/periodic
 runner performs; `bin/cluster_sweep.jl` is a thin shell around it.

src/classify.jl

@@ -8,7 +8,7 @@
 
 An in-memory, ready-to-run classifier: the (stateless) Lux `model` plus the
 learned parameters `ps` and states `st` loaded from the artifact. Immutable and
-shared read-only across all worker threads — Lux inference is a pure function
+shared read-only across all worker threads, since Lux inference is a pure function
 over `ps`/`st`, so no locking is needed.
 """
 struct Classifier{M,P,S}
@@ -21,7 +21,7 @@ end
     load_classifier(path) -> Classifier
 
 Rebuild the fixed architecture and load `ps`/`st` from the JLD2 artifact at
-`path`. Throws if the file is missing or unreadable — the server fails fast at
+`path`. Throws if the file is missing or unreadable, so the server fails fast at
 startup rather than run silently without classification.
 """
 function load_classifier(path::AbstractString)

src/cluster.jl

@@ -1,7 +1,7 @@
 # Stage-5: unknown-format discovery by Bayesian header clustering.
 #
 # See model/DESIGN_clustering.md for the full rationale. In brief: files that
-# stage-3 sorted into `binary/` are the `:unknown` sink — genuinely
+# stage-3 sorted into `binary/` are the `:unknown` sink: genuinely
 # unrecognized bytes. This stage clusters them by *file format* (not producer)
 # using the first `HEADER_N` header bytes, modeled as a Dirichlet-process
 # mixture of per-position categoricals over a 257-symbol alphabet
@@ -11,7 +11,7 @@
 # This file is deliberately dependency-light: everything below is base Julia
 # (only `log`, no `SpecialFunctions`), so it drops into the existing module and
 # the offline calibration script alike without touching the Manifest. The model
-# is categorical on purpose — do NOT reuse model.jl's [0,1] byte scaling here
+# is categorical on purpose. Do NOT reuse model.jl's [0,1] byte scaling here
 # (that metric is meaningful for the Lux net and meaningless for header bytes,
 # where 0x89 and 0x88 are not "close"; see DESIGN §4).
 
@@ -22,7 +22,7 @@ const HEADER_N = 32
 const ALPHABET = 257
 
 "The 'past EOF' symbol (1-based index `ALPHABET`). A file shorter than a given
-position emits this here — real, discriminative signal for fixed-length formats,
+position emits this here: real, discriminative signal for fixed-length formats,
 and it avoids colliding zero-padding with genuine 0x00 header bytes (DESIGN §4)."
 const PAST_EOF = ALPHABET
 
@@ -75,7 +75,7 @@ Sufficient statistics for one cluster: a per-position count table `counts`
 (`n × ALPHABET`; `counts[i, v]` = how many member files show symbol `v` at
 position `i`) and the member count `members`. These are exactly what phase-B
 persists per catalog entry, and everything the collapsed predictive needs.
-A slot with `members == 0` is inactive (reusable) — the Gibbs sweep prunes
+A slot with `members == 0` is inactive (reusable): the Gibbs sweep prunes
 emptied clusters without renumbering, so surviving cluster ids stay stable.
 """
 mutable struct ClusterStats
@@ -111,7 +111,7 @@ Dirichlet-Categorical predictive, given `c`'s current counts: at each position
 `i`, `p(x_i | c) = (counts[i, x_i] + β) / (members + ALPHABET·β)`, summed in log
 space over positions. Call with `c` NOT containing `x` (Gibbs excludes the point
 being resampled), so an emptied cluster reduces to the uniform prior `1/ALPHABET`
-per position — identical to a brand-new cluster, as it should be.
+per position, identical to a brand-new cluster, as it should be.
 """
 function log_predictive(c::ClusterStats, x::AbstractVector{<:Integer}, β::Float64)
     denom = log(c.members + ALPHABET * β)
@@ -129,7 +129,7 @@ log_uniform(n::Integer) = -n * log(ALPHABET)
 
 # Lanczos approximation to log Γ(x) for x > 0, so partition scoring (below) needs
 # the Dirichlet-multinomial marginal's gamma terms without pulling in
-# SpecialFunctions — keeping this stage dependency-flat (no Manifest churn).
+# SpecialFunctions, keeping this stage dependency-flat (no Manifest churn).
 # g = 7, standard coefficients; accurate to ~1e-14 over the range we use.
 const _LANCZOS_G = 7
 const _LANCZOS_C = (0.99999999999980993, 676.5203681218851, -1259.1392167224028,
@@ -148,7 +148,7 @@ function loggamma(x::Float64)
 end
 
 # ---------------------------------------------------------------------------
-# Phase A: collapsed Gibbs sampler (offline — the science)
+# Phase A: collapsed Gibbs sampler (offline; the science)
 # ---------------------------------------------------------------------------
 
 """
@@ -207,7 +207,7 @@ function _gibbs_once(X::AbstractMatrix{<:Integer};
     # from the background deadlocks: at a format's first file, a fresh cluster
     # and the background are equally uniform, so with bg_mass ≥ α the background
     # always wins and no real cluster is ever seeded. Singleton init sidesteps
-    # this — same-format singletons merge and snowball, while a lone
+    # this: same-format singletons merge and snowball, while a lone
     # random-blob singleton dissolves on resample and is reclaimed by the
     # (stickier) background. See DESIGN §4a.
     z = collect(1:N)
@@ -274,7 +274,7 @@ function _gibbs_once(X::AbstractMatrix{<:Integer};
     return GibbsResult(z, clusters, partition_logmarginal(X, z, clusters, α, β))
 end
 
-"Argmax of `logw .+ Gumbel noise` — an exact draw from softmax(logw) without
+"Argmax of `logw .+ Gumbel noise`: an exact draw from softmax(logw) without
 normalizing (numerically safe for the tiny header-likelihood magnitudes)."
 function _gumbel_argmax(logw::AbstractVector{Float64}, rng::AbstractRNG)
     best_i = 1
@@ -293,12 +293,12 @@ end
     partition_logmarginal(X, z, clusters, α, β) -> Float64
 
 The joint log-evidence `log p(z, X)` of a partition under the CRP prior and the
-collapsed Dirichlet-Categorical likelihood — the principled score for ranking
+collapsed Dirichlet-Categorical likelihood. This is the principled score for ranking
 Gibbs restarts (higher = better). It is the sum of:
 
   * the Dirichlet-multinomial **marginal** of each cluster's per-position counts,
     `lΓ(Aβ) − lΓ(mₖ+Aβ) + Σ_v [lΓ(c_v+β) − lΓ(β)]`, whose normalizer supplies the
-    Occam penalty that a plain product-of-predictives lacks — it is what makes a
+    Occam penalty that a plain product-of-predictives lacks. It is what makes a
     *merged, heterogeneous* cluster score **worse** than two clean ones (an
     earlier pseudo-likelihood scorer omitted this and wrongly rewarded merging);
   * the CRP prior over the clustered points, `K·log α + Σₖ lΓ(mₖ) + lΓ(α) −
@@ -335,7 +335,7 @@ function partition_logmarginal(X::AbstractMatrix{<:Integer}, z::AbstractVector{<
 end
 
 # ---------------------------------------------------------------------------
-# Phase B: sequential CRP-predictive assignment (online — the catalog)
+# Phase B: sequential CRP-predictive assignment (online; the catalog)
 # ---------------------------------------------------------------------------
 
 """
@@ -380,7 +380,7 @@ function assign_file(x::AbstractVector{<:Integer}, clusters::Dict{Int,ClusterSta
 end
 
 # ---------------------------------------------------------------------------
-# Signatures and promotion (closing the loop to the classifier — DESIGN §6)
+# Signatures and promotion (closing the loop to the classifier; DESIGN §6)
 # ---------------------------------------------------------------------------
 
 """
@@ -390,13 +390,13 @@ Turn a cluster's counts into a libmagic-style template: at each position, if the
 modal symbol's posterior probability exceeds `peak_threshold`, that byte is
 *required* (returned as the 0–255 byte value, or `PAST_EOF`); otherwise the
 position is a wildcard (`nothing`). The vector of required bytes IS the
-magic-number template — the whole point of the categorical model (DESIGN §4).
+magic-number template, the whole point of the categorical model (DESIGN §4).
 
 Peakedness is a **Bernoulli** question ("is this position fixed to byte `v`, or
-not?"), so it uses a 2-way posterior mean `(count + β)/(members + 2β)` — NOT the
+not?"), so it uses a 2-way posterior mean `(count + β)/(members + 2β)`, NOT the
 257-way mixture predictive. The alphabet-wide version would smear the estimate
 across 257 symbols (`members + 257β` in the denominator), pulling even a
-unanimous position below any sane threshold once β is small — which would make
+unanimous position below any sane threshold once β is small, which would make
 promotion impossible. This decouples signature detection from the clustering
 pseudocount and the alphabet size.
 """
@@ -415,7 +415,7 @@ function signature(c::ClusterStats; peak_threshold::Float64=0.9, β::Float64=0.5
     return sig
 end
 
-"Number of fixed (non-wildcard) positions in a signature — its 'magic length'."
+"Number of fixed (non-wildcard) positions in a signature: its 'magic length'."
 magic_positions(sig::AbstractVector) = count(!isnothing, sig)
 
 """
@@ -423,7 +423,7 @@ magic_positions(sig::AbstractVector) = count(!isnothing, sig)
 
 A cluster qualifies for *nomination* (still human-gated, DESIGN §6) when it has
 at least `min_members` files AND at least `min_magic` fixed signature positions.
-The background (id 0) is never passed here — it is never promotable by design.
+The background (id 0) is never passed here; it is never promotable by design.
 """
 function is_promotable(c::ClusterStats, sig::AbstractVector;
                        min_members::Integer=20, min_magic::Integer=3)
@@ -448,7 +448,7 @@ end
 
 Adjusted Rand Index between two labelings of the same items: 1.0 = identical
 partitions (up to relabeling), ~0.0 = chance agreement, can go negative. This is
-the §7 calibration objective — grid-tuning maximizes ARI of recovered-vs-truth
+the §7 calibration objective: grid-tuning maximizes ARI of recovered-vs-truth
 (magic-collapsed) labels. Hand-rolled to keep the dependency footprint flat;
 matches `Clustering.randindex`.
 """

src/config.jl

@@ -13,7 +13,7 @@ Base.@kwdef struct Config
     known_worker_count::Int   = Threads.nthreads()
     known_queue_capacity::Int = 1000
     # Stage 3 (content triage) also gets its own pool + queue: sorting an
-    # unrecognized file into binary/ vs text/ is cheap I/O, tuned independently
+    # unrecognized file into binary vs text is cheap I/O, tuned independently
     # of the classify and enrich pools.
     unknown_worker_count::Int   = Threads.nthreads()
     unknown_queue_capacity::Int = 1000
@@ -22,17 +22,42 @@ Base.@kwdef struct Config
     # github-linguist) is a mix of CPU and process-spawn work, tuned independently.
     text_worker_count::Int   = Threads.nthreads()
     text_queue_capacity::Int = 1000
-    spool_dir::String   = "data/spool"   # files land here on intake (pending classification)
+    # Which `JobQueue` implementation backs the four stage queues (src/queue.jl):
-    known_dir::String   = "data/known"   # classified-known, awaiting enrichment (stage 2)
+    # `:channel` (in-process, the default, nothing external to run) or
-    unknown_dir::String = "data/unknown" # classified-unknown, awaiting content triage (stage 3)
+    # `:rabbitmq` (durable broker queues, src/rabbit.jl). The queue *capacities*
+    # above apply to both, but mean different things: a hard limit in-process, an
+    # advisory one against a polled depth on the broker (see `RabbitQueue`).
+    queue_backend::Symbol = :channel
+    amqp_url::String      = "amqp://guest:guest@localhost:5672/"
+    amqp_prefix::String   = "darkstruct"   # queue names are "<prefix>.<stage>"
+    # Unacked messages the broker will hand one stage at a time. Defaults to that
+    # stage's worker count (0 below means "use it"), so a worker holds at most the
+    # one it is working on and a restart redelivers the minimum.
+    amqp_prefetch::Int    = 0
+    # Wait for a publisher confirm before intake reports a file accepted, so a
+    # 202 means the broker has it rather than only that our socket took it. Costs
+    # a round trip per file on the intake path *only*: inter-stage publishes stay
+    # fire-and-forget, because a lost one leaves its source message unacked and
+    # redelivery repairs it. See "Queue backends" in the README.
+    amqp_confirms::Bool   = true
+    # Broker-backed startup does not re-drive `spool/`: the broker is the record
+    # of what is in flight, and re-driving would duplicate the whole backlog at
+    # stage 1 on every restart. Set this when the broker was purged or recreated
+    # and the spooled files are the only surviving record. Ignored (always on)
+    # for the in-process backend, whose queues never survive a restart.
+    recover_spool::Bool   = false
+    # Every in-flight file lives here, at every stage, from intake until it is
+    # committed to a terminal sink below. Stages route by enqueueing a reference,
+    # not by moving bytes (src/worker.jl header), so there are no per-stage
+    # directories; the queues are what know how far a file has got.
+    spool_dir::String   = "data/spool"
     binary_dir::String  = "data/binary"  # stage-3 sink: unknown files that look like binary data
-    text_dir::String    = "data/text"    # classified-text, awaiting language enrichment (stage 4)
     done_dir::String    = "data/done"    # fully enriched known files (+ .meta.json sidecars)
     text_done_dir::String = "data/text_done" # fully enriched text files (+ .meta.json sidecars)
     failed_dir::String  = "data/failed"  # files move here if a worker throws
     model_path::String  = "model/classifier.jld2"  # committed classifier artifact, loaded at startup
     # Intake reads each upload off the socket in chunks of this size and streams
-    # them straight to the spool file, so this — not the file size — is what
+    # them straight to the spool file, so this, not the file size, is what
     # bounds intake memory per in-flight upload (see src/multipart.jl).
     upload_chunk_bytes::Int = UPLOAD_CHUNK_BYTES
     exiftool_timeout::Int = 30           # seconds before a stuck exiftool is killed → degraded sidecar
@@ -55,39 +80,66 @@ Base.@kwdef struct Config
     nominated_dir::String        = "data/nominated"     # one JSON per self-nominated cluster, awaiting a human promote
 end
 
+"""
+    parse_backend(s) -> Symbol
+
+Turn `FS_QUEUE_BACKEND` into a `Config.queue_backend`. Unknown values are a
+startup error rather than a silent fallback: a typo that quietly left you on the
+in-process queue would look exactly like durability working right up until a
+crash proved it wasn't.
+"""
+function parse_backend(s::AbstractString)
+    b = Symbol(lowercase(strip(s)))
+    b in (:channel, :rabbitmq) ||
+        throw(ArgumentError("FS_QUEUE_BACKEND must be \"channel\" or \"rabbitmq\", got \"$s\""))
+    return b
+end
+
+"Parse a boolean env var. Accepts the usual spellings; anything else is an error."
+function parse_bool(s::AbstractString)
+    v = lowercase(strip(s))
+    v in ("1", "true", "yes", "on")  && return true
+    v in ("0", "false", "no", "off") && return false
+    throw(ArgumentError("expected a boolean (true/false), got \"$s\""))
+end
+
 """
     config_from_env(; overrides...)
 
 Build a `Config` from environment variables, falling back to the struct
 defaults. Any keyword `overrides` win over the environment (useful for tests
-and for `FileServer.run(; port=...)`).
+and for `DarkStruct.run(; port=...)`).
 
 Recognised variables:
   FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY,
   FS_KNOWN_WORKERS, FS_KNOWN_QUEUE_CAPACITY,
   FS_UNKNOWN_WORKERS, FS_UNKNOWN_QUEUE_CAPACITY,
   FS_TEXT_WORKERS, FS_TEXT_QUEUE_CAPACITY,
-  FS_SPOOL_DIR, FS_KNOWN_DIR, FS_UNKNOWN_DIR, FS_BINARY_DIR, FS_TEXT_DIR,
+  FS_SPOOL_DIR, FS_BINARY_DIR,
   FS_DONE_DIR, FS_TEXT_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH,
   FS_UPLOAD_CHUNK_BYTES, FS_EXIFTOOL_TIMEOUT, FS_LINGUIST_TIMEOUT,
   FS_CLUSTER_DIR, FS_CLUSTER_N, FS_CLUSTER_ALPHA, FS_CLUSTER_PSEUDOCOUNT,
   FS_CLUSTER_BG_MASS, FS_PROMOTE_MIN_MEMBERS, FS_PROMOTE_MIN_MAGIC,
-  FS_CLUSTER_CATALOG, FS_NOMINATED_DIR
+  FS_CLUSTER_CATALOG, FS_NOMINATED_DIR,
+  FS_QUEUE_BACKEND, FS_AMQP_URL, FS_AMQP_PREFIX, FS_AMQP_PREFETCH,
+  FS_AMQP_CONFIRMS, FS_RECOVER_SPOOL
 """
 function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
                           queue_capacity=nothing, known_worker_count=nothing,
                           known_queue_capacity=nothing, unknown_worker_count=nothing,
                           unknown_queue_capacity=nothing, text_worker_count=nothing,
                           text_queue_capacity=nothing, spool_dir=nothing,
-                          known_dir=nothing, unknown_dir=nothing, binary_dir=nothing,
+                          binary_dir=nothing, done_dir=nothing, text_done_dir=nothing,
-                          text_dir=nothing, done_dir=nothing, text_done_dir=nothing,
                           failed_dir=nothing, model_path=nothing,
                           upload_chunk_bytes=nothing, exiftool_timeout=nothing,
                           linguist_timeout=nothing, cluster_dir=nothing, cluster_n=nothing,
                           cluster_alpha=nothing, cluster_pseudocount=nothing,
                           cluster_bg_mass=nothing, promote_min_members=nothing,
                           promote_min_magic=nothing, cluster_catalog_path=nothing,
-                          nominated_dir=nothing)
+                          nominated_dir=nothing, queue_backend=nothing,
+                          amqp_url=nothing, amqp_prefix=nothing,
+                          amqp_prefetch=nothing, amqp_confirms=nothing,
+                          recover_spool=nothing)
     Config(
         host                   = something(host,                   get(ENV, "FS_HOST", "127.0.0.1")),
         port                   = something(port,                   parse(Int, get(ENV, "FS_PORT", "8080"))),
@@ -100,10 +152,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
         text_worker_count      = something(text_worker_count,      parse(Int, get(ENV, "FS_TEXT_WORKERS", string(Threads.nthreads())))),
         text_queue_capacity    = something(text_queue_capacity,    parse(Int, get(ENV, "FS_TEXT_QUEUE_CAPACITY", "1000"))),
         spool_dir              = something(spool_dir,              get(ENV, "FS_SPOOL_DIR", "data/spool")),
-        known_dir              = something(known_dir,              get(ENV, "FS_KNOWN_DIR", "data/known")),
-        unknown_dir            = something(unknown_dir,            get(ENV, "FS_UNKNOWN_DIR", "data/unknown")),
         binary_dir             = something(binary_dir,             get(ENV, "FS_BINARY_DIR", "data/binary")),
-        text_dir               = something(text_dir,               get(ENV, "FS_TEXT_DIR", "data/text")),
         done_dir               = something(done_dir,               get(ENV, "FS_DONE_DIR", "data/done")),
         text_done_dir          = something(text_done_dir,          get(ENV, "FS_TEXT_DONE_DIR", "data/text_done")),
         failed_dir             = something(failed_dir,             get(ENV, "FS_FAILED_DIR", "data/failed")),
@@ -120,14 +169,19 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing,
         promote_min_magic      = something(promote_min_magic,      parse(Int, get(ENV, "FS_PROMOTE_MIN_MAGIC", "3"))),
         cluster_catalog_path   = something(cluster_catalog_path,   get(ENV, "FS_CLUSTER_CATALOG", "data/catalog.json")),
         nominated_dir          = something(nominated_dir,          get(ENV, "FS_NOMINATED_DIR", "data/nominated")),
+        queue_backend          = something(queue_backend,          parse_backend(get(ENV, "FS_QUEUE_BACKEND", "channel"))),
+        amqp_url               = something(amqp_url,               get(ENV, "FS_AMQP_URL", "amqp://guest:guest@localhost:5672/")),
+        amqp_prefix            = something(amqp_prefix,            get(ENV, "FS_AMQP_PREFIX", "darkstruct")),
+        amqp_prefetch          = something(amqp_prefetch,          parse(Int, get(ENV, "FS_AMQP_PREFETCH", "0"))),
+        amqp_confirms          = something(amqp_confirms,          parse_bool(get(ENV, "FS_AMQP_CONFIRMS", "true"))),
+        recover_spool          = something(recover_spool,          parse_bool(get(ENV, "FS_RECOVER_SPOOL", "false"))),
     )
 end
 
 "Create all the pipeline-stage directories if they don't already exist."
 function ensure_dirs(cfg::Config)
-    for d in (cfg.spool_dir, cfg.known_dir, cfg.unknown_dir, cfg.binary_dir,
+    for d in (cfg.spool_dir, cfg.binary_dir, cfg.done_dir, cfg.text_done_dir,
-              cfg.text_dir, cfg.done_dir, cfg.text_done_dir, cfg.failed_dir,
+              cfg.failed_dir, cfg.nominated_dir)
-              cfg.nominated_dir)
         mkpath(d)
     end
     return nothing

src/content.jl

@@ -1,12 +1,14 @@
 # Stage-3 content triage for unknown files.
 #
 # A file that stage-1 couldn't recognize is still sorted into one of two coarse
-# buckets so downstream tooling can treat them differently: `text/` for
+# buckets so downstream tooling can treat them differently: text (human-readable)
-# human-readable content, `binary/` for everything else. We sniff only the first
+# and binary (everything else). Only binary is a directory, because it is terminal:
+# the file is committed to `binary/`; a text file has stage 4 still to come, so it
+# stays in `spool/` and only its queue reference moves on. We sniff only the first
 # `CONTENT_SNIFF_BYTES` (no full read) and ask two questions: does the window
 # decode as valid UTF-8, and are any of its control bytes ones that don't belong
-# in text? This is the Unicode-aware successor to the classic "NUL byte" test —
+# in text? This is the Unicode-aware successor to the classic "NUL byte" test.
-# it accepts non-ASCII text (accents, CJK, emoji) instead of misfiling it as
+# It accepts non-ASCII text (accents, CJK, emoji) instead of misfiling it as
 # binary, while still rejecting binary formats, which almost never form valid
 # UTF-8 near their start (and a NUL is never a valid UTF-8 scalar, so it still
 # reads as binary for free).

src/job.jl

@@ -1,8 +1,11 @@
 # A unit of work on the queue. Deliberately lightweight: the file *bytes* live
 # on disk in the spool directory, and only this small reference travels through
 # the queue. This is what keeps intake fast and memory flat regardless of file
-# size, and it's the shape you'd publish to RabbitMQ later (the "claim check"
+# size, and it's the shape published to RabbitMQ (the "claim check" pattern:
-# pattern — enqueue a reference, not the payload).
+# enqueue a reference, not the payload).
+#
+# `path` is a *local* filesystem path, which is why a broker-backed pipeline is
+# still single-consumer-process: see "Queue backends" in the README.
 
 struct Job
     id::String            # server-minted UUID; also the on-disk filename prefix
@@ -10,4 +13,18 @@ struct Job
     path::String          # absolute-ish path to the spooled file
     size::Int             # bytes
     received_at::Float64  # time() at intake
+    # Broker delivery tag, and the one field that is *not* part of the job: it
+    # identifies this delivery on the channel that delivered it, so it is set on
+    # receipt and never serialized into the message body (a tag from a previous
+    # connection is meaningless, and a redelivery gets a fresh one). 0 means "no
+    # broker" — every job under `ChannelQueue`, and every job before publishing.
+    # Int64 rather than UInt64 to match AMQPClient's `TAMQPDeliveryTag`.
+    delivery_tag::Int64
 end
+
+Job(id, original_name, path, size, received_at) =
+    Job(id, original_name, path, size, received_at, Int64(0))
+
+"Copy of `job` carrying the delivery tag it arrived with."
+with_delivery_tag(job::Job, tag::Integer) =
+    Job(job.id, job.original_name, job.path, job.size, job.received_at, Int64(tag))

src/language.jl

@@ -1,13 +1,13 @@
 # Stage-4 language enrichment for text files.
 #
-# A file that stage-3 sorted into `text/` is human-readable, but we don't yet
+# A file that stage-3 sorted as text is human-readable, but we don't yet know
-# know *what* it is. This stage answers two questions and records them in a
+# *what* it is. This stage answers two questions and records them in a
 # `.meta.json` sidecar, exactly like the stage-2 known-file enrichment:
 #
-#   * natural language  — via Languages.jl's `LanguageDetector` (a Julia port of
+#   * natural language, via Languages.jl's `LanguageDetector` (a Julia port of
 #     the `whatlang` n-gram model): English vs. French vs. Japanese, plus a
 #     confidence score. Pure Julia, no subprocess.
-#   * programming language — via the `github-linguist` CLI, which recognizes
+#   * programming language, via the `github-linguist` CLI, which recognizes
 #     source and markup by extension + content heuristics. There is no
 #     comparable native Julia library, so we shell out (mirroring stage-2's
 #     exiftool dependency).
@@ -18,7 +18,7 @@
 #
 # The github-linguist quirk that shapes this code: run against a path *inside* a
 # git repository, linguist reads the file's committed git blob, not the bytes on
-# disk — and an untracked file (which every file under `data/` is) has no blob,
+# disk, and an untracked file (which every file under `data/` is) has no blob,
 # so it crashes. We sidestep this by copying the file to a fresh temp dir outside
 # any repo (preserving its name so linguist's extension heuristics still fire)
 # and pointing linguist there.
@@ -58,7 +58,7 @@ end
 Run the `LanguageDetector` on `text`, returning the language's English name
 (e.g. `"English"`), its ISO 639-3 code (e.g. `"eng"`), and the model's
 confidence in `[0,1]`. Returns `(nothing, nothing, nothing)` when there is no
-usable text (empty/whitespace) or the detector errors — the caller records that
+usable text (empty/whitespace) or the detector errors; the caller records that
 as a degraded result rather than failing the file.
 """
 function detect_natural_language(detector, text::AbstractString)
@@ -80,7 +80,7 @@ when linguist can't identify one. Plain prose reports as `"Text"` and
 unrecognized content as JSON `null`; both collapse to `nothing` here, since only
 a real programming/markup language is worth recording.
 
-`path` MUST be outside any git repository — see the module header for why.
+`path` MUST be outside any git repository; see the module header for why.
 """
 function run_linguist(path::AbstractString, timeout::Integer)
     bytes = run_with_timeout(`github-linguist --json $path`, timeout)
@@ -104,8 +104,8 @@ end
     detect_programming_language(job, cfg) -> Union{String,Nothing}
 
 Programming/markup language of a text file, or `nothing`. Copies the file to a
-throwaway temp dir *outside* the git repo — under its sanitized original name so
+throwaway temp dir *outside* the git repo (under its sanitized original name, so
-linguist's extension heuristics still apply — runs linguist there, and cleans up.
+linguist's extension heuristics still apply), runs linguist there, and cleans up.
 """
 function detect_programming_language(job::Job, cfg::Config)
     lang = nothing

src/metadata.jl

@@ -1,17 +1,17 @@
 # Stage-2 metadata extraction and enrichment.
 #
 # Known files are enriched by shelling out to `exiftool -json -G` (the only tool
-# with broad, multi-format coverage — there is no comparable native Julia
+# with broad, multi-format coverage; there is no comparable native Julia
 # library), then normalizing its output into a small, stable, documented schema
 # that downstream consumers can rely on, while preserving the full raw dump.
 #
 # exiftool being installed is a hard startup prerequisite (see `assert_exiftool`,
 # called from `run`). A per-file extraction failure or hang does NOT quarantine
-# the file — it produces a *degraded* sidecar recording what we know plus the
+# the file. It produces a *degraded* sidecar recording what we know plus the
 # error, because a file that passed classification is wanted regardless of
 # whether we could read its metadata.
 
-"Throw at startup if the `exiftool` binary isn't on PATH — fail fast rather than discover it per file."
+"Throw at startup if the `exiftool` binary isn't on PATH: fail fast rather than discover it per file."
 function assert_exiftool()
     try
         Base.run(pipeline(`exiftool -ver`; stdout=devnull, stderr=devnull))
@@ -27,14 +27,14 @@ end
 # colon. Extend a field simply by appending tag names here.
 #
 # Note the documented `Creator` ambiguity: in PDF it's the authoring app, but
-# elsewhere it's the person. We take the simple route — `Creator` feeds
+# elsewhere it's the person. We take the simple route: `Creator` feeds
 # `created_by` only, and `author` relies on the person-specific tags.
 const CREATED_BY_TAGS = ["Producer", "CreatorTool", "Creator", "Software", "Application", "Encoder", "EncodingTool", "HostComputer"]
 const AUTHOR_TAGS      = ["Author", "Artist", "By-line", "Owner", "Artist"]
 const CREATED_DATE_TAGS  = ["DateTimeOriginal", "CreateDate", "MediaCreateDate", "CreationDate"]
 const MODIFIED_DATE_TAGS = ["ModifyDate", "FileModifyDate"]
 
-"fsync an open file descriptor, throwing on failure — used to make a write durable before a rename commits it."
+"fsync an open file descriptor, throwing on failure. Makes a write durable before a rename commits it."
 function fsync_fd(fd)
     ccall(:fsync, Cint, (Cint,), fd) == 0 || error("fsync failed: $(Base.Libc.strerror())")
     return nothing
@@ -73,7 +73,7 @@ const KILL_GRACE_SECONDS = 2.0
 """
 Send `signum` to the whole process group `pgid` (a negative pid means "the group"
 to `kill(2)`). Julia's `kill(::Process, sig)` signals only the child itself,
-which is not enough to enforce a timeout — see `run_with_timeout`.
+which is not enough to enforce a timeout; see `run_with_timeout`.
 """
 function signal_group(pgid::Integer, signum::Integer)
     ccall(:kill, Cint, (Cint, Cint), -pgid, signum)
@@ -109,7 +109,7 @@ function run_with_timeout(cmd::Cmd, timeout::Integer)
     # A one-shot timer, cancelled the moment the child exits, rather than a
     # polling loop the caller has to join. The polling version charged every
     # call the remainder of its in-flight `sleep(0.1)` *after* the child had
-    # already exited — ~50 ms on average, and a measured 101 ms on a process
+    # already exited: ~50 ms on average, and a measured 101 ms on a process
     # that exits instantly. That is pure latency on the hot path of two stages
     # (exiftool here, github-linguist in stage 4), and it dwarfed the work on
     # anything but a slow file. Waiting on the process directly costs nothing
@@ -122,7 +122,7 @@ function run_with_timeout(cmd::Cmd, timeout::Integer)
         # Escalate: a process that ignores/defers SIGTERM would otherwise pin the
         # worker forever on the wait(proc) below, defeating the timeout. This
         # runs off the timer's task so the event loop isn't held during the
-        # grace period, and it is not joined — by the time it wakes, `wait(proc)`
+        # grace period, and it is not joined: by the time it wakes, `wait(proc)`
         # has long since returned and `process_running` settles it.
         Threads.@spawn begin
             deadline = time() + KILL_GRACE_SECONDS
@@ -235,7 +235,7 @@ Sequence: write `<name>.meta.json` to a temp name, fsync its bytes, rename it
 into place, fsync `dest_dir` so the rename itself is durable, THEN move the file
 into `dest_dir`. A crash between the two leaves only a harmless orphan sidecar in
 `dest_dir` while the file stays in its stage dir, so stage-aware recovery
-re-drives it and overwrites the sidecar — idempotent. The fsyncs make the
+re-drives it and overwrites the sidecar, idempotently. The fsyncs make the
 ordering hold across power loss, not just process crashes.
 """
 function commit_enriched!(dest_dir::AbstractString, job::Job, meta)

src/model.jl

@@ -4,8 +4,8 @@
 # drift out of sync (same input layout, same scaling, same net shape).
 #
 # The including scope is expected to have already done `using Lux` and
-# `using JLD2` (FileServer.jl and bin/train.jl both do), so this file adds no
+# `using JLD2` (DarkStruct.jl and bin/train.jl both do), so this file adds no
-# top-level `using` of its own and works whether it's pulled into the FileServer
+# top-level `using` of its own and works whether it's pulled into the DarkStruct
 # module or a bare script.
 
 "Number of input features: first 16 bytes + last 16 bytes of a file."

src/multipart.jl

@@ -2,14 +2,14 @@
 #
 # Why this exists: HTTP.jl's `parse_multipart_form` takes the *complete* request
 # body as a byte vector, so using it means every file in the request sits in
-# memory at once — and is then copied again per part. That contradicts the whole
+# memory at once, and is then copied again per part. That contradicts the whole
 # point of this service: file bytes belong on disk, and only a small reference
 # travels through the queue. So intake needs a parser that never holds a file.
 #
 # This reader walks the body incrementally: it pulls fixed-size chunks off the
 # socket and hands each part's bytes straight to a sink (the spool file). Peak
 # memory per connection is `chunk_bytes` + the boundary length, regardless of how
-# large — or how many — the uploaded files are.
+# large, or how many, the uploaded files are.
 #
 # Interface: two calls in a loop, so the caller keeps ordinary control flow
 # rather than inverting into callbacks.
@@ -29,7 +29,7 @@
 # So the delimiter that *closes* a body is CRLF + "--" + boundary, and the two
 # bytes after it say whether another part follows (CRLF) or the form is over
 # ("--"). Every read is bounded, and the buffer retains only the last
-# `length(delimiter)-1` bytes when no delimiter is found — that tail is what
+# `length(delimiter)-1` bytes when no delimiter is found; that tail is what
 # makes a delimiter split across two chunks parse correctly.
 
 "Default socket read size, and therefore the memory bound per in-flight upload."
@@ -91,8 +91,8 @@ end
     multipart_boundary(content_type) -> String | nothing
 
 Pull the boundary out of a `multipart/form-data` Content-Type header. Returns
-`nothing` if the header is missing, is some other media type, or has no boundary
+`nothing` if the header is missing, is some other media type, or has no boundary.
-— all of which are the same 400 to a caller.
+All of those are the same 400 to a caller.
 """
 function multipart_boundary(content_type::Union{AbstractString,Nothing})
     content_type === nothing && return nothing
@@ -139,7 +139,7 @@ function seek_needle!(r::MultipartReader, needle::Vector{UInt8}; limit::Int = 0)
         idx === nothing || return idx
         # Only the last length(needle)-1 bytes can still be part of a match, but
         # the caller may need the skipped bytes (a part body), so trimming is the
-        # caller's job — we only enforce the optional limit.
+        # caller's job; we only enforce the optional limit.
         limit > 0 && navail(r) > limit &&
             throw(MultipartError("no delimiter within $limit bytes"))
         compact!(r)
@@ -174,7 +174,7 @@ end
 
 Advance to the next part and return its headers, or `nothing` at the end of the
 form. The previous part's body must have been consumed first (with
-[`write_part_body!`](@ref) or [`skip_part_body!`](@ref)) — the reader cannot skip
+[`write_part_body!`](@ref) or [`skip_part_body!`](@ref)). The reader cannot skip
 a body it hasn't been told to, because the body is only bounded by finding the
 next delimiter.
 """
@@ -226,7 +226,7 @@ function read_part_headers!(r::MultipartReader)
         r.pos += 2
         return MultipartPart(nothing, nothing, nothing)
     end
-    # The `limit` here bounds *buffering* — it only fires when the headers span
+    # The `limit` here bounds *buffering*, and only fires when the headers span
     # chunks. The explicit length check below is the actual policy, so the rule
     # doesn't depend on how the body happened to be chunked on the wire.
     idx = seek_needle!(r, CRLFCRLF; limit = MAX_PART_HEADER_BYTES)

src/queue.jl

@@ -1,13 +1,36 @@
 # The queue seam.
 #
-# The rest of the app only ever calls `enqueue!`, `dequeue!`, and `close!`.
+# The rest of the app only ever calls `enqueue!`, `dequeue!`, `ack!`, `nack!`
-# Today those are backed by an in-process, bounded, thread-safe buffer
+# and `close!`. Two implementations sit behind those five methods:
-# (the Go-channel / Julia-`Channel` model). To move to RabbitMQ (or any broker)
+#
-# later, implement a new `JobQueue` subtype with these three methods and swap
+#   ChannelQueue  in-process, bounded, thread-safe (the Go-channel / Julia-
-# the construction in `run` — no HTTP handler or worker code needs to change.
+#                 `Channel` model). Lost on crash: everything still in `spool/`
+#                 is re-driven from stage 1 by `recover_dir!`.
+#   RabbitQueue   durable RabbitMQ queues (src/rabbit.jl). Survives a crash: a
+#                 job stays unacked until its handler commits, so a restart
+#                 resumes each file at the stage it had actually reached.
+#
+# `FS_QUEUE_BACKEND` picks one; `run` constructs accordingly. No HTTP handler or
+# worker code knows which it got.
+#
+# The ack pair is what makes the broker worth having. Without it a delivery is
+# settled the moment it is handed to a worker, so a crash mid-handler loses the
+# job exactly as the in-process queue does, and the network hop buys nothing.
+# `ChannelQueue` implements both as no-ops, which is honest rather than lazy:
+# an in-process queue has no delivery to settle, and its recovery story is
+# `recover_dir!`.
 
 abstract type JobQueue end
 
+# How long a producer backs off before retrying an enqueue onto a full queue.
+# Blocking backpressure: a file that is already on disk is never dropped, so the
+# producer parks until the consumer makes room. Used by the stage-1 and stage-3
+# routing handoffs and by startup recovery (see `enqueue_blocking!`). Intake is
+# deliberately *not* a user: the HTTP path's `enqueue!` stays non-blocking and
+# turns a full queue into a 503, so a saturated pipeline slows uploads down
+# rather than holding request threads hostage.
+const ROUTE_ENQUEUE_RETRY_SECONDS = 0.05
+
 """
     enqueue!(q, job) -> Bool
 
@@ -21,10 +44,35 @@ function enqueue! end
     dequeue!(q) -> Union{Job,Nothing}
 
 Blocks until a job is available and returns it. Returns `nothing` only when the
-queue has been closed *and* fully drained — the signal for a worker to exit.
+queue has been closed *and* fully drained, which is a worker's signal to exit.
 """
 function dequeue! end
 
+"""
+    ack!(q, job)
+
+Settle `job` as done: the broker may forget it. Called by `worker_loop` after
+the handler returns, and after a quarantine, so the message outlives the process
+for exactly as long as the work is unfinished.
+
+No-op for `ChannelQueue`.
+"""
+function ack! end
+
+"""
+    nack!(q, job)
+
+Settle `job` as rejected, without requeueing. Requeueing is deliberately not
+offered: a job that failed on its bytes will fail again, and redelivering it is
+a poison-message loop. With no dead-letter exchange configured the broker simply
+discards it, so this differs from `ack!` only in what it says, not in what
+happens — which is the point at the one call site that uses it (see
+`worker_loop`).
+
+No-op for `ChannelQueue`.
+"""
+function nack! end
+
 """
     close!(q)
 
@@ -70,6 +118,10 @@ function dequeue!(q::ChannelQueue)::Union{Job,Nothing}
     end
 end
 
+# Nothing to settle: an in-process job was never a delivery.
+ack!(::ChannelQueue, ::Job) = nothing
+nack!(::ChannelQueue, ::Job) = nothing
+
 function close!(q::ChannelQueue)
     lock(q.cond)
     try

src/rabbit.jl

@@ -0,0 +1,512 @@
+# RabbitMQ-backed `JobQueue` (src/queue.jl defines the seam this implements).
+#
+# What it buys, and what it doesn't
+# ---------------------------------
+# The in-process `ChannelQueue` dies with the process, so a crash leaves no
+# record of how far each file had got and `recover_dir!` can only re-drive
+# everything in `spool/` from stage 1. Durable queues plus per-job acks replace
+# that guess with a fact: a job stays unacked until its handler commits, so a
+# restart resumes each file at the stage it had actually reached, and no file is
+# processed from scratch just because the process died.
+#
+# It is *at-least-once*, not exactly-once. Stages 1 and 3 publish downstream and
+# then ack upstream (`worker_loop`), so a crash in that window redelivers a job
+# that was already routed, and the file gets processed twice. That is safe here
+# for the same reason replay was: classification and the UTF-8 sniff are pure
+# functions of the file's bytes, and every terminal commit renames with
+# `force=true`. The one new wrinkle is that a duplicate can now run *concurrently*
+# with the original, and the loser finds the file already moved; `worker_loop`
+# recognises that (a vanished `job.path`) and settles it quietly instead of
+# quarantining a file that in fact succeeded.
+#
+# It also does not buy horizontal scale. `Job.path` is a local filesystem path,
+# so a second consumer process on another host would be handed jobs whose files
+# it cannot see. See "Queue backends" in the README.
+#
+# Shape
+# -----
+# One connection for the whole process, and per stage queue:
+#
+#   con_chan   consumes, with `basic_qos` prefetch bounding unacked messages,
+#              and carries this stage's acks (a delivery tag is only meaningful
+#              on the channel that delivered it)
+#   pub_chan   publishes *into* this queue, from whichever stage feeds it
+#   buffer     a bounded `Channel{Job}` bridging AMQPClient's push consumer to
+#              the blocking `dequeue!` that `worker_loop` expects
+#
+# plus one shared admin channel for the depth polls. Publishing and acking from
+# many worker tasks at once is safe: AMQPClient serializes multi-frame publishes
+# under a per-channel send lock and queues frames onto a thread-safe channel.
+
+using AMQPClient
+
+# The stage queues, in `STAGE_KEYS` order. The suffix is what appears after
+# `FS_AMQP_PREFIX`, so a broker's queue list reads like the pipeline.
+const AMQP_QUEUE_SUFFIXES = (classify = "classify", enrich = "enrich",
+                             triage = "triage", language = "language")
+
+# Startup connect: retry a broker that isn't up yet (compose starts the app long
+# before RabbitMQ is ready) but give up rather than hang, so a genuinely
+# misconfigured URL is a startup failure and not a silent hang.
+const AMQP_CONNECT_RETRY_SECONDS = 30.0
+const AMQP_CONNECT_RETRY_INTERVAL = 2.0
+
+# How often the depth poller asks the broker for each queue's ready-message
+# count. This is what `/stats` reports and what the advisory capacity check
+# compares against, so it trades staleness for one round trip per queue per
+# second instead of one per file.
+const AMQP_DEPTH_POLL_SECONDS = 1.0
+
+# A publisher confirm that never arrives must not park an HTTP request thread
+# forever. The poller doubles as the watchdog (see `poll_loop!`).
+const AMQP_CONFIRM_TIMEOUT_SECONDS = 10.0
+
+"Connection parameters parsed out of an `amqp://` URL."
+struct AMQPTarget
+    host::String
+    port::Int
+    virtualhost::String
+    login::String
+    password::String
+end
+
+"""
+    parse_amqp_url(url) -> AMQPTarget
+
+Parse `amqp://user:pass@host:port/vhost`. Every component is optional and falls
+back to the AMQP defaults, except that we do *not* default the credentials to
+`guest`/`guest`: those only authenticate over loopback in a stock RabbitMQ, so
+inheriting them across a container network would fail at connect time with an
+authentication error that says nothing about the real mistake.
+
+The vhost is the path with its leading `/` removed and percent-escapes decoded,
+so the conventional default vhost `/` can be written either as a bare trailing
+slash or as the `%2F` form.
+"""
+function parse_amqp_url(url::AbstractString)
+    uri = HTTP.URI(url)
+    scheme = lowercase(uri.scheme)
+    scheme in ("amqp", "amqps") ||
+        throw(ArgumentError("FS_AMQP_URL must be an amqp:// or amqps:// URL, got \"$url\""))
+    scheme == "amqps" &&
+        throw(ArgumentError("amqps:// (TLS) is not supported yet; use amqp:// on a trusted network"))
+
+    userinfo = split(uri.userinfo, ':'; limit = 2)
+    login    = isempty(userinfo[1]) ? "guest" : HTTP.unescapeuri(userinfo[1])
+    password = length(userinfo) < 2 ? "guest" : HTTP.unescapeuri(userinfo[2])
+
+    path  = lstrip(uri.path, '/')
+    vhost = isempty(path) ? "/" : HTTP.unescapeuri(path)
+
+    return AMQPTarget(isempty(uri.host) ? "localhost" : uri.host,
+                      isempty(uri.port) ? AMQPClient.AMQP_DEFAULT_PORT : parse(Int, uri.port),
+                      vhost, login, password)
+end
+
+"The process's single broker connection, plus the shared channel depth polls use."
+mutable struct RabbitBackend
+    conn::AMQPClient.Connection
+    admin_chan::AMQPClient.MessageChannel
+    const admin_lock::ReentrantLock   # one synchronous RPC on admin_chan at a time
+    const prefix::String
+end
+
+"""
+    RabbitQueue(backend, name, capacity, prefetch; confirms = false)
+
+One durable queue. `capacity` is *advisory*: `enqueue!` compares it against a
+depth that is polled once a second and adjusted locally in between, so a burst
+can overshoot by up to a poll interval before the queue starts refusing. The
+alternative — `x-max-length` with `overflow: reject-publish` — needs a publisher
+confirm per message to learn about the rejection, which would put a round trip
+on an inter-stage handoff that currently costs 0.12 µs.
+
+`confirms` puts this queue's *publish* channel into confirm mode, so `enqueue!`
+returns only once the broker has accepted the message. It is set on the stage-1
+queue alone, because intake is the only publisher that answers to someone: a
+`202` tells a client its file is safe, and that client may delete its copy.
+Inter-stage publishes stay fire-and-forget, where a lost message leaves its
+source job unacked and redelivery repairs it for free.
+"""
+mutable struct RabbitQueue <: JobQueue
+    const backend::RabbitBackend
+    const name::String
+    const capacity::Int
+    const buffer::Channel{Job}                 # consumer callback → dequeue!
+    const pub_chan::AMQPClient.MessageChannel
+    const con_chan::AMQPClient.MessageChannel
+    const depth::Threads.Atomic{Int}           # last polled ready count, locally adjusted
+    const closed::Threads.Atomic{Bool}
+    const confirms::Bool
+    const pub_lock::ReentrantLock              # serializes publish+confirm on pub_chan
+    const confirm_ch::Channel{Bool}            # broker's verdict on the outstanding publish
+    const pending_since::Threads.Atomic{Float64}  # when it was published (0.0 = nothing pending)
+    const on_lost::Function                    # called once if the connection dies
+    consumer_tag::String
+    poller::Union{Task,Nothing}
+end
+
+capacity(q::RabbitQueue) = q.capacity
+
+"""
+Depth as of the last poll, adjusted by our own publishes and takes since.
+
+The broker's number counts *ready* messages, so it excludes the ones already
+handed to workers and not yet acked. That is the right number for `/stats`:
+depth is meant to say how much work is piled up in front of a stage, and the
+messages in its workers' hands are not piled up, they are being worked.
+"""
+Base.length(q::RabbitQueue) = max(0, q.depth[])
+
+# --- wire format ------------------------------------------------------------
+
+"""
+    job_json(job) -> String
+
+Serialize a `Job` for the wire. `delivery_tag` is deliberately absent: it names a
+delivery on one channel of one connection, not the job, and a redelivery gets a
+fresh one.
+"""
+job_json(job::Job) = JSON3.write((; job.id, job.original_name, job.path,
+                                   job.size, job.received_at))
+
+"Rebuild a `Job` from a message body, carrying the tag it was delivered under."
+function job_from_message(msg::AMQPClient.Message)
+    o = JSON3.read(String(copy(msg.data)))
+    return Job(String(o.id), String(o.original_name), String(o.path),
+               Int(o.size), Float64(o.received_at), Int64(msg.delivery_tag))
+end
+
+# --- connect / declare ------------------------------------------------------
+
+"""
+    connect_backend(cfg) -> RabbitBackend
+
+Open the process's connection, retrying a broker that is not up yet for up to
+`AMQP_CONNECT_RETRY_SECONDS`. Compose starts this container long before RabbitMQ
+finishes booting even with a healthcheck gate, so a first refusal is routine; a
+persistent one is a startup error, because falling back to the in-process queue
+would silently deliver none of the durability that was asked for.
+"""
+function connect_backend(cfg::Config)
+    t = parse_amqp_url(cfg.amqp_url)
+    auth = Dict{String,Any}("MECHANISM" => "AMQPLAIN",
+                            "LOGIN" => t.login, "PASSWORD" => t.password)
+    deadline = time() + AMQP_CONNECT_RETRY_SECONDS
+    attempt = 0
+    while true
+        attempt += 1
+        try
+            conn = AMQPClient.connection(; host = t.host, port = t.port,
+                                           virtualhost = t.virtualhost,
+                                           auth_params = auth)
+            admin = AMQPClient.channel(conn, AMQPClient.UNUSED_CHANNEL, true)
+            @info "connected to broker" host=t.host port=t.port vhost=t.virtualhost attempts=attempt
+            return RabbitBackend(conn, admin, ReentrantLock(), cfg.amqp_prefix)
+        catch e
+            time() < deadline || rethrow()
+            @info "broker not ready, retrying" host=t.host port=t.port attempt=attempt
+            sleep(AMQP_CONNECT_RETRY_INTERVAL)
+        end
+    end
+end
+
+"""
+    open_queue(backend, cfg, key, capacity, workers) -> RabbitQueue
+
+Declare one stage's durable queue and start consuming from it.
+
+Prefetch defaults to the stage's worker count: enough that every worker can hold
+a job, and no more, so a crash redelivers the smallest possible set. The bridge
+`Channel` is sized to match, since it is exactly the set of messages the broker
+has handed us and we have not yet acked.
+"""
+function open_queue(backend::RabbitBackend, cfg::Config, key::Symbol,
+                    capacity::Int, workers::Int; on_lost::Function = () -> nothing)
+    name = string(backend.prefix, ".", AMQP_QUEUE_SUFFIXES[key])
+    prefetch = cfg.amqp_prefetch > 0 ? cfg.amqp_prefetch : max(1, workers)
+
+    con_chan = AMQPClient.channel(backend.conn, AMQPClient.UNUSED_CHANNEL, true)
+    pub_chan = AMQPClient.channel(backend.conn, AMQPClient.UNUSED_CHANNEL, true)
+
+    # durable: the queue definition survives a broker restart. Combined with
+    # PERSISTENT messages below, so do the jobs in it.
+    AMQPClient.queue_declare(con_chan, name; durable = true)
+    AMQPClient.basic_qos(con_chan, 0, prefetch, false)
+
+    confirms = cfg.amqp_confirms && key === :classify
+    q = RabbitQueue(backend, name, capacity, Channel{Job}(prefetch),
+                    pub_chan, con_chan, Threads.Atomic{Int}(0),
+                    Threads.Atomic{Bool}(false), confirms, ReentrantLock(),
+                    Channel{Bool}(1), Threads.Atomic{Float64}(0.0), on_lost, "", nothing)
+
+    if confirms
+        confirm_select!(pub_chan)
+        # The broker reports each publish's fate asynchronously by sequence
+        # number. We hold `pub_lock` across publish-and-wait, so at most one
+        # publish is ever outstanding and any verdict that arrives is this one's.
+        AMQPClient.handle(pub_chan, :Basic, :Ack, (_c, _m, _x) -> settle_confirm!(q, true))
+        AMQPClient.handle(pub_chan, :Basic, :Nack, (_c, _m, _x) -> settle_confirm!(q, false))
+    end
+
+    ok, tag = AMQPClient.basic_consume(con_chan, name, msg -> deliver!(q, msg))
+    ok || error("could not start consuming from queue $name")
+    q.consumer_tag = tag
+    q.poller = Threads.@spawn poll_loop!(q)
+
+    @info "queue ready" queue=name prefetch=prefetch capacity=capacity confirms=confirms
+    return q
+end
+
+"""
+    open_queues(backend, cfg) -> NamedTuple
+
+The four stage queues, keyed by `STAGE_KEYS` so they line up with the per-stage
+worker counts, capacities and metrics without a second mapping to keep in sync.
+"""
+function open_queues(backend::RabbitBackend, cfg::Config; on_lost::Function = () -> nothing)
+    caps    = (classify = cfg.queue_capacity, enrich = cfg.known_queue_capacity,
+               triage = cfg.unknown_queue_capacity, language = cfg.text_queue_capacity)
+    workers = (classify = cfg.worker_count, enrich = cfg.known_worker_count,
+               triage = cfg.unknown_worker_count, language = cfg.text_worker_count)
+    return NamedTuple{STAGE_KEYS}(map(STAGE_KEYS) do key
+        open_queue(backend, cfg, key, caps[key], workers[key]; on_lost)
+    end)
+end
+
+"Close the shared connection. Safe to call on a connection that is already gone."
+function close_backend!(backend::RabbitBackend)
+    try
+        close(backend.conn)
+    catch e
+        @debug "error closing broker connection" exception=e
+    end
+    return nothing
+end
+
+"""
+    confirm_select!(chan)
+
+Put `chan` into publisher-confirm mode.
+
+Not `AMQPClient.confirm_select`, which is broken in AMQPClient 0.5.1: it builds
+the `Confirm.Select` payload with no fields, while the spec (and the library's
+own `CLASS_MAP`) declares a `Nowait` bit, so it throws a `BoundsError` before
+anything reaches the wire. This is the same call with that one field supplied.
+Drop it for the library's version once the fix is released.
+"""
+function confirm_select!(chan::AMQPClient.MessageChannel)
+    AMQPClient._wait_resp(chan, true, false, AMQPClient.on_confirm_select_ok,
+                          :Confirm, :SelectOk, false, AMQPClient.DEFAULT_TIMEOUT) do
+        AMQPClient.send(chan, AMQPClient.TAMQPMethodPayload(:Confirm, :Select, (false,)))
+    end
+end
+
+# --- consume ----------------------------------------------------------------
+
+"""
+Consumer callback: hand a delivery to the bridge channel.
+
+Blocking here is the mechanism, not a problem. The bridge holds `prefetch` jobs,
+so when workers fall behind this parks, the broker's prefetch window fills, and
+it stops sending — backpressure all the way to the queue, with no unbounded
+local buffer.
+"""
+function deliver!(q::RabbitQueue, msg::AMQPClient.Message)
+    try
+        put!(q.buffer, job_from_message(msg))
+        Threads.atomic_sub!(q.depth, 1)
+    catch e
+        # Either we are shutting down (buffer closed after `basic_cancel`, and a
+        # delivery raced it) or the body was unparseable. Neither may kill the
+        # consumer task. The message stays unacked, so the broker redelivers it.
+        q.closed[] || @error "could not accept delivery" queue=q.name exception=(e, catch_backtrace())
+    end
+    return nothing
+end
+
+function dequeue!(q::RabbitQueue)::Union{Job,Nothing}
+    try
+        return take!(q.buffer)
+    catch e
+        # Closed and drained: the shutdown signal `worker_loop` waits for.
+        e isa InvalidStateException && return nothing
+        rethrow()
+    end
+end
+
+"""
+Acks and rejects go out on the channel that made the delivery, because a
+delivery tag is scoped to its channel.
+
+A failure here is logged, not thrown: an unacked job is redelivered, which is
+the correct outcome, and killing the worker over it would shrink the pool.
+"""
+function ack!(q::RabbitQueue, job::Job)
+    job.delivery_tag == 0 && return nothing
+    try
+        AMQPClient.basic_ack(q.con_chan, job.delivery_tag)
+    catch e
+        @error "ack failed; job will be redelivered" queue=q.name id=job.id exception=e
+    end
+    return nothing
+end
+
+function nack!(q::RabbitQueue, job::Job)
+    job.delivery_tag == 0 && return nothing
+    try
+        AMQPClient.basic_reject(q.con_chan, job.delivery_tag; requeue = false)
+    catch e
+        @error "reject failed; job will be redelivered" queue=q.name id=job.id exception=e
+    end
+    return nothing
+end
+
+# --- publish ----------------------------------------------------------------
+
+"""
+    enqueue!(q, job) -> Bool
+
+Publish `job` as a persistent message. Returns `false` — which intake turns into
+a `503` and the routing stages turn into park-and-retry — when the queue is
+closed, when the polled depth says it is at capacity, or when a confirm-mode
+publish is not confirmed.
+
+`delivery_mode = PERSISTENT` against a durable queue is what makes a broker
+restart survivable; without it the queue would come back empty and the whole
+point of the backend would be gone.
+"""
+function enqueue!(q::RabbitQueue, job::Job)::Bool
+    q.closed[] && return false
+    q.depth[] >= q.capacity && return false   # advisory: see the `RabbitQueue` docstring
+
+    msg = AMQPClient.Message(Vector{UInt8}(job_json(job));
+                             content_type = "application/json",
+                             delivery_mode = AMQPClient.PERSISTENT)
+    ok = q.confirms ? publish_confirmed!(q, msg) : publish!(q, msg)
+    ok && Threads.atomic_add!(q.depth, 1)
+    return ok
+end
+
+function publish!(q::RabbitQueue, msg::AMQPClient.Message)
+    try
+        AMQPClient.basic_publish(q.pub_chan, msg; exchange = "", routing_key = q.name)
+        return true
+    catch e
+        @error "publish failed" queue=q.name exception=(e, catch_backtrace())
+        return false
+    end
+end
+
+"""
+Publish and wait for the broker's confirm.
+
+Serialized on `pub_lock`, so exactly one publish is outstanding per queue and the
+next verdict to arrive belongs to it. That costs intake a round trip per file and
+caps its rate at roughly one file per broker round trip; it is why confirms are
+scoped to intake and configurable (`FS_AMQP_CONFIRMS`).
+"""
+function publish_confirmed!(q::RabbitQueue, msg::AMQPClient.Message)
+    lock(q.pub_lock) do
+        while isready(q.confirm_ch)   # discard a verdict left by a timed-out publish
+            take!(q.confirm_ch)
+        end
+        q.pending_since[] = time()
+        try
+            publish!(q, msg) || return false
+            return take!(q.confirm_ch)   # poll_loop! guarantees this is answered
+        finally
+            q.pending_since[] = 0.0
+        end
+    end
+end
+
+"Record the broker's verdict for the one outstanding publish."
+function settle_confirm!(q::RabbitQueue, ok::Bool)
+    q.pending_since[] == 0.0 && return nothing   # nothing waiting; a late/duplicate frame
+    isready(q.confirm_ch) || put!(q.confirm_ch, ok)
+    return nothing
+end
+
+# --- poll -------------------------------------------------------------------
+
+"""
+    poll_loop!(q)
+
+One task per queue, doing the two things that must happen on a timer:
+
+  * refresh `depth` from the broker (a passive `queue.declare` returns the ready
+    count), which feeds `/stats` and the advisory capacity check, and
+  * answer a publisher confirm that has gone missing, so a broker that accepted
+    a publish and then went quiet can't park an HTTP request thread forever, and
+  * notice that the connection has died and call `on_lost`, which asks `run` to
+    shut down. This backend does not reconnect (every in-flight delivery tag
+    would be invalid, and the jobs holding them would be processed twice), so a
+    lost connection means the process is finished: it drains, exits, and the
+    restart policy brings it back to redelivered messages and an intact
+    `spool/`. Without this the pipeline would sit there looking healthy with
+    four queues that can never receive anything again.
+
+Folding the watchdog in here keeps it free: it is a comparison on a task that was
+going to wake up anyway.
+"""
+function poll_loop!(q::RabbitQueue)
+    while !q.closed[]
+        sleep(AMQP_DEPTH_POLL_SECONDS)
+        q.closed[] && break
+
+        pending = q.pending_since[]
+        if pending != 0.0 && time() - pending > AMQP_CONFIRM_TIMEOUT_SECONDS
+            @warn "publisher confirm timed out; reporting the file as not accepted" queue=q.name
+            settle_confirm!(q, false)
+        end
+
+        try
+            lock(q.backend.admin_lock) do
+                ok, _, count, _ = AMQPClient.queue_declare(q.backend.admin_chan, q.name;
+                                                           passive = true)
+                ok && (q.depth[] = Int(count))
+            end
+        catch e
+            q.closed[] && break
+            # A dead connection is not something this backend tries to ride out:
+            # every in-flight delivery tag becomes invalid on reconnect, so the
+            # honest recovery is a restart, where unacked messages redeliver and
+            # `spool/` is intact. Report it and let the supervisor act.
+            if !isopen(q.backend.conn)
+                # Two causes, indistinguishable from here: the broker or the
+                # network went away, or a SIGINT was delivered to one of
+                # AMQPClient's tasks instead of the main loop (see `run`).
+                # The response is the same either way — drain and exit.
+                @error "broker connection closed (lost broker, or an interrupt delivered to a broker task); shutting down (unacked jobs will be redelivered)" queue=q.name
+                q.on_lost()
+                break
+            end
+            @warn "queue depth poll failed" queue=q.name exception=e
+        end
+    end
+    return nothing
+end
+
+# --- shutdown ---------------------------------------------------------------
+
+"""
+    close!(q)
+
+Stop consuming, then close the bridge so workers drain what is already in hand
+and exit — the same drain-then-exit contract `ChannelQueue` offers.
+
+Jobs the broker has delivered but that we never got to stay unacked and are
+redelivered on the next start. That is the backend working as intended, and the
+reason shutdown does not need to be careful here.
+"""
+function close!(q::RabbitQueue)
+    q.closed[] || try
+        AMQPClient.basic_cancel(q.con_chan, q.consumer_tag)
+    catch e
+        @debug "error cancelling consumer" queue=q.name exception=e
+    end
+    q.closed[] = true
+    isopen(q.buffer) && close(q.buffer)
+    return nothing
+end

src/server.jl

@@ -2,13 +2,13 @@
 #
 # The handler's whole job is to get files onto the queue fast and get out of the
 # way: stream each uploaded file to disk, enqueue a reference, respond 202. It
-# never does real processing — that's the workers' job.
+# never does real processing; that's the workers' job.
 #
 # Intake is *streamed*, not buffered (see src/multipart.jl for the reader). This
 # is what makes the service's memory story hold end to end: bytes go from the
 # socket to the spool file a chunk at a time, so a 4 GB upload costs the same
 # resident memory as a 4 KB one. It is also why /upload is served by its own
-# stream handler rather than an Oxygen route — see `root_stream_handler`.
+# stream handler rather than an Oxygen route; see `root_stream_handler`.
 #
 # NOTE: routes are registered at runtime via `register_routes()` (called from
 # `run`), NOT with top-level macros. In a precompiled package, top-level
@@ -24,10 +24,10 @@ function health_handler(_::HTTP.Request)
 end
 
 """
-`GET /stats` — the pipeline's own counters (src/stats.jl), as JSON.
+`GET /stats`: the pipeline's own counters (src/stats.jl), as JSON.
 
 Read-only and cheap: a few atomic loads and one `length` per queue, no pipeline
-state touched. Two scrapes Δt apart give per-stage throughput and utilization —
+state touched. Two scrapes Δt apart give per-stage throughput and utilization;
 see bin/bench.jl, which is the intended consumer.
 
 Unlike `/upload` this is an ordinary Oxygen route: it has no body to stream, and
@@ -54,7 +54,7 @@ end
 Read and discard whatever is left of the request body.
 
 HTTP.jl's server calls `closeread` after the handler and *errors* if the body was
-only partly consumed — a half-read body can't be followed by another request on a
+only partly consumed, because a half-read body can't be followed by another request on a
 keep-alive connection. So every exit path drains first. Discarding is bounded in
 memory (one chunk) and costs nothing in the normal case, where the body is
 already fully consumed and this returns immediately.
@@ -72,7 +72,7 @@ Did this exception mean the client went away, rather than something being wrong
 on our side?
 
 A client that hangs up mid-upload (user cancels, network drops) is routine, and
-must not be logged as a server error or reported as a failed write — but it looks
+must not be logged as a server error or reported as a failed write. But it looks
 like an I/O failure from inside the parser, so the distinction has to be made
 explicitly. `EOFError` is what `HTTP.Stream` raises when a connection dies with
 bytes still promised by `Content-Length`.
@@ -87,22 +87,22 @@ Stream a multipart upload to disk, one part at a time.
 Each file part is spooled straight from the socket and its reference enqueued.
 The response reports what was actually queued:
 
-  * `202` — every file in the request was spooled and queued
+  * `202`: every file in the request was spooled and queued
-  * `400` — not multipart/form-data, no files present, or a malformed body
+  * `400`: not multipart/form-data, no files present, or a malformed body
-  * `503` — the intake queue filled up; `accepted` lists what got in first
+  * `503`: the intake queue filled up; `accepted` lists what got in first
-  * `500` — a file could not be written to disk
+  * `500`: a file could not be written to disk
 
 Unlike a buffered handler, this one cannot know up front how many files a request
 holds or whether they will fit. So when the queue fills mid-request it does not
 abandon the connection: it stops spooling (discarding the remaining parts rather
 than writing files it can't queue), drains the body, and answers `503` with the
-`accepted` list. Files already queued stay queued — a client can retry the rest.
+`accepted` list. Files already queued stay queued, so a client can retry the rest.
 """
 function upload_stream_handler(stream::HTTP.Stream)
     try
         return serve_upload!(stream)
     catch e
-        # The client vanished — while we were reading its body, draining it, or
+        # The client vanished while we were reading its body, draining it, or
         # answering. Nothing is wrong on our side, so log it as the routine event
         # it is instead of a server error.
         is_client_gone(e) || rethrow()
@@ -163,7 +163,7 @@ function serve_upload!(stream::HTTP.Stream)
             catch e
                 (e isa MultipartError || is_client_gone(e)) && rethrow()
                 # A disk error leaves the reader mid-part, so the rest of the body
-                # can no longer be parsed — stop and report.
+                # can no longer be parsed, so stop and report.
                 @error "spool failed" name=part.filename exception=(e, catch_backtrace())
                 failure = (500, "failed to store file")
                 break
@@ -203,13 +203,13 @@ end
     root_stream_handler(middleware) -> (stream -> nothing)
 
 Oxygen's root handler wraps `HTTP.streamhandler`, which does
-`request.body = read(stream)` — the entire body into memory — *before* any route
+`request.body = read(stream)`, the entire body into memory, *before* any route
 is dispatched. That happens even for an Oxygen `@stream` route, so no route can
 stream an upload. We therefore intercept `POST /upload` at the stream level and
 hand everything else to Oxygen unchanged.
 
 Trade-off: `/upload` bypasses Oxygen's middleware chain, so it is absent from
-Oxygen's built-in metrics and docs. Deliberate — flat intake memory is the point
+Oxygen's built-in metrics and docs. Deliberate: flat intake memory is the point
 of this service, and intake is covered by our own counters (`/stats`) and `@info`
 records anyway.
 """

src/spool.jl

@@ -33,7 +33,7 @@ and build the `Job` from however many bytes it reports writing.
 
 This is the streaming counterpart to `spool_file`: the caller pumps bytes in from
 the network as they arrive, so a file never exists in memory in one piece. A
-partial file left by a failed or abandoned write is removed — intake either
+partial file left by a failed or abandoned write is removed: intake either
 produces a complete spooled file or nothing at all, so recovery on restart never
 picks up a truncated upload.
 """
@@ -48,7 +48,15 @@ function spool_stream(write_body!, cfg::Config, original_name::AbstractString)::
     return Job(id, String(original_name), path, nbytes, time())
 end
 
-"Move a spooled file into `dir` (done/ or failed/), returning the destination."
+"""
+    move_to(dir, job) -> String
+
+Move a spooled file into `dir`, returning the destination.
+
+This runs once per file, at the end: into a terminal sink (`done/`, `text_done/`,
+`binary/`) or into `failed/` when a handler throws. Files are *not* moved between
+stages; see the header of src/worker.jl.
+"""
 function move_to(dir::AbstractString, job::Job)::String
     dest = joinpath(dir, basename(job.path))
     mv(job.path, dest; force=true)
@@ -64,12 +72,24 @@ const UUID_LEN = 36
 
 Re-enqueue any files sitting in `dir` (left by a crash, hard shutdown, or an
 intake that never finished) onto `queue`. This is the payoff of spooling to
-disk: a restart resumes work instead of stranding it. Stage-aware recovery uses
+disk: a restart resumes work instead of stranding it. Returns the number of
-one call per stage — `spool/` → stage-1 queue, `known/` → known queue — so each
+files recovered.
-file re-enters at the correct stage rather than being reclassified from scratch.
-Returns the number of files recovered.
 
-Skips `.meta.json` sidecars: those are stage-2 output, not work to redo.
+A file's stage is not recorded on disk. It lives in whichever in-process queue
+holds its reference, and a crash loses that (src/worker.jl header). So everything
+still in `spool/` re-enters at stage 1 and replays. Replay is safe rather than
+merely tolerable: classification and the binary/text sniff are pure functions of
+the file's bytes, and the terminal commits rename with `force=true`, so a
+replayed file lands where it would have landed and overwrites its own sidecar.
+
+**This blocks on a full queue**, and that is the whole point: `spool/` now holds
+every in-flight file, so a crash under load can easily leave more leftovers than
+one queue's capacity. Dropping the excess would mean recovery quietly losing the
+work it exists to save. Because the caller spawns the worker pools *before*
+recovering, the consumers are live and the queue drains as we fill it, so parking
+here costs latency, never progress.
+
+Skips `.meta.json` sidecars: those are enrichment output, not work to redo.
 """
 function recover_dir!(dir::AbstractString, queue::JobQueue)::Int
     n = 0
@@ -85,7 +105,10 @@ function recover_dir!(dir::AbstractString, queue::JobQueue)::Int
             name = fname
         end
         job = Job(id, name, path, filesize(path), time())
-        enqueue!(queue, job) || @warn "queue full during recovery; leaving file for next start" path
+        # `nothing` stats: recovery is not a stage, so its wait must not be
+        # charged to any stage's blocked_ns (see `enqueue_blocking!`).
+        enqueue_blocking!(queue, job, nothing;
+                          retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
         n += 1
     end
     return n

src/stats.jl

@@ -1,10 +1,12 @@
 # Per-stage pipeline metrics.
 #
 # End-to-end throughput says how fast the pipeline is; it does not say which
-# stage is the reason. The external harness can't answer that either: `known/`,
+# stage is the reason. The external harness can't answer that either, and now
-# `unknown/` and `text/` are *transient* — a file can pass through one between
+# cannot even approximate it: every in-flight file sits in `spool/` regardless of
-# two directory polls — so sampling directories from outside undercounts exactly
+# how far it has got, because stages route by enqueueing a reference rather than
-# the stages we most want to measure. Only the pipeline itself sees every job.
+# by moving bytes (src/worker.jl header). A file's stage is a property of the
+# queue holding it, so only the pipeline itself can see it. There is no
+# directory to poll.
 #
 # So each stage keeps four counters, all incremented in `worker_loop` (the one
 # place every stage's work passes through) and read by `GET /stats`:
@@ -21,15 +23,15 @@
 #   utilization = (Δbusy_ns - Δblocked_ns) / (Δt * workers)   ~1.0 ⇒ bottleneck
 #
 # Utilization is the useful one. Throughput alone can't distinguish a stage that
-# is saturated from one that is merely starved by the stage ahead of it — both
+# is saturated from one that is merely starved by the stage ahead of it: both
-# show the same files/s — whereas utilization separates them: the saturated stage
+# show the same files/s. Utilization separates them: the saturated stage
 # sits near 1.0 with its queue backing up, the starved stage sits near 0.
 #
 # `blocked_ns` is what keeps that true downstream. Stages 1 and 3 apply blocking
 # backpressure: when the next queue is full the handler parks and retries rather
 # than dropping the file (see ROUTE_ENQUEUE_RETRY_SECONDS). That parked time is
 # inside the handler, so counting it as busy would show stage 1 pinned at 1.0
-# whenever stage 2 is the real bottleneck — every stage upstream of the jam would
+# whenever stage 2 is the real bottleneck, and every stage upstream of the jam would
 # look like the jam. Subtracting it leaves utilization meaning "doing its own
 # work", and the blocked share becomes its own signal: a stage blocked 90% of the
 # time is naming its successor as the bottleneck.
@@ -39,7 +41,7 @@
 # a scrape is stateless and two readers can't disturb each other.
 #
 # Cost is a handful of atomic adds per file, against handlers that spawn
-# exiftool — unmeasurable in practice, and the counters are never read on the
+# exiftool: unmeasurable in practice, and the counters are never read on the
 # hot path.
 
 # Stage identity lives here, in declaration order, so the report, the JSON and
@@ -53,7 +55,7 @@ const STAGE_TITLES = (classify = "classify", enrich = "enrich",
 Counters for one pipeline stage. All fields are atomic and monotonic: workers
 only ever add, readers only ever read, so no lock is needed between them.
 
-`in_flight` is the exception to monotonic — it goes up and down — and is the one
+`in_flight` is the exception to monotonic. It goes up and down, and is the one
 counter that is a *level* rather than a total.
 """
 struct StageStats
@@ -72,8 +74,8 @@ StageStats() = StageStats(Threads.Atomic{Int}(0), Threads.Atomic{Int}(0),
 """
 Counters for HTTP intake, which has no worker loop to hang them off.
 
-`files` counts what was actually spooled *and* queued — the same thing the 202
+`files` counts what was actually spooled *and* queued, the same thing the 202
-body reports as `accepted` — so it lines up with stage 1's arrivals. Files
+body reports as `accepted`, so it lines up with stage 1's arrivals. Files
 dropped because the queue was full are counted separately in `rejected`, since a
 run where intake outruns the pipeline should be visible as backpressure rather
 than as slow intake.
@@ -94,7 +96,7 @@ from.
 
 `since` matters to a reader computing rates from a single scrape: without it,
 "1000 files completed" has no denominator. Readers that scrape twice should use
-their own Δt instead — it excludes the time before they started watching.
+their own Δt instead, which excludes the time before they started watching.
 """
 struct Metrics
     intake::IntakeStats
@@ -115,7 +117,7 @@ const METRICS = Metrics()
     reset_metrics!()
 
 Zero every counter and restart the measurement window. For tests, and for a
-benchmark that wants totals covering only its own run — though a harness that
+benchmark that wants totals covering only its own run, though a harness that
 scrapes before and after can subtract instead, which is safer against a server
 that is also serving someone else.
 """
@@ -151,18 +153,25 @@ the parked time to `stats.blocked_ns`.
 
 This is the routing half of stages 1 and 3: a classified file is never dropped,
 so a full downstream queue means waiting, not failing. Wrapping the retry loop
-here — rather than repeating `while !enqueue! sleep end` at each call site — is
+here, rather than repeating `while !enqueue! sleep end` at each call site, is
 what makes that wait measurable at all, and keeps the three routing paths from
 drifting into three different backoff behaviours.
+
+`stats` is `nothing` for callers that are not a pipeline stage. Startup
+recovery (`recover_dir!`) uses the same never-drop retry, but it runs outside any
+handler, so it has no `busy_ns` to charge against. Billing its wait to a stage's
+`blocked_ns` would make `/stats` utilization, `(Δbusy_ns - Δblocked_ns) / …`,
+go *negative* after a long recovery. Skipping the charge keeps `blocked_ns`
+meaning exactly what its docs say: time a handler spent parked on its successor.
 """
-function enqueue_blocking!(queue::JobQueue, job::Job, stats::StageStats;
+function enqueue_blocking!(queue::JobQueue, job::Job, stats::Union{StageStats,Nothing};
                            retry_seconds::Real)
     enqueue!(queue, job) && return nothing
     t0 = time_ns()
     while !enqueue!(queue, job)
         sleep(retry_seconds)
     end
-    Threads.atomic_add!(stats.blocked_ns, Int(time_ns() - t0))
+    stats === nothing || Threads.atomic_add!(stats.blocked_ns, Int(time_ns() - t0))
     return nothing
 end
 
@@ -174,7 +183,7 @@ end
 Read every counter into a plain value tree for `GET /stats`.
 
 `queues` is a NamedTuple of `JobQueue`s keyed by `STAGE_KEYS`, supplying each
-stage's live depth and capacity — the counters above are about work *done*, and
+stage's live depth and capacity. The counters above are about work *done*, and
 a stage's depth is what says whether work is piling up in front of it.
 
 The read is not atomic across stages: counters keep moving while we walk them,

src/worker.jl

@@ -3,33 +3,55 @@
 # stage, so `worker_loop` is parametrized with a `handler` and reused. Today
 # there are four stages:
 #
-#   stage 1  handle_classify_job  spool/  → classify → known/ (+known queue) | unknown/ (+unknown queue)
+#   stage 1  handle_classify_job  classify → known queue | unknown queue
-#   stage 2  handle_known_job     known/  → exiftool enrich → done/ (+ .meta.json)
+#   stage 2  handle_known_job     exiftool enrich → done/ (+ .meta.json)
-#   stage 3  handle_unknown_job   unknown/ → binary-vs-text sniff → binary/ | text/ (+text queue)
+#   stage 3  handle_unknown_job   binary-vs-text sniff → binary/ | text queue
-#   stage 4  handle_text_job      text/   → language enrich → text_done/ (+ .meta.json)
+#   stage 4  handle_text_job      language enrich → text_done/ (+ .meta.json)
 #
 # Adding a stage later is just another queue + pool + handler; the loop below
 # doesn't change.
-
+#
-# How long a stage-1 worker backs off before retrying an enqueue onto a full
+# A file does not move between stages. It is written once into `spool/` at
-# downstream queue (known or unknown). Blocking backpressure: a classified file
+# intake and stays there for its whole in-flight life; only the small `Job`
-# is never dropped, so the stage-1 worker parks until the next stage makes room.
+# reference travels, and `job.path` is therefore constant from intake until the
-# Keeps intake decoupled — the HTTP path's `enqueue!` stays non-blocking; only
+# file is committed. The stage a file has reached lives in the queue holding its
-# this worker-to-worker handoff blocks.
+# reference, not in which directory the bytes sit, so the intermediate hops
-const ROUTE_ENQUEUE_RETRY_SECONDS = 0.05
+# (spool→known, spool→unknown, unknown→text) are three renames per file that buy
+# nothing on the live path. What remains is one move at the end: into a terminal
+# sink (`done/`, `text_done/`, `binary/`) or into `failed/` on a throw.
+#
+# The cost is on restart, and which cost depends on the backend. Under the
+# in-process `ChannelQueue` a crash loses the queues, and recovery can only
+# re-drive everything in `spool/` from stage 1. That is safe, since
+# classification and the UTF-8 sniff are pure functions of the file's bytes and
+# every commit is idempotent, but it redoes work the old directory-per-stage
+# layout could skip. Under `RabbitQueue` (src/rabbit.jl) it doesn't arise: a job
+# stays unacked until its handler commits, so a restart resumes each file at the
+# stage it had reached. The fix was the queue seam, not the directories.
+#
+# (`ROUTE_ENQUEUE_RETRY_SECONDS`, the backoff the routing handoffs below use,
+# now lives in src/queue.jl, since startup recovery shares it.)
 
 # Per-file logging in stage 1 is `@debug`, not `@info`, because it is the
 # stage's dominant cost. Measured by bin/bench_stage1.jl (2000 x 64 KiB files,
-# min of 5 trials): the two log lines cost ~71 µs of the ~118 µs
+# min of 5 trials): a formatted pair of log lines costs 71.2 µs per file, against
-# `handle_classify_job` spent per file — roughly 6x the classifier (10.6 µs) and
+# the 11.7 µs the whole handler takes with them switched off: six times the rest
-# 6x the rename (11.6 µs). Nearly all of it is `ConsoleLogger` formatting
+# of the stage put together. Nearly all of it is `ConsoleLogger` formatting
-# (~64 µs); the FlushLogger's per-message flush is only ~8 µs on top. Demoting
+# (64.2 µs); the FlushLogger's per-message flush is only ~7 µs on top. Switching
-# them takes stage 1 from ~8.5k files/s to ~35k files/s on one worker.
+# them back on with `JULIA_DEBUG=DarkStruct` takes the handler to 106.3 µs, i.e.
+# from 85.4k files/s down to 9.4k on one worker.
+#
+# With logging off the stage is the classifier and nothing else: classify 10.30 µs
+# (of which read_features is 7.82 µs), the queue handoff 0.12 µs, the disabled
+# `@debug` lines 0.29 µs, for 10.7 µs of the 11.7 µs total. Removing the inter-stage
+# rename is what left it that way: that rename was 11.6 µs per file, so it was
+# the equal of the classifier, and dropping it took one worker from ~35k to
+# ~85k files/s (16 workers: ~333k files/s).
 #
 # `@debug` is compiled to a min-level check that doesn't evaluate its arguments,
 # so a disabled line costs ~0.15 µs rather than ~36 µs. The messages are still
-# there when wanted: run with `JULIA_DEBUG=FileServer` to get them back. Errors,
+# there when wanted: run with `JULIA_DEBUG=DarkStruct` to get them back. Errors,
-# quarantines and lifecycle events stay at `@error`/`@info` — they are rare and
+# quarantines and lifecycle events stay at `@error`/`@info`, since they are rare and
 # their cost doesn't scale with throughput. `GET /stats` (src/stats.jl) is the
 # per-file observability that survives, and it is counted, not formatted.
 
@@ -38,11 +60,13 @@ const ROUTE_ENQUEUE_RETRY_SECONDS = 0.05
 
 Stage 1. Classify the spooled file and route it to the next stage's queue,
 retrying on a full queue rather than dropping the file:
-  * `:known`   → move to `known/`,   enqueue onto the known queue for stage 2.
+  * `:known`   → enqueue onto the known queue for stage 2.
-  * `:unknown` → move to `unknown/`, enqueue onto the unknown queue for stage 3.
+  * `:unknown` → enqueue onto the unknown queue for stage 3.
 
-In both cases move first so the file physically lives in its stage dir before the
+Routing is the enqueue and nothing else: the file stays where intake wrote it and
-reference is visible downstream; the moved path becomes the routed job's location.
+the *same* `Job` is handed on, so `job.path` still points at it. Reaching stage 2
+is a fact about which queue holds the reference, not about which directory holds
+the bytes.
 
 Sub-`MIN_FILE_BYTES` files short-circuit to `:unknown` inside `classify`.
 """
@@ -53,18 +77,14 @@ function handle_classify_job(job::Job, cfg::Config, worker_id::Int,
     @debug "classified file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification
 
     if classification === :known
-        dest = move_to(cfg.known_dir, job)
-        routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
         # known queue full → park and retry, don't drop (time charged to blocked_ns)
-        enqueue_blocking!(known_queue, routed, stats;
+        enqueue_blocking!(known_queue, job, stats;
                           retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
-        @debug "routed to enrichment" worker=worker_id id=job.id dest=dest
+        @debug "routed to enrichment" worker=worker_id id=job.id path=job.path
     else
-        dest = move_to(cfg.unknown_dir, job)
+        enqueue_blocking!(unknown_queue, job, stats;
-        routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
-        enqueue_blocking!(unknown_queue, routed, stats;
                           retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
-        @debug "routed to content triage" worker=worker_id id=job.id dest=dest
+        @debug "routed to content triage" worker=worker_id id=job.id path=job.path
     end
     return nothing
 end
@@ -73,10 +93,11 @@ end
     handle_known_job(job, cfg, worker_id)
 
 Stage 2. Extract metadata (exiftool, with timeout) and enrich: build the
-normalized sidecar and commit both to `done/` sidecar-first. Extraction
+normalized sidecar and commit both to `done/` sidecar-first: the file's one and
+only move, straight out of `spool/`. Extraction
 failure/timeout yields a *degraded* sidecar (the file is still a wanted known
 file), so the only way to land in `failed/` is a genuine I/O error writing the
-sidecar or moving the file — handled by `worker_loop`'s quarantine.
+sidecar or moving the file, handled by `worker_loop`'s quarantine.
 """
 function handle_known_job(job::Job, cfg::Config, worker_id::Int)
     meta = build_metadata(job, cfg)
@@ -89,10 +110,14 @@ end
     handle_unknown_job(job, cfg, worker_id, text_queue)
 
 Stage 3. Sort an unrecognized file into a coarse content bucket by sniffing its
-first bytes: `binary/` (terminal — no further stage) if it looks like binary
+first bytes.
-data, `text/` otherwise. A text file is then routed onward to the stage-4
+
-language-enrichment queue, retrying on a full queue rather than dropping the file
+The two outcomes are asymmetric, because one is terminal and one is not. Binary
-(the same blocking backpressure stage 1 uses for its downstream queues).
+is the end of the live path, so the file is committed to `binary/`, which is
+also where the offline stage-5 discovery sweep reads its corpus, so the move is
+load-bearing, not bookkeeping. Text has a stage 4 still to come, so nothing moves:
+the same `Job` goes onto the language-enrichment queue, retrying on a full queue
+rather than dropping the file (the blocking backpressure stage 1 also uses).
 """
 function handle_unknown_job(job::Job, cfg::Config, worker_id::Int,
                             text_queue::JobQueue, stats::StageStats)
@@ -100,12 +125,10 @@ function handle_unknown_job(job::Job, cfg::Config, worker_id::Int,
         dest = move_to(cfg.binary_dir, job)
         @info "sorted unknown" worker=worker_id id=job.id name=job.original_name kind=:binary dest=dest
     else
-        dest = move_to(cfg.text_dir, job)
-        routed = Job(job.id, job.original_name, dest, job.size, job.received_at)
         # text queue full → park and retry, don't drop (time charged to blocked_ns)
-        enqueue_blocking!(text_queue, routed, stats;
+        enqueue_blocking!(text_queue, job, stats;
                           retry_seconds = ROUTE_ENQUEUE_RETRY_SECONDS)
-        @info "routed to language enrichment" worker=worker_id id=job.id dest=dest
+        @info "routed to language enrichment" worker=worker_id id=job.id path=job.path
     end
     return nothing
 end
@@ -115,9 +138,10 @@ end
 
 Stage 4. Enrich a text file with its natural language (via `detector`) and
 programming language (via github-linguist): build the sidecar and commit both to
-`text_done/` sidecar-first. Detection failure yields a *degraded* sidecar (the
+`text_done/` sidecar-first: the file's one and only move, straight out of
+`spool/`. Detection failure yields a *degraded* sidecar (the
 file is still wanted text), so the only way to land in `failed/` is a genuine I/O
-error committing — handled by `worker_loop`'s quarantine.
+error committing, handled by `worker_loop`'s quarantine.
 """
 function handle_text_job(job::Job, cfg::Config, worker_id::Int, detector)
     meta = build_text_metadata(detector, job, cfg)
@@ -130,18 +154,18 @@ end
     worker_loop(worker_id, cfg, queue, handler, stats)
 
 Consume jobs from `queue` until it is closed and drained, running `handler` on
-each. A failure on one job is logged and the file quarantined in `failed/` — it
+each. A failure on one job is logged and the file quarantined in `failed/`. It
 must never kill the worker, or the pool would silently shrink.
 
 This loop is also where per-stage metrics are recorded (`stats`, see
 src/stats.jl). Instrumenting here rather than in each handler means every stage
 is measured the same way, by construction, and a new stage is measured the
-moment it is wired up — there is no per-handler bookkeeping to forget.
+moment it is wired up, with no per-handler bookkeeping to forget.
 
 The timed region is the handler alone, excluding the `dequeue!` above it: time
 parked waiting for work is idleness, and counting it as service time would make
 an idle stage look as busy as a saturated one. A quarantined job still counts
-its time — the work was done, it just ended in `failed/`.
+its time: the work was done, it just ended in `failed/`.
 """
 function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue, handler,
                      stats::StageStats)
@@ -151,21 +175,46 @@ function worker_loop(worker_id::Int, cfg::Config, queue::JobQueue, handler,
         job === nothing && break   # queue closed and drained → exit
         Threads.atomic_add!(stats.in_flight, 1)
         t0 = time_ns()
-        ok = true
+        outcome = :ok
         try
             handler(job, cfg, worker_id)
+            ack!(queue, job)
         catch e
-            ok = false
+            if !isfile(job.path)
-            @error "processing failed" worker=worker_id id=job.id name=job.original_name exception=(e, catch_backtrace())
+                # The file is gone and we did not move it: another worker
-            try
+                # committed it, i.e. this is the duplicate half of an
-                move_to(cfg.failed_dir, job)
+                # at-least-once redelivery (src/rabbit.jl). The work is done, so
-            catch e2
+                # settle the message and say nothing louder than @debug. It is
-                @error "could not quarantine failed file" worker=worker_id id=job.id path=job.path exception=(e2, catch_backtrace())
+                # emphatically not a failure: quarantining here would file a
+                # `failed/` entry against a file that succeeded, and counting it
+                # either way would double-count one file's work.
+                outcome = :duplicate
+                @debug "duplicate delivery; already committed by another worker" worker=worker_id id=job.id name=job.original_name
+                ack!(queue, job)
+            else
+                outcome = :failed
+                @error "processing failed" worker=worker_id id=job.id name=job.original_name exception=(e, catch_backtrace())
+                try
+                    move_to(cfg.failed_dir, job)
+                    ack!(queue, job)   # quarantined: the file's story ends in failed/
+                catch e2
+                    @error "could not quarantine failed file" worker=worker_id id=job.id path=job.path exception=(e2, catch_backtrace())
+                    # Failed *and* unquarantinable: the file is still in spool/.
+                    # Reject rather than ack, because we are not claiming this
+                    # one is done. With no dead-letter exchange the broker
+                    # discards it either way — `nack!` never requeues, so this
+                    # cannot become a poison-message loop — but the distinction
+                    # is free and the message is the honest one.
+                    nack!(queue, job)
+                end
             end
         finally
             # In a `finally` so an InterruptException during shutdown can't leave
             # in_flight permanently above zero, which would read as a stuck job.
-            record_job!(stats, ok, job.size, Int(time_ns() - t0))
+            # A duplicate records nothing: its work was already counted by the
+            # delivery that actually did it.
+            outcome === :duplicate ||
+                record_job!(stats, outcome === :ok, job.size, Int(time_ns() - t0))
             Threads.atomic_sub!(stats.in_flight, 1)
         end
     end

test/runtests.jl

@@ -1,14 +1,15 @@
 using Test
-using FileServer
+using DarkStruct
 using JSON3
 
 # Pull internals into scope. These aren't exported (only `run` is), but the
 # whole risk profile of this pipeline lives in these functions, so we test them
 # directly rather than only through the HTTP surface.
-using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
+using DarkStruct: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
                   sanitize_filename, recover_dir!, normalize_metadata,
                   build_metadata, finalize_known!, run_exiftool,
-                  is_binary, handle_unknown_job, worker_loop,
+                  is_binary, handle_unknown_job, handle_classify_job, worker_loop,
+                  load_classifier, CLASSIFIER,
                   capacity, StageStats, IntakeStats, Metrics, METRICS, reset_metrics!,
                   record_job!, enqueue_blocking!, stats_snapshot, STAGE_KEYS,
                   detect_natural_language, run_linguist, detect_programming_language,
@@ -21,6 +22,9 @@ using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
                   Catalog, load_catalog, save_catalog!, catalog_sweep!, compact!,
                   write_nominations!, run_cluster_sweep, binary_files, record_example!,
                   signature_hex, ensure_dirs,
+                  ack!, nack!, with_delivery_tag, parse_backend, parse_bool, config_from_env,
+                  parse_amqp_url, job_json, job_from_message, AMQPTarget,
+                  RabbitQueue, connect_backend, open_queues, close_backend!,
                   MultipartReader, MultipartError, MultipartPart, next_part!,
                   write_part_body!, skip_part_body!, multipart_boundary,
                   parse_part_headers, spool_stream, UPLOAD_CHUNK_BYTES
@@ -70,10 +74,7 @@ end
 function tmp_config(root; kwargs...)
     cfg = Config(;
         spool_dir   = joinpath(root, "spool"),
-        known_dir   = joinpath(root, "known"),
-        unknown_dir = joinpath(root, "unknown"),
         binary_dir  = joinpath(root, "binary"),
-        text_dir    = joinpath(root, "text"),
         done_dir    = joinpath(root, "done"),
         text_done_dir = joinpath(root, "text_done"),
         failed_dir  = joinpath(root, "failed"),
@@ -82,11 +83,11 @@ function tmp_config(root; kwargs...)
         nominated_dir = joinpath(root, "nominated"),
         kwargs...,
     )
-    FileServer.ensure_dirs(cfg)
+    DarkStruct.ensure_dirs(cfg)
     return cfg
 end
 
-@testset "FileServer" begin
+@testset "DarkStruct" begin
 
     @testset "sanitize_filename" begin
         @test sanitize_filename("report.pdf") == "report.pdf"
@@ -100,7 +101,7 @@ end
         @test sanitize_filename("a b&c*.d") == "a_b_c_.d"
         @test sanitize_filename("") == "unnamed"
         # Length is capped.
-        @test Base.length(sanitize_filename("a"^500)) == FileServer.MAX_NAME_LEN
+        @test Base.length(sanitize_filename("a"^500)) == DarkStruct.MAX_NAME_LEN
     end
 
     @testset "multipart_boundary: extraction from Content-Type" begin
@@ -121,7 +122,7 @@ end
         @test p.filename == "a b.txt"
         @test p.content_type == "text/plain"
 
-        # `name=` must not match inside `filename=` — that would label every
+        # `name=` must not match inside `filename=`; that would label every
         # file part with a bogus name and (worse) hide a missing real name.
         p = parse_part_headers("Content-Disposition: form-data; filename=\"only.txt\"")
         @test p.name === nothing
@@ -279,7 +280,7 @@ end
         mktempdir() do root
             cfg = tmp_config(root; exiftool_timeout=5)
             # Point at a nonexistent file → exiftool exits non-zero → degraded.
-            job = Job("id-3", "gone.dat", joinpath(cfg.known_dir, "id-3-gone.dat"), 99, 0.0)
+            job = Job("id-3", "gone.dat", joinpath(cfg.spool_dir, "id-3-gone.dat"), 99, 0.0)
             m = build_metadata(job, cfg)
             @test m.error !== nothing
             @test m.file_type === nothing
@@ -292,13 +293,13 @@ end
     @testset "run_with_timeout: returns as soon as the child exits" begin
         # Regression guard. The original implementation polled with sleep(0.1)
         # and joined the polling task, so every call paid the remainder of an
-        # in-flight sleep after the child had already exited — ~101 ms on a
+        # in-flight sleep after the child had already exited: ~101 ms on a
         # process that exits instantly, on the hot path of stages 2 and 4. The
         # bound here is deliberately loose (a loaded CI box is slow) but far
         # under the 100 ms floor the polling version could not beat.
-        FileServer.run_with_timeout(`true`, 30)                       # warm up / compile
+        DarkStruct.run_with_timeout(`true`, 30)                       # warm up / compile
         t0 = time()
-        out = FileServer.run_with_timeout(`echo hi`, 30)
+        out = DarkStruct.run_with_timeout(`echo hi`, 30)
         elapsed = time() - t0
         @test out !== nothing
         @test strip(String(out)) == "hi"
@@ -307,7 +308,7 @@ end
 
     @testset "run_with_timeout: kills an overrunning child and reports failure" begin
         t0 = time()
-        out = FileServer.run_with_timeout(`sleep 30`, 1)
+        out = DarkStruct.run_with_timeout(`sleep 30`, 1)
         elapsed = time() - t0
         @test out === nothing            # timed out → no output, caller degrades
         @test elapsed < 5                # killed near the timeout, not after 30 s
@@ -318,11 +319,11 @@ end
         # block on wait(proc) forever and the timeout would be unenforceable.
         cmd = `sh -c "trap '' TERM; sleep 30"`
         t0 = time()
-        out = FileServer.run_with_timeout(cmd, 1)
+        out = DarkStruct.run_with_timeout(cmd, 1)
         elapsed = time() - t0
         @test out === nothing
         # 1 s timeout + up to KILL_GRACE_SECONDS before SIGKILL lands.
-        @test elapsed < 1 + FileServer.KILL_GRACE_SECONDS + 3
+        @test elapsed < 1 + DarkStruct.KILL_GRACE_SECONDS + 3
     end
 
     @testset "run_exiftool: real extraction on a PNG" begin
@@ -341,7 +342,7 @@ end
         mktempdir() do root
             cfg = tmp_config(root)
             # A real known-stage file to enrich.
-            src = joinpath(cfg.known_dir, "id-9-pixel.png")
+            src = joinpath(cfg.spool_dir, "id-9-pixel.png")
             write(src, PNG_1x1)
             job = Job("id-9", "pixel.png", src, Base.length(PNG_1x1), 0.0)
 
@@ -371,7 +372,7 @@ end
             write(txt, "hello, world\nsecond line\n")
             @test is_binary(txt) == false
 
-            # Non-ASCII UTF-8 (accents, CJK, emoji) is valid text — the whole
+            # Non-ASCII UTF-8 (accents, CJK, emoji) is valid text, which is the
             # point of moving off the printable-ASCII/NUL heuristic.
             uni = joinpath(root, "unicode.txt")
             write(uni, "café — 日本語 — 🚀\n")
@@ -402,7 +403,7 @@ end
             # binary: pad to one byte short of the window, then a 2-byte 'é'
             # (0xc3 0xa9) so only its lead byte lands inside the window.
             split = joinpath(root, "split.txt")
-            write(split, vcat(fill(UInt8('a'), FileServer.CONTENT_SNIFF_BYTES - 1),
+            write(split, vcat(fill(UInt8('a'), DarkStruct.CONTENT_SNIFF_BYTES - 1),
                               UInt8[0xc3, 0xa9]))
             @test is_binary(split) == false
 
@@ -413,7 +414,7 @@ end
 
             # Binary garbage past the sniff window is not seen → still text.
             far = joinpath(root, "far.txt")
-            write(far, vcat(fill(UInt8('a'), FileServer.CONTENT_SNIFF_BYTES), UInt8[0x00]))
+            write(far, vcat(fill(UInt8('a'), DarkStruct.CONTENT_SNIFF_BYTES), UInt8[0x00]))
             @test is_binary(far) == false
         end
     end
@@ -425,7 +426,7 @@ end
             stats = StageStats()
 
             # A binary file (embedded NUL) lands in binary/ and is NOT enqueued.
-            bpath = joinpath(cfg.unknown_dir, "id-b-blob.dat")
+            bpath = joinpath(cfg.spool_dir, "id-b-blob.dat")
             write(bpath, UInt8[0x00, 0xFF, 0x10])
             bjob = Job("id-b", "blob.dat", bpath, filesize(bpath), 0.0)
             handle_unknown_job(bjob, cfg, 1, text_queue, stats)
@@ -433,19 +434,49 @@ end
             @test !isfile(bpath)
             @test length(text_queue) == 0
 
-            # A text file lands in text/ AND is routed onto the stage-4 queue,
+            # A text file is routed onto the stage-4 queue and *does not move*:
-            # with its path updated to the new text/ location.
+            # stage 4 is still to come, so the bytes stay in spool/ and the same
-            tpath = joinpath(cfg.unknown_dir, "id-t-notes.log")
+            # reference is handed on. Regression guard against re-introducing an
+            # intermediate hop: the queue, not a directory, is what records that
+            # this file has cleared triage.
+            tpath = joinpath(cfg.spool_dir, "id-t-notes.log")
             write(tpath, "just some log text\n")
             tjob = Job("id-t", "notes.log", tpath, filesize(tpath), 0.0)
             handle_unknown_job(tjob, cfg, 1, text_queue, stats)
-            moved = joinpath(cfg.text_dir, "id-t-notes.log")
+            @test isfile(tpath)                        # stayed put
-            @test isfile(moved)
-            @test !isfile(tpath)
             @test length(text_queue) == 1
             routed = dequeue!(text_queue)
             @test routed.id == "id-t"
-            @test routed.path == moved
+            @test routed.path == tpath                 # reference unchanged
+        end
+    end
+
+    @testset "handle_classify_job: routes by enqueue only, never moves the file" begin
+        # Stage 1's entire job is picking a queue. Whichever way the classifier
+        # votes, the bytes must stay where intake wrote them and the *same* Job
+        # must be handed on: the queue records the phase, not a directory.
+        # Asserted against both queues at once so the test doesn't depend on how
+        # the committed model happens to label these bytes.
+        CLASSIFIER[] = load_classifier(joinpath(@__DIR__, "..", "model", "classifier.jld2"))
+        mktempdir() do root
+            cfg = tmp_config(root)
+            known_queue, unknown_queue = ChannelQueue(10), ChannelQueue(10)
+            stats = StageStats()
+
+            path = joinpath(cfg.spool_dir, "id-c-pixel.png")
+            write(path, PNG_1x1)
+            job = Job("id-c", "pixel.png", path, Base.length(PNG_1x1), 0.0)
+
+            handle_classify_job(job, cfg, 1, known_queue, unknown_queue, stats)
+
+            @test isfile(path)                                          # stayed put
+            @test length(known_queue) + length(unknown_queue) == 1      # routed exactly once
+            routed = length(known_queue) == 1 ? dequeue!(known_queue) : dequeue!(unknown_queue)
+            @test routed.path == path                                   # reference unchanged
+            @test routed.id == "id-c"
+            # No intermediate directory was invented alongside spool/.
+            @test !isdir(joinpath(root, "known"))
+            @test !isdir(joinpath(root, "unknown"))
         end
     end
 
@@ -472,10 +503,10 @@ end
             # Reads at most LANG_SAMPLE_BYTES, and doesn't choke on a multi-byte
             # char straddling that boundary (trailing 'é' half-in the window).
             big = joinpath(root, "big.txt")
-            write(big, vcat(fill(UInt8('a'), FileServer.LANG_SAMPLE_BYTES - 1),
+            write(big, vcat(fill(UInt8('a'), DarkStruct.LANG_SAMPLE_BYTES - 1),
                             UInt8[0xc3, 0xa9]))              # 'é' split by the edge
             s = read_text_sample(big)
-            @test Base.length(s) == FileServer.LANG_SAMPLE_BYTES - 1   # trailing half-char trimmed
+            @test Base.length(s) == DarkStruct.LANG_SAMPLE_BYTES - 1   # trailing half-char trimmed
             @test all(==('a'), s)
         end
     end
@@ -503,7 +534,7 @@ end
             cfg = tmp_config(root)
             d = LanguageDetector()
 
-            src = joinpath(cfg.text_dir, "id-x-script.py")
+            src = joinpath(cfg.spool_dir, "id-x-script.py")
             write(src, join(["# a short program in English prose comment",
                              "import sys",
                              "def greet(name):",
@@ -542,7 +573,7 @@ end
             cfg = tmp_config(root)
             d = LanguageDetector()
 
-            src = joinpath(cfg.text_dir, "id-h-readme.md")
+            src = joinpath(cfg.spool_dir, "id-h-readme.md")
             write(src, "# Project\n\nThis project does something useful and interesting for everyone.\n")
             job = Job("id-h", "readme.md", src, filesize(src), 0.0)
 
@@ -632,7 +663,7 @@ end
     end
 
     @testset "cluster: §10.1 discovers nothing from noise" begin
-        # 25 independent random blobs — the shape of data/binary (structureless
+        # 25 independent random blobs: the shape of data/binary (structureless
         # junk). Correct output: ZERO promoted clusters (random headers never
         # form a ≥20-member, ≥3-magic-byte signature). See DESIGN §10.1.
         rng = MersenneTwister(20260703)
@@ -653,9 +684,9 @@ end
     @testset "cluster: §10.2 recovers known (synthetic) formats" begin
         # Four synthetic "formats": a fixed magic prefix + random tail, mirroring
         # gzip/PDF/JPEG/ELF. Calibrated settings must recover them as clean,
-        # promotable clusters at high ARI — the magic-collapsed recovery of §10.2,
+        # promotable clusters at high ARI: the magic-collapsed recovery of §10.2,
         # here with a hermetic, deterministic corpus.
-        # ~12-byte constant headers + random tails — the shape of a real file
+        # ~12-byte constant headers + random tails: the shape of a real file
         # header (a fixed magic/version region, then variable content). A too-short
         # magic over a fully-random tail is adversarially hard and lets a format
         # over-split; real headers anchor a cluster with ~12+ constant bytes.
@@ -685,7 +716,7 @@ end
         for (id, c) in r.clusters
             sig = signature(c)
             if is_promotable(c, sig; min_members=20, min_magic=3)
-                # ...and every nomination is PURE — the whole point of the human
+                # ...and every nomination is PURE. The whole point of the human
                 # gate is that we never hand it a garbage merged signature.
                 labels = unique(breakdown(id))
                 @test Base.length(labels) == 1
@@ -739,6 +770,39 @@ end
         end
     end
 
+    @testset "recover_dir!: blocks on a full queue, recovers every file" begin
+        # The regression this guards is data loss, not slowness. `spool/` now
+        # holds every in-flight file at every stage, so a crash under load leaves
+        # far more leftovers than one queue's capacity, and the old non-blocking
+        # recovery warned and abandoned the excess, which is exactly the work
+        # recovery exists to save. With the pool live, recovery must park until
+        # the consumer makes room and come back with all of it.
+        mktempdir() do root
+            dir = joinpath(root, "spool"); mkpath(dir)
+            n_files = 25
+            for i in 1:n_files
+                write(joinpath(dir, string("id-", lpad(i, 3, '0'), "-f.dat")), "x")
+            end
+
+            q = ChannelQueue(4)          # deliberately far smaller than n_files
+            drained = Job[]
+            consumer = Threads.@spawn begin
+                while Base.length(drained) < n_files
+                    job = dequeue!(q)
+                    job === nothing && break
+                    push!(drained, job)
+                end
+            end
+
+            n = recover_dir!(dir, q)     # would strand 21 files if it didn't block
+            wait(consumer)
+
+            @test n == n_files
+            @test Base.length(drained) == n_files
+            @test Base.length(Set(j.path for j in drained)) == n_files   # no duplicates
+        end
+    end
+
     # Helper: write a "file" of raw bytes into a dir with a UUID-ish unique name,
     # returning its path. Mirrors what stage-3 deposits into binary/.
     function drop_binary(dir, bytes; name=string(rand(UInt128)))
@@ -826,7 +890,7 @@ end
             @test "match-01" in cat.processed
             @test "blob-01" in cat.processed
 
-            # Re-sweeping the same pile is idempotent — nothing new is seen.
+            # Re-sweeping the same pile is idempotent; nothing new is seen.
             s2 = catalog_sweep!(cat, cfg)
             @test s2.n_seen == 0
             @test cat.clusters[1].members == 41
@@ -850,7 +914,7 @@ end
             r = run_cluster_sweep(cfg; rng=MersenneTwister(10))
             @test r.mode == :compact
             @test isfile(cfg.cluster_catalog_path)
-            # The mission-critical assertion: ZERO promoted clusters from pure noise.
+            # The assertion that matters: ZERO promoted clusters from pure noise.
             @test r.n_nominated == 0
             @test isempty(readdir(cfg.nominated_dir))
             # Every file was accounted for (clustered-as-singleton or background).
@@ -862,13 +926,13 @@ end
         mktempdir() do root
             n = 32
             # β=0.1 over-splits a format into pure sub-clusters (DESIGN §11 known
-            # limitation) — each still carries the full magic and nominates
+            # limitation). Each still carries the full magic and nominates
             # independently, so a modest min_members catches those sub-clusters.
             cfg = tmp_config(root; cluster_n=n, cluster_alpha=1.0,
                              cluster_pseudocount=0.1, cluster_bg_mass=5.0,
                              promote_min_members=10, promote_min_magic=3)
             rng = MersenneTwister(21)
-            # 30 files sharing a fixed 6-byte magic then random payload — a format.
+            # 30 files sharing a fixed 6-byte magic then random payload: a format.
             magic = UInt8[0x89, 0x46, 0x4d, 0x54, 0x21, 0x0a]
             for i in 1:30
                 drop_binary(cfg.cluster_dir, vcat(magic, rand(rng, UInt8, 40)); name="fmt-$(lpad(i,2,'0'))")
@@ -968,7 +1032,7 @@ end
                 @test stats.failed[] == 1
                 @test stats.bytes[] == 30
                 # Each of the three handlers slept 20ms before its outcome, so
-                # busy time covers the failure too — the work was done either way.
+                # busy time covers the failure too; the work was done either way.
                 @test stats.busy_ns[] > 3 * 15_000_000
                 @test stats.blocked_ns[] == 0       # nothing downstream to block on
                 @test stats.in_flight[] == 0        # the finally in worker_loop
@@ -1012,7 +1076,7 @@ end
                 text_queue = ChannelQueue(1)
                 @test enqueue!(text_queue, Job("filler", "f", "/tmp/f", 1, 0.0))
 
-                p = joinpath(cfg.unknown_dir, "id-t-notes.log")
+                p = joinpath(cfg.spool_dir, "id-t-notes.log")
                 write(p, "plain text\n")
                 job = Job("id-t", "notes.log", p, filesize(p), 0.0)
 
@@ -1024,7 +1088,7 @@ end
                 wait(drainer)
                 @test stats.blocked_ns[] > 50_000_000
                 @test length(text_queue) == 1              # the file did get through
-                @test isfile(joinpath(cfg.text_dir, "id-t-notes.log"))
+                @test isfile(p)                            # and stayed in spool/
             end
         end
 
@@ -1057,7 +1121,7 @@ end
                 @test snap.intake.files == 9
                 @test snap.uptime_seconds >= 0
 
-                # It has to survive the trip through JSON — /stats is the only
+                # It has to survive the trip through JSON: /stats is the only
                 # consumer, and bin/bench.jl reads these exact field names.
                 round_tripped = JSON3.read(JSON3.write(snap))
                 @test round_tripped.stages[2].busy_seconds ≈ 2.0
@@ -1071,4 +1135,180 @@ end
         end
     end
 
+    @testset "queue backends" begin
+        @testset "backend selection is explicit or an error" begin
+            @test parse_backend("channel") === :channel
+            @test parse_backend("  RabbitMQ ") === :rabbitmq
+            # A typo must not quietly leave you on the in-process queue: that
+            # looks exactly like durability working, until a crash proves it isn't.
+            @test_throws ArgumentError parse_backend("rabbit")
+            @test_throws ArgumentError parse_backend("")
+
+            @test parse_bool("true") && parse_bool("1") && parse_bool("ON")
+            @test !parse_bool("false") && !parse_bool("0") && !parse_bool("off")
+            @test_throws ArgumentError parse_bool("maybe")
+
+            @test config_from_env().queue_backend === :channel   # default is unchanged
+            cfg = config_from_env(; queue_backend = :rabbitmq, amqp_prefetch = 4,
+                                    amqp_confirms = false, recover_spool = true)
+            @test cfg.queue_backend === :rabbitmq
+            @test cfg.amqp_prefetch == 4 && !cfg.amqp_confirms && cfg.recover_spool
+        end
+
+        @testset "amqp url parsing" begin
+            t = parse_amqp_url("amqp://user:pw@broker.internal:5673/prod")
+            @test t == AMQPTarget("broker.internal", 5673, "prod", "user", "pw")
+
+            # Bare form: default port, default vhost, and the guest credentials
+            # that only work over loopback anyway.
+            t2 = parse_amqp_url("amqp://localhost/")
+            @test t2.port == 5672 && t2.virtualhost == "/" && t2.login == "guest"
+            @test parse_amqp_url("amqp://localhost").virtualhost == "/"
+            # The conventional default vhost is often written percent-escaped.
+            @test parse_amqp_url("amqp://h:1/%2F").virtualhost == "/"
+            # Credentials may contain characters that need escaping in a URL.
+            @test parse_amqp_url("amqp://u%40b:p%2Fw@h/").password == "p/w"
+
+            @test_throws ArgumentError parse_amqp_url("http://localhost:5672/")
+            @test_throws ArgumentError parse_amqp_url("amqps://localhost:5671/")
+        end
+
+        @testset "the delivery tag never goes on the wire" begin
+            job = with_delivery_tag(Job("id-9", "a b.txt", "/tmp/spool/id-9-a_b.txt",
+                                        4096, 1234.5), 77)
+            @test job.delivery_tag == 77
+
+            # A tag names a delivery on one channel of one connection, not the
+            # job, so publishing it would be meaningless at best and would
+            # survive a restart as a lie at worst.
+            wire = JSON3.read(job_json(job))
+            @test !haskey(wire, :delivery_tag)
+            @test wire.id == "id-9" && wire.size == 4096 && wire.received_at == 1234.5
+
+            # Everything else must survive the round trip untouched, including a
+            # name with a space in it.
+            @test wire.original_name == "a b.txt"
+            @test wire.path == "/tmp/spool/id-9-a_b.txt"
+        end
+
+        @testset "acks are a no-op on the in-process queue" begin
+            # ChannelQueue has no delivery to settle, so `worker_loop` calling
+            # these on every job must cost nothing and change nothing.
+            q = ChannelQueue(2)
+            job = Job("id-1", "a.bin", "/tmp/a.bin", 1, 0.0)
+            @test enqueue!(q, job)
+            @test ack!(q, job) === nothing
+            @test nack!(q, job) === nothing
+            @test length(q) == 1
+        end
+
+        @testset "a duplicate delivery is settled, not quarantined" begin
+            # At-least-once means the same file can be handed to two workers, and
+            # the loser finds it already committed. That is a success it arrived
+            # too late for, not a failure: quarantining would file a `failed/`
+            # entry against a file that worked, and counting it either way would
+            # double-count one file.
+            mktempdir() do root
+                cfg = tmp_config(root)
+                q = ChannelQueue(10)
+                stats = StageStats()
+
+                dup = joinpath(cfg.spool_dir, "id-1-gone.bin")
+                write(dup, "x" ^ 10)
+                @test enqueue!(q, Job("id-1", "gone.bin", dup, 10, 0.0))
+
+                real = joinpath(cfg.spool_dir, "id-2-here.bin")
+                write(real, "y" ^ 10)
+                @test enqueue!(q, Job("id-2", "here.bin", real, 10, 0.0))
+                close!(q)
+
+                worker_loop(1, cfg, q, (job, _, _) -> begin
+                    # The duplicate's file was committed by the winner before the
+                    # handler ran; the other job fails with its file still there.
+                    job.id == "id-1" && rm(job.path)
+                    error("handler blew up")
+                end, stats)
+
+                @test stats.completed[] == 0
+                @test stats.failed[] == 1              # only the genuine failure
+                @test stats.bytes[] == 10              # the duplicate records nothing
+                @test stats.in_flight[] == 0
+                @test !isfile(joinpath(cfg.failed_dir, "id-1-gone.bin"))
+                @test isfile(joinpath(cfg.failed_dir, "id-2-here.bin"))
+            end
+        end
+
+        # The rest needs a live broker. Point FS_TEST_AMQP_URL at one to run it:
+        #   docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
+        #   FS_TEST_AMQP_URL=amqp://darkstruct:darkstruct@localhost:5672/ julia --project=. -e 'using Pkg; Pkg.test()'
+        amqp_url = get(ENV, "FS_TEST_AMQP_URL", "")
+        if isempty(amqp_url)
+            @info "skipping RabbitMQ integration tests (set FS_TEST_AMQP_URL to run them)"
+        else
+            @testset "rabbitmq round trip, redelivery and depth" begin
+                # A prefix per run, so a leftover queue from a previous run can
+                # never make this pass (or fail) for the wrong reason.
+                prefix = "fstest-" * string(rand(UInt32); base = 16)
+                cfg = config_from_env(; queue_backend = :rabbitmq, amqp_url = amqp_url,
+                                        amqp_prefix = prefix, queue_capacity = 5,
+                                        worker_count = 2, known_worker_count = 2,
+                                        unknown_worker_count = 2, text_worker_count = 2)
+                backend = connect_backend(cfg)
+                queues = open_queues(backend, cfg)
+                q = queues.classify
+                @test q isa RabbitQueue
+                @test capacity(q) == 5
+
+                @testset "a job survives the wire intact" begin
+                    @test enqueue!(q, Job("id-1", "hello.txt", "/tmp/hello.txt", 123, 1.5))
+                    got = dequeue!(q)
+                    @test got.id == "id-1"
+                    @test got.original_name == "hello.txt"
+                    @test got.path == "/tmp/hello.txt"
+                    @test got.size == 123
+                    @test got.received_at == 1.5
+                    @test got.delivery_tag != 0   # a real delivery, ackable
+                    ack!(q, got)
+                end
+
+                @testset "an unacked job comes back after a crash" begin
+                    # The whole point of the backend: take a job, never ack it,
+                    # lose the connection the way a SIGKILL would, and find it
+                    # waiting on restart.
+                    @test enqueue!(q, Job("id-2", "again.txt", "/tmp/again.txt", 7, 2.5))
+                    taken = dequeue!(q)
+                    @test taken.id == "id-2"
+
+                    foreach(close!, values(queues))
+                    close_backend!(backend)
+                    sleep(1.0)
+
+                    backend2 = connect_backend(cfg)
+                    queues2 = open_queues(backend2, cfg)
+                    q2 = queues2.classify
+                    redelivered = dequeue!(q2)
+                    @test redelivered.id == "id-2"
+                    ack!(q2, redelivered)
+
+                    sleep(2 * 1.0 + 0.5)          # let the depth poller catch up
+                    @test length(q2) == 0
+
+                    # Advisory capacity: publishing without draining must start
+                    # refusing rather than let the queue grow without bound.
+                    refused = 0
+                    for i in 1:40
+                        enqueue!(q2, Job("f$i", "f$i", "/tmp/f$i", 1, 0.0)) && continue
+                        refused = i
+                        break
+                    end
+                    @test refused > 0
+                    @test length(q2) >= capacity(q2)
+
+                    foreach(close!, values(queues2))
+                    close_backend!(backend2)
+                end
+            end
+        end
+    end
+
 end