frutebox-node 0.1.0__py3-none-any.whl

frutebox/serving.py

@@ -0,0 +1,691 @@
+"""Serving-config RESOLVER (POST_TEST_REWORK #2 / "any model serves when armed").
+
+The benchmark session proved that *how* to serve a model is model-specific and easy to get wrong:
+Ollama cannot serve sharded GGUF (every Kimi quant is sharded) so those need llama.cpp's server;
+big MoE models need multi-GPU placement and the right quant; reasoning models need a large output
+budget. The owner must not babysit any of this — arming a toggle must Just Work for ANY catalog
+model. This module is the single place that answers, for a given model id:
+
+  * which ENGINE serves it (llama.cpp for sharded GGUF, Ollama for simple single-file models),
+  * the container IMAGE + how the model is referenced (HF GGUF repo:quant, or an Ollama tag),
+  * how much VRAM the chosen quant needs, and
+  * a COST-RANKED, cheapest-first list of GPU configs that actually FIT it — which both selects the
+    cheapest placement and fixes the "no capacity / won't place" failures (try the next config).
+
+Pure + deterministic (no network); RunPod stock filtering happens at deploy time by walking the
+returned prioritized list. Unit-tested.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import logging
+
+from .envvars import env as envvar
+from .manifest import _BY_ID, classify_capability
+
+log = logging.getLogger(__name__)
+
+# --- GPU inventory: type -> (usable VRAM GB, on-demand $/hr). Prices are grounded in what we
+# observed on RunPod this session (A100 PCIe x8 = $9.52/hr -> ~$1.19/gpu; H200 x5 = $22.95 ->
+# ~$4.59/gpu; RTX PRO 6000 x8 = $16.72 -> ~$2.09/gpu) plus standard small-GPU rates. Cheapest-first
+# ordering here drives "cheapest GPU that fits". Operator can override via config later.
+# `fp8` = the card has NATIVE FP8 tensor cores (Hopper / Ada / Blackwell). Ampere (A40, A100,
+# A6000, Ax000) does NOT: an FP8 checkpoint placed there either refuses to load or falls back to
+# emulation that is slower than the FP16 it was supposed to beat. Cost-ranking alone would pick
+# 6xA40 at $2.40/hr over 2xH200 at $9.18/hr and produce a pod that cannot serve the model — the
+# cheapest config that CANNOT RUN THE JOB is not the cheapest config.
+GPU_INVENTORY: list[dict] = [
+    {"type": "RTX A4000", "vram_gb": 16, "usd_per_hr": 0.17, "fp8": False},
+    {"type": "RTX A5000", "vram_gb": 24, "usd_per_hr": 0.26, "fp8": False},
+    {"type": "A40", "vram_gb": 48, "usd_per_hr": 0.40, "fp8": False},
+    {"type": "RTX A6000", "vram_gb": 48, "usd_per_hr": 0.49, "fp8": False},
+    {"type": "A100 PCIe", "vram_gb": 80, "usd_per_hr": 1.19, "fp8": False},
+    {"type": "A100 SXM", "vram_gb": 80, "usd_per_hr": 1.29, "fp8": False},
+    {"type": "RTX PRO 6000", "vram_gb": 96, "usd_per_hr": 2.09, "fp8": True},
+    {"type": "H100 SXM", "vram_gb": 80, "usd_per_hr": 2.69, "fp8": True},
+    {"type": "H200 SXM", "vram_gb": 141, "usd_per_hr": 4.59, "fp8": True},
+]
+# Max GPUs we'll pool into a single node (RunPod caps multi-GPU pods).
+_MAX_GPUS = 8
+
+
+@dataclass(frozen=True)
+class GpuConfig:
+    gpu_type: str
+    count: int
+    total_vram_gb: int
+    usd_per_hr: float
+
+    def label(self) -> str:
+        return f"{self.gpu_type} x{self.count} ({self.total_vram_gb}GB, ${self.usd_per_hr:.2f}/hr)"
+
+
+def gpu_plan(vram_needed_gb: float, *, headroom: float = 1.15, max_gpus: int = _MAX_GPUS,
+             inventory=GPU_INVENTORY, require_fp8: bool = False) -> list[GpuConfig]:
+    """Cost-ranked (cheapest-first) list of GPU configs whose total VRAM covers vram_needed*headroom.
+    A node arms by walking this list and taking the first that RunPod has in stock — so it always
+    lands on the CHEAPEST placement available, and never fails just because the ideal config is out
+    of stock."""
+    need = vram_needed_gb * headroom
+    out: list[GpuConfig] = []
+    for g in inventory:
+        if require_fp8 and not g.get("fp8"):
+            continue
+        # smallest count of this type that covers need (cap at max_gpus)
+        for n in range(1, max_gpus + 1):
+            if g["vram_gb"] * n >= need:
+                out.append(GpuConfig(g["type"], n, g["vram_gb"] * n, round(g["usd_per_hr"] * n, 2)))
+                break
+    out.sort(key=lambda c: (c.usd_per_hr, c.total_vram_gb))
+    return out
+
+
+# --- Per-model serving recipes.
+#
+# FILE 01-B — WHY THIS CHANGED. The frontier pod served Kimi through llama.cpp with
+# `--parallel 1`: one request at a time on a $17.95-22.95/hr node, which works out at roughly
+# $165-210 per 1M output tokens. The GPU is idle for most of every request and the cost is
+# carried by a single user. Batching is the whole fix — the same pod serving 8-16 concurrent
+# streams divides that fixed hourly cost across all of them.
+#
+# Engine choice, in order of preference:
+#   * SGLang  — PRIMARY. RadixAttention reuses the KV prefix across turns automatically, which
+#               is exactly our workload (multi-turn chat over a stable system prompt + brain
+#               context). Serves safetensors/FP8, not GGUF.
+#   * vLLM    — FALLBACK for models SGLang cannot serve. Broad coverage, PagedAttention.
+#   * llama.cpp — the ONLY engine for GGUF-only quants (every Kimi quant is sharded GGUF), and
+#               the right one for the small always-warm C/B node. Now batched, not `--parallel 1`.
+#   * Ollama  — simple single-file models on the small node.
+LLAMACPP_IMAGE = "ghcr.io/ggml-org/llama.cpp:server-cuda"
+OLLAMA_IMAGE = "ollama/ollama:latest"
+SGLANG_IMAGE = "lmsysorg/sglang:latest"
+#: ComfyUI, for the diffusion models (image/video). VERIFIED TO EXIST before being written here:
+#: `runpod/worker-comfyui` and `yanwk/comfyui-boot` are both 404 on Docker Hub; this one answers 200.
+#: An image reference nobody checked is the "RUNNING with no runtime" failure waiting to happen.
+COMFYUI_IMAGE = "ghcr.io/ai-dock/comfyui:latest"
+#: ComfyUI's HTTP port, and the path that proves the SERVER is up rather than the container.
+#: `/` serves the web UI; `/system_stats` is the API's own health answer — the same distinction
+#: `PodSpec.health_path` exists for (SGLang has no `/`).
+COMFYUI_PORT = 8188
+COMFYUI_HEALTH_PATH = "/system_stats"
+
+#: Concurrent sequences a tier-S pod serves. 16 is the file-11-B5 starting point: high enough
+#: to divide the hourly cost properly, low enough that per-stream latency stays acceptable.
+TIER_S_MAX_SEQS = 16
+#: llama.cpp's equivalent, for the GGUF models SGLang cannot serve.
+LLAMACPP_PARALLEL = 8
+
+# HF GGUF repo + a default quant (repo:quant) + that quant's approx VRAM, for the models that must
+# run on llama.cpp. Quant chosen as the quality/size balance that fits a reasonable node.
+#: Models served by SGLang from FP8 safetensors. FP8 weights + an FP8 KV cache roughly halve
+#: memory versus FP16 with minimal quality loss on Hopper, which is what lets the 235B MoE fit
+#: on 2xH200 (~$7.18/hr Community) instead of the 5xH200 pod at $17.95-22.95/hr.
+_FP8_MODELS: dict[str, dict] = {
+    # 235B params at FP8 ~= 235 GB of weights; 2xH200 = 282 GB leaves headroom for the KV
+    # cache, which is what makes the $7.18/hr Community placement possible at all.
+    #
+    # `reasoning: True` and the parser are not a guess — MEASURED on a live pod 2026-08-28: this
+    # model emits <think> blocks. Without `--reasoning-parser` SGLang returns that reasoning
+    # inside `content`, so the user is shown the model thinking out loud instead of its answer,
+    # and short max_tokens truncates mid-thought with no answer at all (observed: 120 tokens of
+    # "Okay, so I need to find 17.3% of 4821. Hmm..." and nothing else).
+    "qwen3-235b-a22b": {"repo": "Qwen/Qwen3-235B-A22B-FP8", "vram_gb": 235,
+                        "reasoning": True, "ctx": 32768, "min_gpus": 2,
+                        "reasoning_parser": "qwen3"},
+    # The tier-A default, and one of the two models file 02-C2 keeps on a seeded backup volume
+    # ("the tier-S default + tier-A default"). It was in the catalog with no way to serve it, so
+    # the backup lane could seed it and then not bring it up.
+    #
+    # SGLang, not llama.cpp: the engine order above makes SGLang primary and reserves llama.cpp
+    # for GGUF-ONLY quants. Qwen publishes official FP8 safetensors, so this is not GGUF-only.
+    #
+    # vram_gb is MEASURED from the repo (30.9 GB of FP8 weights), not taken from the catalog's
+    # 22.0 — that figure appears to describe a smaller quant, and a serving spec that under-states
+    # its own weights picks a GPU the model does not fit on. Rounded up, leaving KV-cache room on
+    # a 48 GB card.
+    #
+    # FP8 needs Hopper/Ada/Blackwell; Ampere (A40/A6000/A5000) cannot run it. `gpu_plan(...,
+    # require_fp8=True)` already enforces that, so an Ampere-only region will simply find no
+    # placement rather than starting and failing.
+    #
+    # reasoning=True with the qwen3 parser: the 235B in this same family was MEASURED emitting
+    # <think> blocks on live hardware, and the asymmetry favours declaring it — a parser on a
+    # model that does not think is harmless, while no parser on one that does puts the reasoning
+    # in `content` and truncates the answer. To be confirmed on this model's first live run.
+    "qwen-3.6-27b": {"repo": "Qwen/Qwen3.6-27B-FP8", "vram_gb": 31,
+                     "reasoning": True, "ctx": 32768, "min_gpus": 1,
+                     "reasoning_parser": "qwen3"},
+}
+
+_LLAMACPP_MODELS: dict[str, dict] = {
+    "kimi-k2.7-code": {"repo": "unsloth/Kimi-K2.7-Code-GGUF", "quant": "UD-Q8_K_XL",
+                       "vram_gb": 594, "reasoning": True, "ctx": 32768},
+    "kimi-k2.6": {"repo": "unsloth/Kimi-K2.6-GGUF", "quant": "UD-Q4_K_XL",
+                  "vram_gb": 590, "reasoning": False, "ctx": 16384},
+    "qwen3-235b-a22b": {"repo": "unsloth/Qwen3-235B-A22B-GGUF", "quant": "UD-Q4_K_XL",
+                        "vram_gb": 140, "reasoning": False, "ctx": 16384},
+}
+
+
+# ---- FILE 11-B2: speculative decoding ---------------------------------------------------------
+#
+# WHY IT IS WORTH THE COMPLICATION. Speculative decoding is quality-LOSSLESS — the target model
+# verifies every drafted token, so the output distribution is unchanged — and it buys roughly 2-3x on
+# decode, which is the half of latency the user watches. 11-A's stream-rate budgets (>=30 tok/s local,
+# >=20 tok/s tier-S) are decode budgets.
+#
+# WHICH METHOD IS A FACT ABOUT THE MODEL, so it lives in the registry (`speed.spec_decode`, added to
+# `frutebox/registry/models.yaml`), not in a table here. A serving module with its own opinion about
+# a model's drafting method is a second source of truth, and the registry already owns the others.
+#
+# P-EAGLE IS NOT AVAILABLE ON OUR PRIMARY ENGINE, and the plan says to use it. `implementation_plan/
+# libraries.md` line 16 (restated at 151) records that P-EAGLE is vLLM v0.16+ only and not merged in
+# SGLang; SGLang is primary. So `p-eagle` in the registry renders as EAGLE3 on SGLang and gets its
+# parallel-drafting flag only on vLLM.
+#
+# AND THE HEAD PATH IS NOT GUESSED. SGLang's EAGLE3 needs `--speculative-draft-model-path <head>`,
+# published per target model as a separate artefact. A guessed repo name produces a pod that fails to
+# start — or one that starts with the wrong head, which is worse because it looks like it worked. So
+# the path comes from configuration, and when it is missing speculative decoding is OFF and SAYS SO.
+# The alternative — shipping a plausible-looking path — is how an unverifiable claim reaches a
+# runbook.
+_SPEC_ENV_PREFIX = "EAGLE3_HEAD_"
+
+
+def eagle3_head(model_id: str) -> str:
+    """The configured EAGLE-3 head for a model, or "" when nobody has named one.
+
+    Env name is the model id upper-cased with `-`/`.` as `_`, e.g.
+    `FRUTEBOX_EAGLE3_HEAD_QWEN3_CODER`.
+    """
+    key = _SPEC_ENV_PREFIX + model_id.upper().replace("-", "_").replace(".", "_")
+    return (envvar(key, "") or "").strip()
+
+
+#: ONE VARIABLE TURNS IT OFF. These flags have not yet been accepted by a live SGLang process on this
+#: account — 11's own done-criterion is a MEASURED >=1.6x, which needs an armed pod — and an engine
+#: that rejects an unknown argument does not start. So the switch exists to make recovery a config
+#: change rather than a code change, and the A/B the criterion asks for is one flag each way.
+SPEC_DECODE_ENABLED = envvar("SPEC_DECODE", "true").strip().lower() == "true"
+
+
+def spec_decode_args(model_id: str, engine: str, *, speed: "dict | None" = None,
+                     enabled: "bool | None" = None) -> str:
+    """The engine flags for this model's speculative decoding, or "" when it has none.
+
+    Returns flags only for a method the ENGINE can actually run with the pieces we HAVE. Every other
+    case returns nothing and logs the reason at resolve time rather than at boot: a pod is the most
+    expensive place to discover a missing head.
+    """
+    if not (SPEC_DECODE_ENABLED if enabled is None else enabled):
+        return ""
+    if speed is None:
+        try:
+            from .registry_gen import BY_ID
+            speed = (BY_ID.get(model_id) or {}).get("speed") or {}
+        except Exception:                       # pragma: no cover - registry import guard
+            speed = {}
+    method = str((speed or {}).get("spec_decode") or "none")
+    k = int((speed or {}).get("K") or 7)
+    if method == "none":
+        return ""
+    if engine == "llamacpp":
+        # MTP comes from the target's own heads; llama.cpp enables it by serving a model that has
+        # them, with no flag of its own. Nothing to add, and nothing to claim.
+        return ""
+    if method == "mtp":
+        return ""                               # an MTP target on a non-llama.cpp engine: no flag
+    if method == "draft":
+        draft = str((speed or {}).get("draft_model") or "").strip()
+        if not draft:
+            return ""
+        if engine == "sglang":
+            return (f"--speculative-algorithm EAGLE3 --speculative-draft-model-path {draft} "
+                    f"--speculative-num-steps {k}")
+        if engine == "vllm":
+            return f"--speculative-config {{\"model\":\"{draft}\",\"num_speculative_tokens\":{k}}}"
+        return ""
+    # eagle3 / p-eagle: both need a HEAD, and only vLLM can do the parallel-drafting variant.
+    head = str((speed or {}).get("draft_model") or "").strip() or eagle3_head(model_id)
+    if not head:
+        log.warning("speculative decoding OFF for %s: method %s needs an EAGLE-3 head and none is "
+                    "configured (set FRUTEBOX_%s%s). Serving correctly, just not faster.",
+                    model_id, method, _SPEC_ENV_PREFIX,
+                    model_id.upper().replace("-", "_").replace(".", "_"))
+        return ""
+    if engine == "sglang":
+        return (f"--speculative-algorithm EAGLE3 --speculative-draft-model-path {head} "
+                f"--speculative-num-steps {k}")
+    if engine == "vllm":
+        cfg = (f'{{"method":"eagle3","model":"{head}","num_speculative_tokens":{k}'
+               + (',"parallel_drafting":true' if method == "p-eagle" else "") + "}")
+        return f"--speculative-config {cfg}"
+    return ""
+
+
+@dataclass(frozen=True)
+class ServingSpec:
+    model: str
+    engine: str            # "llamacpp" | "ollama"
+    image: str
+    port: int
+    model_ref: str         # "repo:quant" (llama.cpp -hf) or an ollama pull tag
+    vram_gb: float         # VRAM the chosen quant needs
+    reasoning: bool
+    ctx: int
+    args: str              # container args (llama.cpp) — empty for ollama (uses env/pull)
+    #: THE FOREGROUND COMMAND a pod must run, `args` included. A recipe that stops at flags is not
+    #: deployable: `lmsysorg/sglang` has no server CMD of its own, so a pod created with only the
+    #: flags starts, exits, and shows up as the "RUNNING with no runtime" failure the RunPod KB
+    #: opens with. llama.cpp's image DOES have one (`/app/llama-server`), so its flags are the
+    #: whole command. Ollama's comes from the node image's entrypoint and stays empty.
+    #:
+    #: This field exists because `ServingSpec.image`/`args` had no launcher: until 01-B4's bench
+    #: runner, `serving_spec` was called only by `model_fit_error`, which reads `vram_gb` and
+    #: `engine` and never starts anything. The tier-S recipe was therefore never executed.
+    command: str = ""
+    gpu_configs: list = field(default_factory=list)  # cost-ranked placements that fit
+
+    def cheapest(self):
+        return self.gpu_configs[0] if self.gpu_configs else None
+
+
+#: The runtimes `serving_spec` can actually produce a deployable command for.
+#:
+#: ComfyUI IS listed, and only because `_COMFYUI_MODELS` now builds a real fetch-and-launch command
+#: for the four models in it (image, music, video). It is still NOT a blanket claim about the runtime:
+#: the remaining comfyui rows have no entry there and `source_problem` reports them as missing a
+#: recipe. What this line must never go back to is naming a runtime whose branch does not exist —
+#: see the vllm note below for what that cost.
+#:
+#: **`vllm` WAS LISTED HERE AND SHOULD NOT HAVE BEEN** (caught 2026-10-01, by checking). There is no
+#: vLLM branch in `serving_spec` — the engines it builds for are SGLang, llama.cpp and Ollama — so
+#: naming vllm here made `source_problem()` report nothing for 34 models that resolve to an empty
+#: command, three of them EU models whose sources were pinned the same day (`mistral-large-3`,
+#: `pixtral`, `apertus-70b`). A list of what we can serve, kept by hand beside the code that serves,
+#: is the same drift this whole section is about; `test_servable_runtimes_are_runtimes_we_can_really
+#: _serve` now derives the check from the code instead of trusting this line.
+SERVABLE_RUNTIMES = frozenset({"sglang", "llamacpp", "ollama", "comfyui"})
+
+
+def _shquote(s: str) -> str:
+    """Quote one shell word. `json.dumps` would escape non-ASCII into \\uXXXX inside a shell string."""
+    return "'" + s.replace("'", "'\\''") + "'"
+
+
+def registry_source(model_id: str) -> dict:
+    """The PINNED source the registry records for this model, or `{}`.
+
+    28 §B. The weights' location is a fact about the MODEL, so it lives in `registry/models.yaml`
+    beside its licence and its bloc, and codegen refuses an enabled model that has neither a source
+    nor a stated reason for having none. It used to live in two hand-maintained dicts in this file,
+    which covered 4 of 29 enabled models — the other 25 fell through to the Ollama branch below and
+    got an EMPTY command, while this function's docstring claimed "nothing is unservable".
+    """
+    # `registry_gen.BY_ID`, NOT `manifest._BY_ID`. They are different dictionaries: the manifest holds
+    # its own hand-written `MODEL_CATALOG` with a reduced field set and no `source` at all, so reading
+    # it here returned `{}` for every model and the pins would have been dead data.
+    from .registry_gen import BY_ID as _REG
+    return dict((_REG.get(model_id) or {}).get("source") or {})
+
+
+def hf_token_configured() -> bool:
+    """Is a Hugging Face credential resolvable? PRESENCE ONLY — the value is never read here or logged.
+
+    `registry/providers.yaml` declares `HF_TOKEN` on the `huggingface` row and **nothing in the codebase
+    read it**, which matters now that five pinned sources are GATED (both meta-llama Llamas,
+    FLUX.1-schnell, SD-3.5-large, stable-audio-open). A gated repo pulled without a token answers 401, so
+    the pod reaches RUNNING, fails to start a server, is reaped by the readiness guard, and the whole
+    thing is diagnosed as a capacity shortage — the "fails after the money is spent" shape this plan
+    keeps finding. Asking BEFORE the launch turns that into a refusal that costs nothing.
+
+    Goes through `gpuproviders.provider_credential`, the one door: file 28 §O made the database the
+    source of truth, so an `os.environ` read here would answer "absent" for a deployment whose token
+    lives in the DB.
+    """
+    try:
+        from .gpuproviders import provider_credential
+        return bool(provider_credential("huggingface", "HF_TOKEN"))
+    except Exception:                          # pragma: no cover - registry unreachable offline
+        return False
+
+
+def source_problem(model_id: str) -> str:
+    """Why this model cannot be served from a pinned source, or "" when it can.
+
+    Said out loud rather than returned as an empty command. An unservable model used to be
+    indistinguishable from a servable one until a pod had been paid for and reaped.
+    """
+    from .registry_gen import BY_ID as _REG
+    cat = _REG.get(model_id)
+    if cat is None:
+        return f"{model_id} is not in the model registry"
+    if cat.get("source_unavailable"):
+        return f"{model_id} has no open weights: {cat['source_unavailable']}"
+    src = registry_source(model_id)
+    if not src:
+        return (f"{model_id} has no pinned source in registry/models.yaml, so there is nowhere to "
+                f"fetch its weights from")
+    runtimes = list(cat.get("runtime") or [])
+    if "ollama" in runtimes and not src.get("ollama_tag"):
+        return (f"{model_id} declares runtime 'ollama' and the registry has no ollama_tag for it "
+                f"(the catalog id is not a pullable tag)")
+    if not runtimes:
+        return f"{model_id} declares no runtime"
+    # AND A RUNTIME THIS MODULE CANNOT BUILD A COMMAND FOR IS A REASON, NOT A SILENCE. The first
+    # version of this function returned "" for the 13 `comfyui` models, which have real pinned
+    # sources and no serving recipe anywhere in the codebase — so it reported them as fine while
+    # `serving_spec` handed back an empty command. A pinned source is necessary and not sufficient.
+    if not any(r in SERVABLE_RUNTIMES for r in runtimes):
+        return (f"{model_id} declares runtime(s) {runtimes} and this module builds commands only for "
+                f"{sorted(SERVABLE_RUNTIMES)}. Its source is pinned; what is missing is a serving "
+                f"recipe for that runtime.")
+    if "comfyui" in runtimes and model_id not in _COMFYUI_MODELS:
+        return (f"{model_id} declares runtime 'comfyui' and has no entry in the ComfyUI serving "
+                f"table; this module builds commands only for the ComfyUI models listed there. "
+                f"Its source is pinned; what is missing is a serving recipe for that model")
+    if "llamacpp" in runtimes and "ollama" not in runtimes and model_id not in _LLAMACPP_MODELS:
+        return (f"{model_id} declares runtime 'llamacpp' and has no entry in the llama.cpp serving "
+                f"table, so there is no quant or context length to serve it with")
+    # THE SAME GAP, FOR SGLANG, and it was silent until the 2026-10-10 scan walked into it.
+    # `SERVABLE_RUNTIMES` includes sglang, but the recipe comes from `_FP8_MODELS`, which has two
+    # entries. So a model declaring sglang with a PINNED source passed every check above and
+    # returned "" — reported as fine while `serving_spec` produced an empty command. The llamacpp
+    # branch directly above exists for exactly this shape; this is its missing sibling, found by
+    # `test_28_pinned_model_sources.py::test_no_model_anywhere_is_silently_unservable` the moment
+    # two rows with real pinned sources (deepseek-v4.1-flash, glm-5.2) declared sglang.
+    if ("sglang" in runtimes and "ollama" not in runtimes and "llamacpp" not in runtimes
+            and model_id not in _FP8_MODELS):
+        return (f"{model_id} declares runtime 'sglang' and has no entry in the FP8/SGLang serving "
+                f"table, so there is no tensor-parallel degree or context length to serve it with")
+    # GATED AND NO TOKEN: pinned, real, and not fetchable. REPORTED LAST, after the runtime questions,
+    # because a missing recipe is the more fundamental blocker — telling someone to accept a licence and
+    # store a token for a model that nothing in this codebase can stand up sends them to do useless work.
+    # All three gated models that reach here are ComfyUI, and that is what they are told.
+    #
+    # Only where the weights would actually come from the gated repo: `gated` is a fact about the HUGGING
+    # FACE repo, and Ollama serves several of these with no HF token at all — `meta-llama/Llama-3.2-1B` is
+    # gated and `llama3.2:1b` is not. Without that narrowing this reported both Llamas as unservable,
+    # which would have been a false refusal of two models that work today.
+    if src.get("gated") and not src.get("ollama_tag") and not hf_token_configured():
+        return (f"{model_id} is pinned to the GATED repo {src.get('hf_repo')!r} and no Hugging Face "
+                f"credential resolves, so the pull would answer 401 after the GPU is already billing. "
+                f"Accept the model's licence on Hugging Face and store an HF_TOKEN credential.")
+    return ""
+
+
+
+#: ComfyUI models we have a REAL recipe for. Two shapes:
+#:   `file`   — one checkpoint that exists inside the REGISTRY-PINNED repo (SDXL publishes one).
+#:   `repack` — the pinned repo is not ComfyUI-loadable, so the weights come from a repackaged
+#:              republication, named explicitly with its own revision. See the rows for why.
+#: Keyed per model, NOT per runtime, and that distinction is load-bearing: an earlier version of
+#: this keyed the branch on `runtime == comfyui` and so claimed a recipe for all 13 comfyui models
+#: while hardcoding SDXL`s filename. The remaining rows have pinned sources but no checked file
+#: layout, and a generic fetch would download nothing and serve nothing.
+#:
+#: ADD A ROW ONLY WITH EVERY PATH CHECKED AGAINST THE HF FILE TREE AT THAT EXACT REVISION — not
+#: from memory of what these repos usually contain. The nine paths here were verified 200, with a
+#: deliberately bogus path checked for 404 first to prove the check could fail at all.
+_COMFYUI_MODELS = {
+    "sdxl-1.0": {"file": "sd_xl_base_1.0.safetensors", "vram_gb": 10.0},
+    # AUDIO GENERATION (sound/SFX, distinct from ACE-Step`s music). The pinned repo publishes a
+    # top-level single-file checkpoint that ComfyUI core loads, so this is the `file` shape and
+    # needs no repack — checked in the HF tree at the pinned revision.
+    #
+    # `as` EXISTS BECAUSE OF THIS ROW. The published name is literally `model.safetensors`, and
+    # the network volume is SHARED by every model in this table: dropped in as-is it would sit at
+    # models/checkpoints/model.safetensors, where the next model that publishes a generically
+    # named checkpoint silently overwrites it — and the `[ -s ]` guard would then treat the WRONG
+    # 4.9 GB file as already-downloaded and skip the fetch forever. A distinct basename is what
+    # makes the idempotence guard mean what it says.
+    #
+    # STILL GATED: the pull answers 401 without an accepted licence and an HF_TOKEN credential.
+    # That is deliberately NOT worked around here; `source_problem` reports it, and it reports it
+    # only now that a recipe exists, because a licence is useless for a model nothing can start.
+    "stable-audio-open-1.5": {"file": "model.safetensors", "vram_gb": 6.0,
+                              "as": "stable_audio_open_1.0.safetensors"},
+    # THE OTHER THREE CANNOT USE `file`, AND THE REASON IS THE WHOLE POINT OF `repack`.
+    # Their registry pins are the ORIGINAL publications, and none of them is a single file
+    # ComfyUI can load (checked against the HF file tree at the pinned revision, not recalled):
+    #   Qwen/Qwen-Image              -> diffusers layout: model_index.json + transformer/ in
+    #                                   9 shards + text_encoder/ in 4. No single checkpoint.
+    #   Wan-AI/Wan2.2-T2V-A14B       -> original Wan layout: high_noise_model/ and
+    #                                   low_noise_model/ in 6 shards each + an 11.4 GB .pth
+    #                                   text encoder. ~115 GB, and not a ComfyUI checkpoint.
+    #   ACE-Step/ACE-Step-v1-3.5B    -> ace_step_transformer/ + umt5-base/, diffusers-style.
+    # So the REGISTRY PIN REMAINS THE PROVENANCE RECORD and `repack` names the ComfyUI-loadable
+    # re-publication of those same weights that the pod actually downloads. The served bytes are
+    # therefore NOT the pinned bytes, which is a real difference and is why it is a named field
+    # with its own revision rather than a quiet substitution inside the fetch string.
+    "qwen-image-20b": {"vram_gb": 24.0, "repack": {
+        "repo": "Comfy-Org/Qwen-Image_ComfyUI",
+        "revision": "1f12b17be14c89b026c51a91d67c32f84bb047bc",
+        # fp8 over bf16: 20.4 GB vs 40.9 GB, and the registry sizes this model at 24 GB VRAM —
+        # the bf16 weights alone would not fit the placement the catalog promises.
+        "files": [("split_files/diffusion_models/qwen_image_fp8_e4m3fn.safetensors",
+                   "diffusion_models"),
+                  ("split_files/text_encoders/qwen_2.5_vl_7b_fp8_scaled.safetensors",
+                   "text_encoders"),
+                  ("split_files/vae/qwen_image_vae.safetensors", "vae")]}},
+    "ace-step-1.5": {"vram_gb": 8.0, "repack": {
+        "repo": "Comfy-Org/ACE-Step_ComfyUI_repackaged",
+        "revision": "e39503e8265a02363b8c6d3fed3732944f4fc67f",
+        # The only all-in-one in this set: transformer + text encoder + VAE in one checkpoint,
+        # so it loads from models/checkpoints like SDXL does.
+        "files": [("all_in_one/ace_step_v1_3.5b.safetensors", "checkpoints")]}},
+    "wan-2.2": {"vram_gb": 24.0, "repack": {
+        "repo": "Comfy-Org/Wan_2.2_ComfyUI_Repackaged",
+        "revision": "ee6f4a40737a995bf5818954cfce6d59443b0f04",
+        # Wan 2.2 A14B is a TWO-EXPERT model: a high-noise expert for the early steps and a
+        # low-noise one for the late steps. BOTH are required — a graph with one of them is not
+        # this model — which is why two diffusion_models rows is the minimum, not a choice.
+        # fp8_scaled (14.3 GB each) over fp16 (28.6 GB each) for the same 24 GB reason as above;
+        # the two experts load in sequence, so peak VRAM is one expert, not both.
+        "files": [("split_files/diffusion_models/wan2.2_t2v_high_noise_14B_fp8_scaled.safetensors",
+                   "diffusion_models"),
+                  ("split_files/diffusion_models/wan2.2_t2v_low_noise_14B_fp8_scaled.safetensors",
+                   "diffusion_models"),
+                  ("split_files/text_encoders/umt5_xxl_fp8_e4m3fn_scaled.safetensors",
+                   "text_encoders"),
+                  ("split_files/vae/wan2.2_vae.safetensors", "vae")]}},
+}
+
+#: Disk the weights occupy on the network volume, per model, from the sizes in the HF file tree at
+#: the revisions above. Recorded because the volume is a FIXED, DATACENTER-LOCKED size: the recipes
+#: below total ~81 GB, so a 60 GB volume cannot hold all four and the bring-up fails LATE — after
+#: the GPU is billing — with a disk-full error rather than a refusal. Callers that place these
+#: models are expected to check this against the volume before renting anything.
+COMFYUI_WEIGHT_GB = {
+    "sdxl-1.0": 6.94,
+    "stable-audio-open-1.5": 4.85,
+    "qwen-image-20b": 30.06,   # 20.43 + 9.38 + 0.25
+    "ace-step-1.5": 7.70,
+    "wan-2.2": 36.73,          # 14.29 + 14.29 + 6.74 + 1.41
+}
+
+def serving_spec(model_id: str) -> ServingSpec:
+    """Resolve the correct, deployable serving recipe for ANY catalog model. Falls back to an Ollama
+    recipe sized from the manifest VRAM for models without an explicit entry, so nothing is
+    unservable — the resolver always returns a working plan."""
+    if model_id in _FP8_MODELS:
+        m = _FP8_MODELS[model_id]
+        # Tensor parallelism splits attention heads across GPUs, so the degree has to divide the
+        # head count — in practice a power of two. A 3-GPU placement is cheaper on paper and
+        # then refuses to start, so it is not a placement at all.
+        plan = [c for c in gpu_plan(m["vram_gb"], require_fp8=True)
+                if c.count & (c.count - 1) == 0]
+        # Tensor-parallel degree must equal the GPU count of the placement we actually take,
+        # so it is resolved per config at deploy time; the cheapest fitting config sets it.
+        tp = max(int(m.get("min_gpus", 1)), plan[0].count if plan else 1)
+        parser = f" --reasoning-parser {m['reasoning_parser']}" if m.get("reasoning_parser") else ""
+        # FILE 11-B2. Appended rather than interleaved so the flags are readable in a log line, and
+        # EMPTY when this model has no method or no head — never a half-configured one.
+        spec = spec_decode_args(model_id, "sglang")
+        # REPO AND REVISION FROM THE REGISTRY (28 §B). `m['repo']` stays as the fallback for a model
+        # this dict knows and the registry does not, but the registry is the source of truth and
+        # carries the PIN — without `--revision` this pod served whatever `main` was that morning.
+        src = registry_source(model_id)
+        repo = src.get("hf_repo") or m["repo"]
+        rev = f" --revision {src['revision']}" if src.get("revision") else ""
+        args = (f"--model-path {repo}{rev} --host 0.0.0.0 --port 8080 "
+                f"--tp {tp} --context-length {m['ctx']}{parser} "
+                # SGLang spells the concurrency cap --max-running-requests; vLLM calls the same
+                # thing --max-num-seqs. Using the wrong name is silently ignored, which would
+                # leave the pod unbatched and put us straight back at $165-210/1M tokens.
+                f"--max-running-requests {TIER_S_MAX_SEQS} "
+                f"--kv-cache-dtype fp8_e5m2 --enable-hierarchical-cache"
+                + (f" {spec}" if spec else ""))
+        return ServingSpec(model_id, "sglang", SGLANG_IMAGE, 8080, repo, m["vram_gb"],
+                           m["reasoning"], m["ctx"], args,
+                           command=f"python3 -m sglang.launch_server {args}", gpu_configs=plan)
+    if model_id in _LLAMACPP_MODELS:
+        m = _LLAMACPP_MODELS[model_id]
+        # The GGUF repo and quant come from the registry where it has them. The revision is RECORDED
+        # there and deliberately NOT passed here: llama.cpp's `-hf` takes `repo:quant` and has no
+        # revision argument, so the pin documents what was tested rather than constraining the pull.
+        # Claiming otherwise with a flag the binary ignores would be worse than saying it plainly.
+        src = registry_source(model_id)
+        repo = src.get("gguf_repo") or m["repo"]
+        quant = src.get("gguf_quant") or m["quant"]
+        ref = f"{repo}:{quant}"
+        # --parallel 1 served ONE request at a time on the most expensive node we run. With
+        # continuous batching the same pod overlaps 8 streams, dividing the hourly cost.
+        args = (f"-hf {ref} --host 0.0.0.0 --port 8080 -ngl 999 -c {m['ctx']} --jinja "
+                f"--parallel {LLAMACPP_PARALLEL} --cont-batching")
+        return ServingSpec(model_id, "llamacpp", LLAMACPP_IMAGE, 8080, ref, m["vram_gb"],
+                           m["reasoning"], m["ctx"], args,
+                           # The image's ENTRYPOINT is llama-server itself, so the flags ARE the
+                           # command — passing it again would run llama-server inside llama-server.
+                           command=args, gpu_configs=gpu_plan(m["vram_gb"]))
+    # ComfyUI path: the diffusion models (image/video). 28 §B pins their weights; what was missing
+    # until now was a RECIPE, so `serving_spec` returned a spec with an EMPTY command and
+    # `SERVABLE_RUNTIMES` honestly excluded `comfyui`.
+    #
+    # WHY THE WEIGHTS GO TO THE NETWORK VOLUME AND NOT THE CONTAINER DISK. SDXL is ~7 GB; on the
+    # container disk it is re-downloaded on every boot and the pod bills the whole time it pulls.
+    # RunPod mounts a network volume at /workspace, which is also where this image keeps ComfyUI,
+    # so pointing the checkpoint dir there makes the download happen ONCE — the same trade
+    # `_volume_attachment` documents for the frontier model.
+    if model_id in _COMFYUI_MODELS:
+        m = _COMFYUI_MODELS[model_id]
+        src = registry_source(model_id)
+        repo, rev = src.get("hf_repo") or "", src.get("revision") or ""
+        if not repo:
+            log.warning("serving_spec(%s) has no runnable command: %s", model_id,
+                        source_problem(model_id) or "no pinned source for comfyui")
+            return ServingSpec(model_id, "comfyui", COMFYUI_IMAGE, COMFYUI_PORT, "",
+                               float(m["vram_gb"]), False, 0, "", command="",
+                               gpu_configs=gpu_plan(float(m["vram_gb"])))
+        from .registry_gen import BY_ID as _REG_V
+        vram = float((_REG_V.get(model_id) or {}).get("vram_gb")
+                     or (_BY_ID.get(model_id) or {}).get("vram_gb") or m["vram_gb"])
+        models_root = "/workspace/ComfyUI/models"
+        ckpt = f"{models_root}/checkpoints"
+        # WEIGHTS ON THE NETWORK VOLUME, not the container disk. RunPod mounts the volume at
+        # /workspace, so the weights are downloaded ONCE and every later boot is fast —
+        # the same trade `_volume_attachment` documents for the frontier model. The `[ -s ... ]`
+        # guard is what makes the second boot skip the download instead of repeating it.
+        pack = m.get("repack")
+        if pack:
+            # A REPACKAGED SOURCE, so `repo`/`rev` from the registry are NOT what we download —
+            # see the comment on `_COMFYUI_MODELS`. Each file is fetched into its own ComfyUI
+            # subfolder and renamed to its bare basename, because ComfyUI's loader nodes list
+            # `models/<kind>/` and would not see a file still nested under `split_files/`.
+            steps = []
+            for path, sub in pack["files"]:
+                base = path.rsplit("/", 1)[-1]
+                top = path.split("/", 1)[0]
+                d = f"{models_root}/{sub}"
+                # Download, lift the file out of the repo-relative path, drop the empty tree.
+                # `[ -s ... ]` first so a restart costs nothing.
+                steps.append(
+                    f"mkdir -p {d} && ( [ -s {d}/{base} ] || ( "
+                    f"huggingface-cli download {pack['repo']} {path}"
+                    + (f" --revision {pack['revision']}" if pack.get("revision") else "")
+                    + f" --local-dir {d} && mv -f {d}/{path} {d}/{base} && rm -rf {d}/{top} ) )")
+            fetch = " && ".join(steps)
+        else:
+            # `as` renames the checkpoint to a distinct basename; see the stable-audio row for
+            # why a generic published name is a correctness problem on a shared volume, not a
+            # cosmetic one. Without `as` the published name is kept (SDXL`s already distinct).
+            dest = m.get("as") or m["file"]
+            ren = f" && mv -f {ckpt}/{m['file']} {ckpt}/{dest}" if dest != m["file"] else ""
+            fetch = (f"mkdir -p {ckpt} && ( [ -s {ckpt}/{dest} ] || ( "
+                     f"huggingface-cli download {repo} {m['file']}"
+                     + (f" --revision {rev}" if rev else "")
+                     + f" --local-dir {ckpt}{ren} ) )")
+        args = f"--listen 0.0.0.0 --port {COMFYUI_PORT}"
+        return ServingSpec(model_id, "comfyui", COMFYUI_IMAGE, COMFYUI_PORT, repo, vram,
+                           False, 0, args,
+                           command=f"bash -lc '{fetch}; exec python3 /opt/ComfyUI/main.py {args}'",
+                           gpu_configs=gpu_plan(vram))
+    # Ollama path: single-file models, pulled by their PINNED TAG.
+    #
+    # THIS BRANCH USED TO RETURN AN UNSERVABLE SPEC AND SAY NOTHING. `model_ref` was `model_id` — the
+    # catalog id, e.g. `qwen3-8b`, which Ollama has never heard of; the real tag is `qwen3:8b` — and
+    # `command` was the empty string, so a pod built from it starts `ollama/ollama` with no server
+    # and no model: RUNNING with no runtime, billing until reaped. 25 of 29 enabled models reached
+    # this branch on 2026-10-01, which is the whole of 28 §B.
+    # VRAM FROM THE REGISTRY FIRST. `manifest._BY_ID` is the reduced hand-written catalog, so any model
+    # absent from it fell through to a FABRICATED 8.0 — and unfinished.md L13 names that as a money risk
+    # in as many words: "`serving_spec()` FABRICATES an 8 GB ollama spec for any unknown id ... so a
+    # provisioning script that trusts it rents an 8 GB pod for a 400 GB model". `mistral-large-3`
+    # (registry: 400 GB) reported 8.0. The empty command already stops a pod being created from it, but
+    # `vram_gb` and `gpu_configs` are read by other callers — `model_fit_error` among them — and an
+    # under-stated weight is the "bills and cannot start" shape.
+    from .registry_gen import BY_ID as _REG
+    reg = _REG.get(model_id) or {}
+    cat = _BY_ID.get(model_id)
+    vram = float(reg.get("vram_gb") or (cat or {}).get("vram_gb") or 8.0)
+    ctx = 16384
+    src = registry_source(model_id)
+    tag = str(src.get("ollama_tag") or "")
+    if not tag:
+        # NO GUESSING. `model_id` is not a tag, and a wrong tag is a pod that pulls nothing or pulls
+        # the wrong model. An empty command is returned ONLY here, where there is genuinely nothing
+        # to run — and the reason is LOGGED rather than left for the caller to discover by deploying.
+        #
+        # This is also `source_problem`'s production caller. Written without one it was a function the
+        # tests proved and nothing used (rule 12, caught by `test_no_orphan_implementations`), which is
+        # the same shape as the 25 unservable models: a truth recorded nowhere anybody reads.
+        log.warning("serving_spec(%s) has no runnable command: %s", model_id,
+                    source_problem(model_id) or "no pinned source for its runtime")
+        return ServingSpec(model_id, "ollama", OLLAMA_IMAGE, 11434, "", max(vram, 1.0),
+                           False, ctx, "", command="",
+                           gpu_configs=gpu_plan(vram if vram >= 8 else 8))
+    # `ollama serve` is the foreground process and `ollama pull` needs it already listening, so the
+    # pull happens in a subshell that waits for the port rather than sleeping a guessed number of
+    # seconds. If the pull fails the container EXITS instead of serving an endpoint with no model
+    # behind it — the same choice the 11-B2 two-server pod makes, and for the same reason.
+    args = (f'(until ollama list >/dev/null 2>&1; do sleep 1; done; '
+            f'ollama pull {tag} || kill 1) & exec ollama serve')
+    return ServingSpec(model_id, "ollama", OLLAMA_IMAGE, 11434, tag, max(vram, 1.0),
+                       False, ctx, args, command=f"bash -lc {_shquote(args)}",
+                       gpu_configs=gpu_plan(vram if vram >= 8 else 8))
+
+
+# `engine_for()` AND `vllm_args()` WERE DELETED HERE, 2026-09-23 (rule 12).
+#
+# Both were second spellings of a decision `serving_spec()` already owns, and this module's own
+# docstring says it is "the single place that answers, for a given model id: which ENGINE serves
+# it ... [and] the container IMAGE". Neither had a caller.
+#
+#   engine_for(m)  ==  serving_spec(m).engine, off the same two tables, in the same order. The
+#     one production consumer of this module — `gpuproviders.model_fit_error` — already holds a
+#     `ServingSpec` when it needs the engine, so a caller for `engine_for` could only ever have
+#     been a redundant second lookup that drifts the day the resolver grows a branch.
+#
+#   vllm_args(m)   was the 01-B1 fallback engine, and it DISAGREED with the primary path in
+#     three ways that each produce a pod that bills and cannot serve:
+#       * it emitted no `--reasoning-parser`, the flag `serving_spec` carries because a live pod
+#         on 2026-08-28 was MEASURED returning its <think> block inside `content` without it;
+#       * `tp` defaulted to 1, where `serving_spec` derives the tensor-parallel degree from the
+#         placement it actually takes (a 235 GB model on 2xH200 at --tp 1 does not load);
+#       * for a GGUF-only model it cheerfully produced vLLM args for an engine that cannot read
+#         GGUF at all.
+#     Nothing in the deployment serves vLLM: no entrypoint, no spec, and `VLLM_IMAGE` — which
+#     went with them — was referenced by nothing at all, not even by `vllm_args`. The fallback
+#     engine of 01-B1 is unbuilt, and a weaker copy of the args is not a head start on building
+#     it: the reasoning-parser flag above is the record of what shipping the weaker copy costs.