frutebox-node 0.1.0__py3-none-any.whl

frutebox/provider.py

@@ -0,0 +1,1867 @@
+"""frutebox provider node.
+
+This is the piece an organisation or individual installs on their own server /
+GPU box (the "downloadable software"). It makes a single *outbound* WebSocket
+connection to a network **gateway** and receives inference jobs over it. Because
+the connection is outbound, the node needs no public IP, no inbound firewall
+rules and no port forwarding — it works from inside a corporate network or
+behind home NAT. Prompts arrive from the gateway (the network), never from the
+Google-Cloud directory.
+
+Run:
+    python -m frutebox.provider --name berlin-gpu-1 --region eu-central \\
+        --gateway-url ws://127.0.0.1:8090/provider/ws
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import subprocess
+import time
+import uuid
+from collections import deque
+from pathlib import Path
+from typing import Deque
+
+import websockets
+
+from . import attest, engine_guard, inference, packaging
+from .config import CONFIG
+from .envvars import env as envvar
+
+#: Reply fields that carry CONTENT and must be sealed back to the gateway (file 22-A1). An
+#: embedding is a lossy but real function of the user's text — the owner's ruling treats it as
+#: user data — so it may no more travel home in the clear than the document travelled out in it.
+#: `type`, `id` and `model` stay readable: the gateway matches and meters the reply on those.
+#:
+#: PHASE TWO of the `result` cutover adds the answer itself. A completion is the user's data
+#: exactly as much as their prompt is, and it used to come home in the clear on every chat turn:
+#: `content` and `reasoning` are what the model wrote, `tool_calls` are the arguments it chose
+#: (a tool call carries the user's own words as often as not), `text` is one streamed delta of
+#: the same answer, and `artifact` is a whole generated image/audio/video, base64 on the frame.
+#:
+#: WHAT DELIBERATELY STAYS READABLE: `tokens`, `prompt_tokens`, `backend` and `compute_ms`. The
+#: gateway meters and reputation-scores on those, they are counts and names rather than content,
+#: and they are the same class of routing metadata that `_seal_job` leaves in the clear on the
+#: way out. `channel` stays readable for the same reason — it says which stream a delta belongs
+#: on, not what is in it.
+_SEALED_REPLY_FIELDS = ("embeddings", "content", "reasoning", "tool_calls", "text", "artifact")
+
+
+def _provider_ws_url(public_url: str) -> str:
+    """The gateway's provider socket, derived from the https url it heartbeats to the directory.
+
+    The directory stores what a gateway calls itself (`STEVAI_GATEWAY_URL`), which is an http(s)
+    origin. Nodes need the WebSocket endpoint on that same origin, so the scheme is upgraded and
+    the well-known path appended — rather than asking every gateway to publish a second url that
+    could drift out of step with the first.
+    """
+    u = (public_url or "").strip().rstrip("/")
+    if not u:
+        return ""
+    if u.startswith("https://"):
+        u = "wss://" + u[len("https://"):]
+    elif u.startswith("http://"):
+        u = "ws://" + u[len("http://"):]
+    elif not u.startswith(("ws://", "wss://")):
+        return ""
+    return u if u.endswith("/provider/ws") else u + "/provider/ws"
+
+
+def _detect_gpu_vram_gb() -> float:
+    """Best-effort total GPU VRAM in GB via nvidia-smi; 0.0 if unavailable.
+
+    Lets a real GPU (e.g. an A40's ~48GB) declare its true capacity so the
+    directory tiers it correctly instead of defaulting to the CPU/small tier.
+    """
+    try:
+        out = subprocess.run(
+            ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
+            capture_output=True, text=True, timeout=5,
+        )
+        if out.returncode == 0:
+            mibs = [int(x) for x in out.stdout.split() if x.strip().isdigit()]
+            if mibs:
+                return round(max(mibs) / 1024.0, 1)
+    except Exception:
+        pass
+    return 0.0
+
+
+def _detect_system_memory_gb() -> float:
+    """Best-effort total system RAM in GB (stdlib only); 0.0 if undetermined.
+
+    Lets an ordinary laptop/desktop declare its REAL RAM to the directory instead
+    of a hardcoded guess, so CPU-RAM pooling budgets are honest. Linux: parse
+    /proc/meminfo (MemTotal, kB). macOS/BSD: `sysctl -n hw.memsize` (bytes). Last
+    resort: POSIX sysconf. No new pip deps.
+    """
+    # Linux (and any OS exposing /proc/meminfo).
+    try:
+        with open("/proc/meminfo", "r") as f:
+            for line in f:
+                if line.startswith("MemTotal:"):
+                    return round(int(line.split()[1]) / (1024.0 * 1024.0), 1)
+    except Exception:
+        pass
+    # macOS / BSD: hw.memsize is total physical RAM in bytes.
+    try:
+        out = subprocess.run(["sysctl", "-n", "hw.memsize"],
+                             capture_output=True, text=True, timeout=5)
+        if out.returncode == 0 and out.stdout.strip().isdigit():
+            return round(int(out.stdout.strip()) / (1024.0 ** 3), 1)
+    except Exception:
+        pass
+    # POSIX fallback (many Unixes expose these sysconf names).
+    try:
+        pages = os.sysconf("SC_PHYS_PAGES")
+        page_size = os.sysconf("SC_PAGE_SIZE")
+        if pages > 0 and page_size > 0:
+            return round(pages * page_size / (1024.0 ** 3), 1)
+    except (ValueError, OSError, AttributeError):
+        pass
+    return 0.0
+
+
+def _detect_cpu_cores() -> int:
+    """Logical CPU count via the stdlib; 1 if the OS won't say."""
+    return os.cpu_count() or 1
+
+
+# Gateway close codes with a meaning for the client (mirrors frutebox/gateway.py).
+# Blind-retrying either of these is what turned a lapsed grant into a permanent
+# eviction, so they are handled explicitly in run().
+_CLOSE_REJECTED = 4003        # credential refused (or attestation failed)
+_CLOSE_DUPLICATE = 4004       # this node id already has a live socket on that gateway
+_CLOSE_CRED_EXPIRED = 4401    # credential merely lapsed — fetch a new one
+_CLOSE_ROTATED = 4005         # our OWN replacement socket took over (F-20.3)
+# How long to wait before dialling again once the gateway told us the credential is
+# no good. Far longer than the normal backoff: nothing changes by trying sooner, and
+# a tight loop is exactly the pattern that looked like an attack.
+_CRED_RETRY_S = 60.0
+
+
+#: Where the node publishes whether it is ACTUALLY joined to a gateway. The RunPod status sidecar
+#: serves this file, and it exists because the boot-state file could not answer the only question
+#: that matters. `ready` there means "engine up, models pulled, agent about to launch" — the
+#: entrypoint says so in full — and a pod on 2026-09-28 sat at `ready`, both models loaded, with
+#: the gateway's provider count at 0 for its whole life. Another did the same for ~18 hours on
+#: 2026-09-29, billing the entire time, and nothing on the pod could be asked "are you serving?".
+#: The node is the only party that knows, so the node writes it down.
+NODE_STATE_FILE = envvar("NODE_STATE_FILE", "/var/run/frutebox_node_state")
+
+#: Rotate a socket once it reaches this age (F-20.3, make-before-break). It must sit comfortably
+#: BELOW the gateway's request timeout — Cloud Run caps a WebSocket at `timeoutSeconds`, currently
+#: 3600 — because the point is to replace the socket on OUR schedule, while it is still healthy,
+#: rather than to discover it gone on the platform's.
+SOCKET_MAX_AGE_S = float(envvar("SOCKET_MAX_AGE_S", "3000"))
+
+
+def _publish_link_state(state: str) -> None:
+    """Record the gateway link's state. Best effort, and NEVER fatal.
+
+    A laptop node has no business failing because `/var/run` is not writable, so every error is
+    swallowed — an unobservable node is a smaller problem than a node that will not run.
+    """
+    try:
+        with open(NODE_STATE_FILE, "w") as fh:
+            fh.write(f"{state} {time.time():.0f}\n")
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def _close_code(exc: BaseException) -> int:
+    """Best-effort WebSocket close code from a websockets exception, across versions
+    (`.code` on older releases, `.rcvd.code` on newer ones). 0 = unknown."""
+    code = getattr(exc, "code", None)
+    if isinstance(code, int):
+        return code
+    rcvd = getattr(exc, "rcvd", None)
+    code = getattr(rcvd, "code", None)
+    return code if isinstance(code, int) else 0
+
+
+def _frame_id(msg: dict):
+    """A HASHABLE job id for a frame off the wire.
+
+    The gateway sends a string, but the frame is attacker-influenced data and a list or
+    dict there would raise ``TypeError: unhashable`` from ``set.add``/``set.discard`` —
+    including from the ``finally`` that releases the concurrency slot, which turns a
+    typo into a leaked slot.
+    """
+    jid = msg.get("id")
+    if isinstance(jid, (str, int, float, bool)) or jid is None:
+        return jid
+    return repr(jid)[:256]
+
+
+def _load_credential(args) -> str:
+    """Resolve this node's signed admission credential: --credential, else the file
+    named by --credential-file, else $FRUTEBOX_NODE_CREDENTIAL (or the legacy
+    $STEVAI_NODE_CREDENTIAL), else "" (no credential).
+
+    A credential is ~4.5 KB (hybrid Ed25519 + ML-DSA-65 signatures), so the FILE form
+    is the practical one for a service unit or a shell — and it never lands in the
+    process list the way a flag does. It is a bearer grant for this node id: keep it
+    out of logs and never put it in a URL.
+    """
+    direct = (getattr(args, "credential", "") or "").strip()
+    if direct:
+        return direct
+    path = (getattr(args, "credential_file", "") or "").strip()
+    if path:
+        try:
+            return Path(path).read_text(encoding="utf-8").strip()
+        except OSError as e:
+            print(f"[node] WARNING: could not read --credential-file {path}: "
+                  f"{type(e).__name__} — connecting without a credential")
+    return envvar("NODE_CREDENTIAL", "").strip()
+
+
+class ProviderNode:
+    def __init__(self, args):
+        self.id = args.id or f"{args.region}-{uuid.uuid4().hex[:6]}"
+        self.name = args.name or self.id
+        self.region = args.region
+        # Hardware class. An explicit --compute-type wins; otherwise AUTO-DETECT:
+        # a real GPU (nvidia-smi reports VRAM) → "gpu", else fall back to "cpu" so
+        # a plain laptop/desktop contributes as a CPU node instead of falsely
+        # declaring a GPU it doesn't have.
+        vram_gb = _detect_gpu_vram_gb()
+        self.compute_type = getattr(args, "compute_type", None) or (
+            "gpu" if vram_gb > 0 else "cpu")
+        # Memory this node contributes (VRAM for a GPU, system RAM for a CPU).
+        # This is what lets many small CPUs POOL their RAM to hold a big model.
+        self.memory_gb = float(getattr(args, "memory_gb", 0.0) or 0.0)
+        if self.memory_gb <= 0:  # not declared → measure the real hardware, don't guess
+            if self.compute_type == "gpu":
+                self.memory_gb = vram_gb or 24.0
+            else:
+                self.memory_gb = _detect_system_memory_gb() or 32.0
+        # CPU cores: explicit flag wins, else probe the machine (stdlib os.cpu_count()).
+        self.cpu_cores = int(getattr(args, "cpu_cores", 0) or 0) or _detect_cpu_cores()
+        self.models = [m.strip() for m in args.models.split(",") if m.strip()]
+        if not self.models:  # advertise the concrete model this node actually runs,
+            # A COMMA-SEPARATED LIST, same as `--models`. One card can hold several models
+            # (a 48 GB A40 fits qwen2.5:32b + gemma-27b), and treating the env value as a
+            # single name would advertise a model called "qwen2.5:32b,gemma-27b" — which
+            # matches nothing in routing, so the node registers and is never routed to.
+            raw = envvar("OLLAMA_MODEL", "")  # not "serves anything"
+            self.models = [m.strip() for m in raw.split(",") if m.strip()]
+        self.cost_per_1k = args.cost_per_1k
+        # Simultaneous jobs this node will actually run. Advertised to the directory AND
+        # enforced below — before, every inbound job frame became a task regardless, so
+        # one user could start unlimited generations on somebody's laptop. An explicit
+        # --concurrency wins; otherwise the node-governance default applies.
+        self.max_concurrency = max(1, int(getattr(args, "concurrency", 0)
+                                          or CONFIG.node_max_concurrent_jobs))
+        # Hard wall-clock ceiling per job (cancel included).
+        self.job_timeout_s = float(getattr(args, "job_timeout_s", 0.0)
+                                   or CONFIG.node_job_timeout_s)
+        self.gateway_url = args.gateway_url
+        # Signed admission credential (minted by the directory for THIS node id and
+        # handed to the machine's owner). Presented on every (re)connect; the gateway
+        # verifies the operator's signature and admits us without waiting for a manual
+        # approval that would be lost on the next control-plane restart. Optional —
+        # without one we fall back to the digest + approval-list path.
+        # `_cred_args` is kept so the credential can be RE-READ before each connect:
+        # dropping a fresh file at --credential-file is then enough to recover from an
+        # expiry, with no restart and no reconnect loop hammering the stale one.
+        self._cred_args = args
+        self.credential = _load_credential(args)
+        # What this node can GENERATE. Ollama chat models are text-only; a capable
+        # node (image/audio/video) advertises more via STEVAI_GEN_MODALITIES.
+        self.output_modalities = [
+            m.strip() for m in envvar("GEN_MODALITIES", "text").split(",")
+            if m.strip()] or ["text"]
+        # PLAN 23 §B2 — "The router never offers a video/music job to a Tier-C/edge node —
+        # Tier B/A GPUs only (8-16 GB+)." The refusal belongs HERE, at the advertisement,
+        # not only at the router: a node that advertises a modality it must then refuse
+        # wastes a dispatch and looks like capacity that does not exist. A machine below the
+        # 8 GB floor stays text-only no matter which model libraries happen to be installed
+        # on it, and `GEN_MODALITIES` cannot be used to opt an edge node back in.
+        # Guarded like every other media import in this block: an import failure here must
+        # degrade the node to text-only, not stop it starting. A node that serves text is
+        # useful; a node that refuses to boot is not.
+        try:
+            from . import mediaserving
+            from .imagegen import device_vram_gb
+            _vram = device_vram_gb()
+            self.media_tier = mediaserving.node_tier_for_vram(_vram)
+            self.output_modalities = mediaserving.media_modalities_for_node(
+                self.output_modalities, _vram)
+            _media_ok = mediaserving.tier_accepts(self.media_tier)
+        except Exception:
+            self.media_tier = "C"
+            self.output_modalities = [m for m in self.output_modalities
+                                      if m not in ("image", "audio", "video")] or ["text"]
+            _media_ok = False
+        if _media_ok:
+            # Auto-advertise generation modalities this node can actually serve (the model libs
+            # are only present on a capable GPU node).
+            try:
+                from . import imagegen
+                if imagegen.available() and "image" not in self.output_modalities:
+                    self.output_modalities.append("image")
+            except Exception:
+                pass
+            try:
+                from . import audiogen
+                if audiogen.available() and "audio" not in self.output_modalities:
+                    self.output_modalities.append("audio")
+            except Exception:
+                pass
+            try:
+                from . import videogen
+                if videogen.available() and "video" not in self.output_modalities:
+                    self.output_modalities.append("video")
+            except Exception:
+                pass
+
+        # AND DROP WHAT THIS NODE CANNOT ACTUALLY DO, however it came to be listed.
+        #
+        # The three blocks above only ever ADD. A modality named in `GEN_MODALITIES` was therefore
+        # advertised whether or not its generator could run, and `mediaserving` says plainly why
+        # that is wrong: "A node that advertises a modality it must then refuse is capacity that
+        # does not exist: the dispatch is wasted and the router has no way to know."
+        #
+        # MEASURED 2026-10-14. The media pod's spec sets GEN_MODALITIES=text,image,audio,video, and
+        # its boot had just deleted the venv holding torch. So it advertised `image`, cleared the
+        # gateway's modality gate, took the job, and answered "No module named 'torch'" — 38 times.
+        # The gateway reported "no node is serving 'image'", which sent the search to capacity and
+        # routing while the real answer was a missing import on a node that was up and connected.
+        #
+        # Declared-but-unavailable now degrades to TEXT, which makes the gate's refusal honest and
+        # immediate: "no node in the network can generate 'image' yet" is true and actionable,
+        # where a wasted dispatch is neither.
+        for _mod, _mod_name in (("image", "imagegen"), ("audio", "audiogen"), ("video", "videogen")):
+            if _mod not in self.output_modalities:
+                continue
+            try:
+                _gen = __import__(f"{__package__}.{_mod_name}", fromlist=[_mod_name])
+                if _gen.available():
+                    continue
+                _why = "its generator reports unavailable"
+            except Exception as _e:
+                _why = f"{type(_e).__name__}: {_e}"
+            self.output_modalities.remove(_mod)
+            print(f"[{self.name}] NOT advertising '{_mod}' — {_why}. "
+                  f"The operator asked for it in GEN_MODALITIES; this node cannot serve it, and "
+                  f"advertising it would turn every such job into a wasted dispatch.")
+
+        # Arm the engine guard for THIS machine: from here on a job may only name a
+        # model this node advertises. The engine has no authentication and treats an
+        # unknown model as one to go and download, so an unclamped model string was a
+        # remote-controlled disk/bandwidth spend on the owner's hardware.
+        #
+        # REFUSE TO START UNARMED. `arm_models([])` (or `[""]`) leaves the allowlist
+        # None, which makes `check_model` a process-wide PASS-THROUGH — the guard
+        # silently disarms itself and every model name a job sends reaches the engine
+        # verbatim. That happened for real with `--models` unset and a set-but-EMPTY
+        # $FRUTEBOX_OLLAMA_MODEL, because the default only applies to a *missing* var.
+        # A node with nothing to serve is useless anyway, so fail loudly at startup
+        # rather than run wide open.
+        serves = [m for m in [*self.models, inference.DEFAULT_MODEL] if str(m or "").strip()]
+        if not serves:
+            raise SystemExit(
+                "frutebox node: no model to serve — pass --models, or set "
+                "$FRUTEBOX_OLLAMA_MODEL to a model this machine actually has. "
+                "(Refusing to start: an empty allowlist would disarm the engine guard "
+                "and let any job name any model for the engine to download.)")
+        # The embedding model is armed too: `inference.embed()` goes through the same
+        # guard, so retrieval would otherwise be refused on every node.
+        engine_guard.arm_models([*serves, inference.EMBED_MODEL])
+        if not engine_guard.models_armed():          # belt to the check above's braces
+            raise SystemExit("frutebox node: engine guard failed to arm — refusing to start")
+
+        # Placement trust label, assigned by the OPERATOR and delivered by the gateway
+        # in the `registered` frame — never self-declared. Gates node-side tool
+        # execution (which is additionally off by default; see engine_guard).
+        self.trust = "untrusted"
+
+        self.active = 0
+        self.rejected_jobs = 0
+        # Job ids the gateway asked to cancel mid-stream (§5A live steering / client stop).
+        # A running stream job checks this each delta and stops — freeing the GPU and
+        # halting billing at the partial. The receive loop adds ids; the job clears them.
+        #
+        # A cancel is only ever recorded for a job that is IN FLIGHT right now (see
+        # `_inflight`). Recording any id the gateway names was an unbounded, never-swept
+        # set — and a cancel sent *before* the matching job frame silently killed that
+        # job the moment it started. Bounded by max_concurrency by construction.
+        self._cancel_jobs: set = set()
+        # Job ids this node has taken a slot for and not yet finished.
+        self._inflight: set = set()
+        #: Registered sockets per gateway url. A rotated-out socket must NOT trigger a reconnect
+        #: while its replacement is already holding the link — that would turn make-before-break
+        #: into two sockets fighting for one node id, which the gateway refuses as a duplicate.
+        self._live_sockets: dict = {}
+        #: Replacement-socket tasks, held so they are not garbage collected mid-flight.
+        self._rotations: set = set()
+        # PER-STREAM REPLY SESSIONS (file 22-A1, phase two). One `pqctransport.Session` per job
+        # the gateway asked for a sealed answer on, keyed by job id and dropped when the job
+        # ends. One handshake per stream, AEAD per frame — the owner's ruling on the measured
+        # per-chunk cost. Bounded by max_concurrency by construction, like `_cancel_jobs`.
+        self._reply_sessions: dict = {}
+        self.recent_ms: Deque[float] = deque(maxlen=20)
+        self._send_lock = asyncio.Lock()
+        self._stop = False
+
+        # Live stats for the local dashboard.
+        self.gui_port = int(getattr(args, "gui_port", 0) or 0)
+        self.connected = False
+        self.jobs_served = 0
+        self.tokens_served = 0
+        #: Per-job token counts, paired with `recent_ms`, so throughput is measured and
+        #: never estimated (file 16 mock audit 2026-08-31).
+        self.recent_tokens: Deque[int] = deque(maxlen=20)
+        self.started_at = time.time()
+
+    # --- helpers ----------------------------------------------------------
+    @property
+    def load(self) -> float:
+        return min(1.0, self.active / max(1, self.max_concurrency))
+
+    def _record_job(self, tokens: int) -> None:
+        self.jobs_served += 1
+        n = int(tokens or 0)
+        self.tokens_served += n
+        # Paired with the latency sample recorded for the same job, so `measured_tps` is
+        # arithmetic over real observations rather than a table of guesses.
+        self.recent_tokens.append(n)
+
+    # --- concurrency cap --------------------------------------------------
+    # Taken/released in the RECEIVE LOOP, synchronously, before the task is created:
+    # counting inside the task leaves a window where N frames all pass the check.
+    def _acquire_slot(self) -> bool:
+        if self.active >= self.max_concurrency:
+            return False
+        self.active += 1
+        return True
+
+    def _release_slot(self) -> None:
+        self.active = max(0, self.active - 1)
+
+    @property
+    def estimated_earnings(self) -> float:
+        # Illustrative: the node's declared cost per 1k tokens for the work done.
+        return round(self.tokens_served / 1000.0 * self.cost_per_1k, 2)
+
+    def stats(self) -> dict:
+        hw = (f"{self.cpu_cores} CPU cores · {self.memory_gb:.0f}GB RAM"
+              if self.compute_type == "cpu"
+              else f"GPU · {self.memory_gb:.0f}GB")
+        return {
+            "name": self.name,
+            "hardware": hw,
+            "region": self.region,
+            "type": "CPU" if self.compute_type == "cpu" else "GPU",
+            "load": round(self.load, 3),
+            "active": self.active,
+            "jobs": self.jobs_served,
+            "tokens": self.tokens_served,
+            "earnings": self.estimated_earnings,
+            "uptime": time.time() - self.started_at,
+            "connected": self.connected,
+            # Owner-visible proof the limits are real: jobs this machine refused
+            # because it was already at its own concurrency cap.
+            "rejected": self.rejected_jobs,
+            "concurrency_cap": self.max_concurrency,
+            "trust": self.trust,
+        }
+
+    def stop(self) -> None:
+        """The dashboard kill switch: stop sharing and let the process exit."""
+        self._stop = True
+        self.connected = False
+
+    @property
+    def avg_latency_ms(self) -> float:
+        return sum(self.recent_ms) / len(self.recent_ms) if self.recent_ms else 250.0
+
+    @property
+    def measured_tps(self) -> float:
+        """FILE 16/09 — REAL measured throughput: tokens actually produced divided by the
+        time actually spent producing them. 0.0 until this node has served real work —
+        the control plane then refuses to score it rather than inventing a number, which
+        is exactly what the removed `simulateBenchmark` used to do."""
+        if not self.recent_ms or not self.recent_tokens:
+            return 0.0
+        total_ms = sum(self.recent_ms)
+        total_tokens = sum(self.recent_tokens)
+        if total_ms <= 0 or total_tokens <= 0:
+            return 0.0
+        return round(total_tokens / (total_ms / 1000.0), 2)
+
+    async def _send(self, ws, msg: dict) -> None:
+        async with self._send_lock:
+            await ws.send(json.dumps(msg))
+
+    def _pqc_keys(self):
+        """This node's EPHEMERAL hybrid keypair for the gateway->node hop (file 22-C).
+
+        Generated once per process and never written to disk: forward secrecy means a node that
+        is later compromised cannot decrypt traffic recorded before the compromise. A node
+        without a PQC runtime simply publishes no key and the gateway decides what to do about
+        that (file 22-B5 — fail closed once STEVAI_PQC_REQUIRE is armed).
+        """
+        if getattr(self, "_pqc", None) is None:
+            try:
+                from . import pqctransport
+                self._pqc = pqctransport.generate()
+            except Exception as e:
+                print(f"[{self.name}] no post-quantum transport available: {type(e).__name__}")
+                self._pqc = False
+        return self._pqc or None
+
+    def _open_job(self, job: dict) -> dict:
+        """Decrypt a sealed job payload at the node boundary (file 22-A3).
+
+        This IS the authorized decryption point: the node is where the model runs, so it is the
+        first and only place the prompt may exist in plaintext on this machine.
+        """
+        sealed = job.get("sealed")
+        if not sealed:
+            return job
+        keys = self._pqc_keys()
+        if keys is None:
+            raise RuntimeError("received a sealed job but this node has no PQC transport")
+        from . import pqctransport
+        import json as _json
+        opened = pqctransport.open_sealed(pqctransport.from_wire(sealed), keys)
+        payload = _json.loads(opened.decode())
+        # Metadata stays as the gateway sent it; content comes from the sealed half only.
+        return {**{k: v for k, v in job.items() if k != "sealed"}, **payload}
+
+    def _register_msg(self, rotate: bool = False) -> dict:
+        return {
+            # MAKE-BEFORE-BREAK (F-20.3). Set only on a socket opened to REPLACE this node's own
+            # live one. It is a request, not a right: the gateway does not act on it until the
+            # credential has proved this is the same node, because node ids are public.
+            "rotate": bool(rotate),
+            # File 22-C: the gateway seals this node's jobs to this key. Published as raw bytes
+            # in hex so it survives JSON; absent when the node has no PQC runtime.
+            "pqc_public": (self._pqc_keys().public().encode().hex()
+                           if self._pqc_keys() else ""),
+            # File 22-A1: this build opens sealed `embed` frames and seals the vectors back.
+            # The key above does not say that on its own — nodes have published keys since 22-C
+            # while unsealing ran on the `job` path only — so the gateway waits to be TOLD
+            # before it seals a document to a node that might not be able to read it.
+            "sealed_embed": True,
+            "type": "register",
+            "id": self.id,
+            "name": self.name,
+            "region": self.region,
+            "compute_type": self.compute_type,
+            "memory_gb": self.memory_gb,
+            # The directory tiers a node on vram_gb (GPU) and budgets CPU-RAM
+            # pooling on ram_gb, so report the right one for this hardware class.
+            "vram_gb": self.memory_gb if self.compute_type == "gpu" else 0.0,
+            "ram_gb": self.memory_gb if self.compute_type == "cpu" else 0.0,
+            "cpu_cores": self.cpu_cores,
+            "models": self.models,
+            "cost_per_1k": self.cost_per_1k,
+            "max_concurrency": self.max_concurrency,
+            "latency_hint_ms": self.avg_latency_ms,
+            # Measured, not simulated: 0.0 until this node has actually served work.
+            "measured_tps": self.measured_tps,
+            # Output modalities this node can actually generate. Ollama chat models
+            # produce text; image/audio/video generation lives on capable nodes
+            # (Tier A/S) that advertise those here.
+            "output_modalities": self.output_modalities,
+            # Software-integrity attestation: a digest of this agent's own code,
+            # checked by the gateway against the operator's published digests.
+            # Telemetry once a credential is present — the signature is the anchor.
+            "agent_digest": attest.agent_digest(),
+            # Signed admission credential, if the owner was issued one. Empty string
+            # = legacy node: the gateway then falls back to digest + approval list.
+            # Re-sent on every reconnect (see run()), so restarting the process is
+            # all it takes to pick up a refreshed credential before this one lapses.
+            "credential": self.credential,
+        }
+
+    # --- tasks ------------------------------------------------------------
+    async def _heartbeat_loop(self, ws) -> None:
+        while not self._stop:
+            await self._send(ws, {
+                "type": "heartbeat",
+                "load": round(self.load, 3),
+                "latency_ms": round(self.avg_latency_ms, 1),
+                "active": self.active,
+            })
+            await asyncio.sleep(CONFIG.heartbeat_seconds)
+        # Kill switch was pressed — drop the connection so the node stops serving.
+        try:
+            await ws.close()
+        except Exception:
+            pass
+
+    def _seal_reply(self, reply: dict, target, job_id) -> dict:
+        """Seal the content half of a reply back to the gateway that asked for it (22-A1).
+
+        THE ONE PLACE A CONTENT-BEARING ANSWER LEAVES THIS NODE. Every reply frame that carries
+        anything the user said or the model wrote goes through here, so the policy — which fields
+        move into the sealed half, what stays readable, and what happens when sealing fails —
+        lives in one function rather than once per call site. `tests/test_transport_seal_guard.py`
+        is built on that: it reads the node's syntax tree and fails on an answer frame that
+        reaches the socket without passing through this.
+
+        `target` says HOW, and there are exactly two forms:
+
+          * A HEX PUBLIC KEY — the gateway's ephemeral reply key, as `reply_to` carries it. One
+            fresh X25519 ephemeral and one fresh ML-KEM encapsulation for this frame. Right for a
+            one-shot answer: `/v1/embeddings` sends a single reply, and a handshake amortised
+            over one 4 MB vector batch costs nothing worth measuring.
+
+          * A `pqctransport.Session` — one handshake for the whole stream, AEAD per frame. Right
+            for a completion, which is delivered as hundreds of `chunk` frames and then a
+            `result`: measured, per-frame sealing cost +1703 B and ~0.8 ms PER CHUNK, so a
+            500-token answer paid 400 ms of CPU and 831 KiB of handshake to protect 2 KiB of
+            text. The owner ruled the session key for exactly that reason, and the guarantee is
+            unchanged — every one of those handshakes encapsulated to the SAME gateway key, and
+            forward secrecy comes from that key's lifetime rather than from how many times
+            anyone encapsulates to it.
+
+        RAISES on any failure, and the caller turns that into an error frame. There is
+        deliberately no "send it in the clear instead" branch: the gateway only asks for a
+        sealed reply on a hop it sealed, and answering that in plaintext would be a downgrade
+        anyone on the path could force by stripping a field.
+        """
+        from . import pqctransport
+        payload = {k: reply[k] for k in _SEALED_REPLY_FIELDS if k in reply}
+        md = {"job_id": job_id} if isinstance(job_id, (str, int)) else {}
+        clear = {k: v for k, v in reply.items() if k not in _SEALED_REPLY_FIELDS}
+        if isinstance(target, pqctransport.Session):
+            frame = target.seal_frame(json.dumps(payload).encode(), metadata=md)
+            # The handshake rides on the session's FIRST frame and on no other. That is where
+            # the saving is: frame 0 costs what a one-shot seal used to cost, and every frame
+            # after it costs an AEAD tag.
+            clear["sealed"] = pqctransport.session_wire_format(
+                frame, handshake=target.handshake if frame.seq == 0 else None)
+            return clear
+        pub = pqctransport.HybridPublic.decode(bytes.fromhex(target))
+        sealed = pqctransport.seal(json.dumps(payload).encode(), pub, metadata=md)
+        clear["sealed"] = pqctransport.wire_format(sealed)
+        return clear
+
+    def _begin_reply_session(self, job_id, reply_to: str):
+        """Open the per-stream session this job's answer comes home on, or RAISE.
+
+        The gateway's ephemeral public key arrives as `reply_to` INSIDE the sealed half of the
+        job — never beside it — and that is what makes it trustworthy: in the clear it could be
+        swapped on the path for an attacker's key, and this node would then seal the user's
+        answer to the attacker. AEAD-authenticated, it cannot be touched.
+
+        FAILS CLOSED. A node that cannot open a session for a job whose request asked for one
+        does not fall back to answering in the clear; it errors, and the gateway re-plans. The
+        clear answer is the thing phase two exists to stop sending.
+        """
+        from . import pqctransport
+        pub = pqctransport.HybridPublic.decode(bytes.fromhex(reply_to))
+        session, _handshake = pqctransport.begin_session(pub)
+        self._reply_sessions[job_id] = session
+        return session
+
+    async def _handle_embed(self, ws, msg: dict) -> None:
+        """Embed texts for the network's persistent RAG index (§5.2).
+
+        A separate frame type from `job` on purpose. An embedding is not a generation: it
+        runs a different model, produces vectors rather than tokens, and must NOT be
+        metered as a chat turn or a user would be billed twice for one document. It still
+        takes a concurrency slot, because it is still real work on the owner's machine.
+
+        A node whose engine has no embed endpoint (a llama.cpp chat node) answers with an
+        error, and the gateway re-plans onto one that can — the same shape as any other
+        capability the fleet is uneven about.
+        """
+        job_id = msg.get("id")
+        try:
+            was_sealed = bool(msg.get("sealed"))
+            # File 22-A3: the node boundary is the authorized decryption point, for an embed
+            # exactly as for a job — the documents in `texts` are content. `_open_job` RAISES
+            # when a sealed frame cannot be opened, and that is the whole point of calling it
+            # here: the old code read `texts` straight off the frame, so a sealed frame would
+            # have yielded no texts at all and been embedded as an empty batch. The gateway
+            # would have got a successful-looking reply for a document that was never indexed.
+            msg = self._open_job(msg)
+            texts = msg.get("texts") or msg.get("input") or []
+            if not isinstance(texts, list):
+                raise ValueError("embed frame needs a list of texts")
+            # Bound the batch: an unbounded list is an easy way to pin someone's CPU.
+            texts = [str(t or "") for t in texts][:512]
+            if not texts:
+                # AN EMPTY BATCH IS NEVER AN ANSWER. Whether the frame arrived empty or arrived
+                # in a shape this node could not read, embedding nothing and replying "fine"
+                # loses the document silently. Say so instead and let the gateway re-plan.
+                raise ValueError("embed frame carried no texts" +
+                                 (" after opening its sealed half" if was_sealed else ""))
+            vectors = await inference.embed(texts)
+            if len(vectors) != len(texts):
+                # The engine answered, but not for every chunk. The caller indexes vectors
+                # positionally, so a short list is a document indexed against the wrong text —
+                # and a silently partial index is the same defect as an empty one.
+                raise ValueError(f"engine returned {len(vectors)} vectors for {len(texts)} texts")
+            reply = {"type": "embeddings", "id": job_id,
+                     "embeddings": vectors, "model": inference.EMBED_MODEL}
+            # The gateway put its own public key in the sealed half when it wants the answer
+            # sealed too. Absent = a gateway or a hop that could not seal this request either,
+            # so the clear reply is the one it is already expecting.
+            reply_to = msg.get("reply_to") or ""
+            if reply_to:
+                reply = self._seal_reply(reply, str(reply_to), job_id)
+            await self._send(ws, reply)
+        except BaseException as e:
+            # THE DETAIL STAYS ON THIS MACHINE. `{e}` used to go on the wire, and an
+            # exception's text routinely quotes the input that produced it — a JSON error
+            # quotes the offending fragment, a tokeniser error quotes the text it choked on.
+            # That is the model-name leak wearing a different coat, and `_seal_reply` leaves
+            # `message` in the CLEAR by construction, so it is plaintext on the wire. The
+            # class name says what kind of failure it was; the owner's own log says the rest.
+            print(f"[{self.name}] embed job {job_id} failed: {type(e).__name__}: {e}")
+            await self._send(ws, {"type": "error", "id": job_id,
+                                  "message": f"embed failed: {type(e).__name__}"})
+        finally:
+            self._inflight.discard(job_id)
+            self._release_slot()
+
+    def _can_serve_model(self, model: str) -> bool:
+        """Do we actually run this model? Compared across catalog-id and engine-tag spellings,
+        the same equivalence routing uses — a node advertising "qwen3:4b" must recognise a job
+        asking for "qwen3-4b", or it silently never races for work it can do."""
+        if not model or model == "stevai-default":
+            return True
+        mine = {str(m).strip().lower() for m in (self.models or []) if str(m).strip()}
+        if not mine:
+            return True                     # advertises nothing = serves whatever it has
+        want = model.strip().lower()
+        forms = {want, want.replace("-", ":"), want.replace(":", "-")}
+        return bool(forms & mine) or any(m.split(":")[0] == want.split("-")[0] for m in mine)
+
+    async def _maybe_claim(self, ws, offer: dict) -> None:
+        """Race for a posted job — or decline by staying silent.
+
+        A busy node simply does not claim. That is the self-deflection the design asks for, and
+        it is more truthful than any load number we could publish: we know our own free slots
+        at this instant.
+
+        When we cannot run the model but CAN still take work, we claim as a BACKSTOP: someone
+        has to catch the job so it never falls on the floor, and the gateway will bring the
+        right model up and hand over. A backstop claim never outbids a node that actually runs
+        the model — the gateway enforces that ordering.
+        """
+        job_id = offer.get("id")
+        if not isinstance(job_id, (str, int)) or job_id == "":
+            return
+        # `paused` is a DESKTOP-node concept; the Python node has no such attribute. Reading it
+        # directly raised AttributeError here, the async task swallowed it, and the node
+        # silently never claimed — every live race found nobody while every unit test passed,
+        # because the tests set the attribute by hand on a stub.
+        if getattr(self, "paused", False) or self.active >= self.max_concurrency:
+            return                          # busy or paused → self-deflect, silently
+        model = str(offer.get("model") or "")
+        serves = self._can_serve_model(model)
+        if not serves and not bool(offer.get("allow_backstop")):
+            return
+        await self._send(ws, {"type": "claim", "id": job_id, "backstop": not serves,
+                              "price_quote": self._price_quote()})
+
+    #: Rough wattage by VRAM class, mirroring the settlement tiers in `economics`. A node that
+    #: has been benchmarked can publish its real draw instead (STEVAI_NODE_POWER_W).
+    _TIER_WATTS = ((16, 60.0), (48, 200.0), (80, 350.0))
+
+    def _price_quote(self) -> float:
+        """What this node asks to serve one hour of work, in EUR (file 02-B3).
+
+        The gateway keeps the K CHEAPEST bids and then draws uniformly among them, so this
+        number decides whether the node is in the running — which is exactly why the node
+        computes it from its OWN power draw and local tariff rather than the gateway guessing.
+        An operator on cheap night-rate power should be able to underbid one on expensive
+        daytime power, and only the node knows that.
+
+        Never raises: a node that cannot price itself still races, and is simply ranked as
+        unpriced (last) rather than dropped from the fleet.
+        """
+        try:
+            explicit = envvar("NODE_PRICE_EUR_PER_HR", "").strip()
+            if explicit:
+                return max(0.0, float(explicit))
+
+            watts = float(envvar("NODE_POWER_W", "") or 0.0)
+            if watts <= 0:
+                vram = float(getattr(self, "memory_gb", 0) or 0)
+                watts = 700.0
+                for limit, w in self._TIER_WATTS:
+                    if vram < limit:
+                        watts = w
+                        break
+
+            from datetime import datetime, timezone
+            from .economics import rates_for
+            hourly = rates_for(self.region).get("per_kwh_by_hour") or [0.20] * 24
+            per_kwh = float(hourly[datetime.now(tz=timezone.utc).hour % len(hourly)])
+            return round((watts / 1000.0) * per_kwh, 6)
+        except Exception:
+            return 0.0
+
+    async def _handle_job(self, ws, job: dict) -> None:
+        """Run one job under this node's own wall clock, and report — never leave it running.
+
+        The timeout is the node's half of the amplification fix: the gateway already
+        gives up at CONFIG.job_timeout_s and re-plans onto ANOTHER node, so a job that
+        outlives that is pure burn on the volunteer's machine. Cancelling the coroutine
+        closes the engine's HTTP stream, which is what actually stops generation.
+        """
+        job_id = job.get("id")
+        try:
+            # File 22-A3: the node boundary is the authorized decryption point. Everything
+            # before this line has only ever held ciphertext plus routing metadata.
+            job = self._open_job(job)
+        except Exception as e:
+            print(f"[{self.name}] could not open sealed job {job_id}: {type(e).__name__}")
+            await self._send(ws, {"type": "error", "id": job_id,
+                                  "error": "sealed payload could not be opened"})
+            # THE SLOT WAS ALREADY TAKEN, synchronously, by the receive loop — and this branch
+            # used to `return` straight past the release. A node that could not open four sealed
+            # jobs sat pinned at load 1.0, still heartbeating as alive, exactly as the
+            # `_handle_shard_job` docstring describes for its own version of this bug. An error
+            # reply is not an exit from the slot's lifetime.
+            self._inflight.discard(job_id)
+            self._release_slot()
+            return
+        try:
+            # PHASE TWO of the result-seal cutover. `reply_to` came out of the SEALED half just
+            # above, so it is the gateway's own key and not one a relay substituted. Its
+            # presence is the request: seal the answer home. Its absence means the gateway could
+            # not seal this hop either (a keyless node, a build that cannot), and the clear
+            # answer is the one it is already expecting — `ACCEPT_UNSEALED_RESULT` is still True
+            # for exactly the length of this rollout.
+            #
+            # Opened BEFORE the job runs, so a node that cannot seal the answer burns none of
+            # the owner's compute discovering it at the end.
+            reply_to = job.get("reply_to") or ""
+            if reply_to:
+                self._begin_reply_session(job_id, str(reply_to))
+        except Exception as e:
+            print(f"[{self.name}] could not open a reply session for {job_id}: "
+                  f"{type(e).__name__}")
+            await self._send(ws, {"type": "error", "id": job_id,
+                                  "message": (f"this node cannot seal its answer back to the "
+                                              f"gateway ({type(e).__name__}) and will not send "
+                                              f"it in the clear")})
+            self._inflight.discard(job_id)
+            self._release_slot()
+            return
+        try:
+            await asyncio.wait_for(self._run_job(ws, job), timeout=self.job_timeout_s)
+        except asyncio.TimeoutError:
+            await self._send(ws, {
+                "type": "error", "id": job_id,
+                "message": (f"job exceeded this node's {self.job_timeout_s:.0f}s limit "
+                            f"and was cancelled"),
+            })
+        except Exception as e:  # never let one bad job kill the node
+            # Same reason as `_handle_embed`: `{e}` put request-derived text on the wire in
+            # the clear. It was not hypothetical — `engine_guard.resolve_model` raises
+            # "model {requested!r} is not served by this node" and `_run_job` raises "this
+            # node does not generate {modality!r}", so the caller's own model string and
+            # modality came straight back out through this frame.
+            print(f"[{self.name}] job {job_id} failed: {type(e).__name__}: {e}")
+            await self._send(ws, {
+                "type": "error",
+                "id": job_id,
+                "message": f"job failed: {type(e).__name__}",
+            })
+        finally:
+            self._cancel_jobs.discard(job_id)
+            self._inflight.discard(job_id)
+            self._reply_sessions.pop(job_id, None)
+            self._release_slot()
+
+    async def _run_job(self, ws, job: dict) -> None:
+        """The actual work. Raises on any failure — `_handle_job` owns reporting,
+        the clock and the concurrency slot.
+
+        PHASE TWO: every answer frame below goes through `_seal_reply` when the gateway asked
+        for a sealed answer, and `reply_session` is what that request left behind. One session
+        for this whole job, so a stream pays one handshake rather than one per token delta.
+        `None` means the gateway did not ask — a hop neither end could seal — and the frame goes
+        home exactly as it did before, which is what keeps this a phase and not a flag day.
+        """
+        job_id = job["id"]
+        reply_session = self._reply_sessions.get(job_id)
+        start = time.time()
+        modality = (job.get("profile") or {}).get("modality") or job.get("modality") or "text"
+        # THE JOB DOES NOT GET TO PICK WHAT THIS MACHINE DOES. `modality` is
+        # attacker-controlled data off the wire, and the gateway only checks that
+        # *some* node in the region is capable — it does not filter the node it picked.
+        # Without this, a text-only laptop could be told to load SDXL / a video
+        # diffusion model and spend minutes of GPU it never advertised.
+        if modality != "text" and modality not in self.output_modalities:
+            raise engine_guard.EngineGuardError(
+                f"this node does not generate {modality!r} "
+                f"(serves: {', '.join(self.output_modalities)})")
+        if modality in ("image", "audio", "video"):
+            import base64
+            prompt = ""
+            for m in reversed(job.get("messages", [])):
+                if m.get("role") == "user":
+                    c = m.get("content", "")
+                    prompt = c if isinstance(c, str) else " ".join(p.get("text", "") for p in c if isinstance(p, dict))
+                    break
+            # Media generation is SYNCHRONOUS, CPU/GPU-bound torch work that runs for
+            # seconds to minutes. Called directly it froze the whole event loop: the
+            # heartbeat stopped (so the gateway marked this node stale mid-job), every
+            # other concurrent job on the machine stalled, and — because a blocked loop
+            # can never run the timeout callback — `_handle_job`'s wall-clock ceiling
+            # could not fire. Off the loop it goes.
+            # Honest scope: the ceiling can now *report* and free the slot, but it
+            # cannot kill the thread; truly stopping torch mid-generation needs a
+            # child process (see engine_guard's "what is NOT contained").
+            if modality == "image":
+                from . import imagegen
+                # FILE 03-E2: the paid "enhance" conditions on the sketch the user's own device
+                # already made, rather than re-rolling from the prompt. `init` is that sketch;
+                # its absence is an ordinary text-to-image generation.
+                _init_b64 = job.get("init") or ""
+                _init = base64.b64decode(_init_b64) if _init_b64 else b""
+                _strength = float(job.get("strength") or 0.4)
+                # Name the model that ACTUALLY ran. This said "sdxl" unconditionally, which
+                # stopped being true the moment model choice moved to the registry resolver —
+                # and this string is what the artifact reply and the metering path report.
+                _img_model = imagegen.resolved_model().model_id
+                data, mime, backend = (
+                    await asyncio.to_thread(imagegen.generate_image, prompt,
+                                            init=_init, strength=_strength),
+                    "image/png", f"{_img_model}-img2img" if _init else _img_model)
+            elif modality == "audio":
+                from . import audiogen
+                # A consented voice sample rides with the job and is used for
+                # this utterance only; audiogen deletes it immediately after.
+                # It is the one place attacker-controlled BYTES reach a decoder on
+                # this machine, so it is size-capped here as well as scanned at the
+                # gateway — a node does not assume its gateway did the right thing.
+                sample_b64 = job.get("voice_sample") or ""
+                sample = engine_guard.check_voice_sample(
+                    base64.b64decode(sample_b64) if sample_b64 else None)
+                data, mime, backend = (
+                    await asyncio.to_thread(audiogen.generate_audio, prompt,
+                                            voice_sample=sample,
+                                            language=job.get("language", "")),
+                    "audio/wav",
+                    "voice-clone" if sample else "audio",
+                )
+            else:
+                from . import videogen
+                data, mime, backend = (
+                    await asyncio.to_thread(videogen.generate_video, prompt),
+                    "video/mp4", "video")
+            took_ms = (time.time() - start) * 1000.0
+            self.recent_ms.append(took_ms)
+            # A generated image/audio/video is the user's content in full, base64 on the frame —
+            # it is sealed like a completion, not exempted for being bytes rather than text.
+            artifact_reply = {
+                "type": "result", "id": job_id, "content": "",
+                "artifact": {"modality": modality, "mime": mime, "data_b64": base64.b64encode(data).decode()},
+                "tool_calls": [], "tokens": 0, "prompt_tokens": 0,
+                "backend": backend, "compute_ms": round(took_ms, 1),
+            }
+            if reply_session is not None:
+                artifact_reply = self._seal_reply(artifact_reply, reply_session, job_id)
+            await self._send(ws, artifact_reply)
+            return
+        if job.get("stream"):
+            # Streaming text (§5A/§5B foundation): forward the model's REASONING and
+            # CONTENT deltas as `chunk` frames as they arrive, then the aggregated
+            # `result` frame (same shape as non-stream, so the future-based path still
+            # resolves for metering/history). Dormant unless the gateway sets stream.
+            agg: dict = {}
+            partial: list = []
+            async for ev in inference.stream_turn(   # dispatches: llama.cpp (Kimi) or Ollama
+                job.get("model", "stevai-default"), job.get("messages", []),
+                job.get("tools"), job.get("tool_choice"), think=job.get("think"),
+                think_effort=job.get("think_effort") or "",
+            ):
+                if job_id in self._cancel_jobs:
+                    # Client bailed/steered → stop compute + billing at the partial.
+                    # Returning closes stream_ollama's httpx stream, halting Ollama.
+                    self._cancel_jobs.discard(job_id)
+                    self._record_job(inference.estimate_tokens("".join(partial)))
+                    return
+                if ev["type"] in ("reasoning", "content"):
+                    if ev["type"] == "content":
+                        partial.append(ev["text"])
+                    # THE FRAME THE WHOLE COST QUESTION TURNED ON. One of these per token
+                    # delta, `text` in the clear until now, and a completion spends most of
+                    # its life here. Sealed under the per-stream session key: an AEAD tag and
+                    # a sequence number per chunk, and the handshake only on the first.
+                    chunk_reply = {"type": "chunk", "id": job_id,
+                                   "channel": ev["type"], "text": ev["text"]}
+                    if reply_session is not None:
+                        chunk_reply = self._seal_reply(chunk_reply, reply_session, job_id)
+                    await self._send(ws, chunk_reply)
+                elif ev["type"] == "done":
+                    agg = ev
+            took_ms = (time.time() - start) * 1000.0
+            self.recent_ms.append(took_ms)
+            self._record_job(int(agg.get("tokens", 0)))
+            # The aggregate the gateway meters and stores history from. It repeats the whole
+            # answer, so sealing the deltas and leaving this in the clear would protect nothing.
+            stream_reply = {
+                "type": "result", "id": job_id,
+                "content": agg.get("content", ""),
+                "reasoning": agg.get("reasoning", ""),
+                "tool_calls": agg.get("tool_calls", []),
+                "tokens": int(agg.get("tokens", 0)),
+                "prompt_tokens": int(agg.get("prompt_tokens", 0)),
+                "backend": agg.get("backend", ""),
+                "compute_ms": round(took_ms, 1),
+            }
+            if reply_session is not None:
+                stream_reply = self._seal_reply(stream_reply, reply_session, job_id)
+            await self._send(ws, stream_reply)
+            return
+        result = await inference.run_inference(
+            model=job.get("model", "stevai-default"),
+            messages=job.get("messages", []),
+            region=self.region,
+            load=self.load,
+            compute_type=self.compute_type,
+            tools=job.get("tools"),
+            tool_choice=job.get("tool_choice"),
+            # THE NON-STREAM PATH DROPPED THIS, and `run_inference` has accepted it all along.
+            #
+            # `think` is the gateway's per-turn decision about whether a question is worth
+            # reasoning over. The streaming handler forwards it (see `stream_turn` above); this
+            # one did not, so on the JSON path the flag was computed, sealed into the job, carried
+            # across the wire, and then dropped one line before the engine.
+            #
+            # MEASURED 2026-09-29 against a live node. Ollama honours it: `think:false` returned
+            # 3 completion tokens and an empty thinking channel, while the same prompt without it
+            # returned 134 tokens and 588 characters of reasoning. Through the gateway, a
+            # non-streaming turn with `quality:"speed"` — which forces the hint to False on both
+            # the old and the new code — still spent ~172 hidden tokens. That gap was this
+            # argument.
+            think=job.get("think"),
+            think_effort=job.get("think_effort") or "",
+        )
+        took_ms = (time.time() - start) * 1000.0
+        self.recent_ms.append(took_ms)
+        self._record_job(result["tokens"])
+        reply = {
+            "type": "result",
+            "id": job_id,
+            "content": result["content"],
+            # THE THINKING CHANNEL. The streaming handler above has always sent this; this frame — the
+            # one a NON-STREAMING job gets — omitted it, so the gateway received a result with no
+            # `reasoning` key at all (measured: keys were content/tool_calls/tokens/prompt_tokens/
+            # backend/compute_ms/id/type). `reasoning` is in `_SEALED_REPLY_FIELDS`, so it is sealed
+            # with the answer rather than travelling beside it in the clear.
+            "reasoning": result.get("reasoning", ""),
+            "tool_calls": result.get("tool_calls", []),  # agentic mode passthrough
+            "tokens": result["tokens"],
+            "prompt_tokens": result["prompt_tokens"],
+            "backend": result["backend"],
+            "compute_ms": round(took_ms, 1),
+        }
+        if reply_session is not None:
+            reply = self._seal_reply(reply, reply_session, job_id)
+        await self._send(ws, reply)
+
+    async def _handle_shard_job(self, ws, job: dict) -> None:
+        """One member of a CPU pool: hold and run this node's slice of the model.
+
+        The pool exists when no single node has enough memory for the whole
+        model. Each member owns a shard (a range of layers/experts); the lead
+        (shard 0) returns the assembled text, the others return their timing so
+        the gateway can report the collaboration honestly.
+        """
+        # EVERY exit path from here MUST hit the `finally` that releases the
+        # concurrency slot — the slot was already taken, synchronously, by the receive
+        # loop. Reading `job["id"]` / `int(shard_index)` *above* the try meant a
+        # `shard_job` frame with no id (or `shard_index: "NaN"`) raised before the try
+        # was entered: the slot leaked and no error frame was sent. Four such frames
+        # pinned a node at load 1.0, still heartbeating as alive, until the process was
+        # restarted. So: tolerant reads, and the shape checks live inside the try.
+        job_id = job.get("id")
+        shard_index = 0
+        start = time.time()
+        try:
+            # THE POOLED PATH OPENS ITS SEALED HALF TOO. The gateway began sealing `shard_job`
+            # on 2026-09-24 (the `_execute_pool` bypass fix), but this handler read `messages`
+            # RAW — so a sealed shard arrived with its content in the `sealed` blob, `messages`
+            # defaulted to `[]`, and every pool member ran inference on an EMPTY PROMPT and
+            # returned a confident answer to nothing. Sealing one end and not the other is the
+            # same defect the embed frame had: silently empty input wearing the costume of
+            # protection. `_open_job` RAISES on a sealed frame it cannot open, and the `except`
+            # below turns that into an error frame — which is the honest outcome, not `[]`.
+            job = self._open_job(job)
+            shard_index = int(job.get("shard_index", 0) or 0)
+            shard_count = int(job.get("shard_count", 1) or 1)
+            # THE POOLED REPLY, SEALED — the other half of the fix above, and the last frame in
+            # this file that carried a completion in the clear. `reply_to` comes out of the sealed
+            # half just opened, so it is the gateway's own key and not one a relay substituted.
+            #
+            # Opened BEFORE inference, exactly as `_handle_job` does it: a node that cannot seal
+            # its answer must discover that before it spends the owner's compute, not after. The
+            # `except` below turns the raise into an error frame, which is the honest outcome — a
+            # node that cannot seal does NOT fall back to answering in the clear, because the clear
+            # answer is the thing this phase exists to stop sending.
+            #
+            # Non-lead shards return timing only, but they return it on the SAME frame shape, so
+            # they go through the same policy. A shape that sometimes carries a completion is
+            # treated as one that always does; deciding per-frame is how the single path's leak
+            # survived as long as it did.
+            reply_to = job.get("reply_to") or ""
+            reply_session = (self._begin_reply_session(job_id, str(reply_to))
+                             if reply_to else None)
+            result = await asyncio.wait_for(inference.run_inference(
+                model=job.get("model", "stevai-default"),
+                messages=job.get("messages", []),
+                region=self.region,
+                load=self.load,
+                compute_type=self.compute_type,
+                shard=(shard_index, shard_count),
+                tools=job.get("tools"),
+                tool_choice=job.get("tool_choice"),
+                # The SHARDED path dropped it too. A shard is still a turn, and a turn the
+                # gateway decided needs no reasoning does not acquire the need by being split
+                # across nodes — it would just pay for the thinking once per shard.
+                think=job.get("think"),
+                think_effort=job.get("think_effort") or "",
+            ), timeout=self.job_timeout_s)
+            took_ms = (time.time() - start) * 1000.0
+            self.recent_ms.append(took_ms)
+            if shard_index == 0:
+                self._record_job(result["tokens"])
+            shard_reply = {
+                "type": "result",
+                "id": job_id,
+                "shard_index": shard_index,
+                # Only the lead returns the assembled completion; others just timing.
+                "content": result["content"] if shard_index == 0 else "",
+                # Lead-only for the same reason `content` is: a pooled turn has one answer and one
+                # line of thinking behind it, and a follower's slice is not either of them.
+                "reasoning": result.get("reasoning", "") if shard_index == 0 else "",
+                "tokens": result["tokens"] if shard_index == 0 else 0,
+                "prompt_tokens": result["prompt_tokens"] if shard_index == 0 else 0,
+                "backend": result["backend"],
+                "compute_ms": round(took_ms, 1),
+                "memory_gb": self.memory_gb,
+            }
+            # `content` is in `_SEALED_REPLY_FIELDS`; `shard_index`, `tokens` and the timings stay
+            # readable, because the gateway needs them to assemble the pool's answer and to meter
+            # it, and none of them reveals what the user said (22-A2).
+            if reply_session is not None:
+                shard_reply = self._seal_reply(shard_reply, reply_session, job_id)
+            await self._send(ws, shard_reply)
+        except asyncio.TimeoutError:
+            await self._send(ws, {
+                "type": "error", "id": job_id, "shard_index": shard_index,
+                "message": (f"shard job exceeded this node's {self.job_timeout_s:.0f}s "
+                            f"limit and was cancelled"),
+            })
+        except Exception as e:
+            # The pooled path's copy of the same leak, and it had to be fixed in the same
+            # commit: a defect closed on one path and left open on the other just moves the
+            # leak to whichever path the caller can reach.
+            print(f"[{self.name}] shard {shard_index} of job {job_id} failed: "
+                  f"{type(e).__name__}: {e}")
+            await self._send(ws, {
+                "type": "error", "id": job_id, "shard_index": shard_index,
+                "message": f"shard job failed: {type(e).__name__}",
+            })
+        finally:
+            self._cancel_jobs.discard(job_id)
+            self._inflight.discard(job_id)
+            # The session dies with the job it was opened for. Left behind, it would hold a key
+            # for a request that is over and let the next job to reuse this id look like one the
+            # gateway had asked to have sealed.
+            self._reply_sessions.pop(job_id, None)
+            self._release_slot()
+
+    async def _serve_once(self, url: str = "", rotate: bool = False) -> None:
+        target = url or self.gateway_url
+        async with websockets.connect(target, max_size=8 * 1024 * 1024) as ws:
+            await self._send(ws, self._register_msg(rotate))
+            self.connected = True
+            print(f"[{self.name}] connected to gateway {url or self.gateway_url} "
+                  f"(region={self.region}, models={self.models or 'any'})")
+            # Connected is NOT serving: registration can still be refused (duplicate id, revoked
+            # credential). Only the register reply below may claim the link is up.
+            _publish_link_state("connected")
+            hb = asyncio.create_task(self._heartbeat_loop(ws))
+            # Open the replacement BEFORE this socket's deadline, so there is never a moment
+            # when this node has no socket at all. The gateway then drains and closes this one.
+            rot = asyncio.create_task(self._rotate_when_due(target))
+            self._live_sockets[target] = self._live_sockets.get(target, 0) + 1
+            try:
+                async for raw in ws:
+                    # ONE BAD FRAME IS NOT A REASON TO DROP THE CONNECTION. `json.loads`
+                    # and `msg.get` used to sit bare inside this loop, so `'{"type":"job"'`
+                    # (JSONDecodeError), `'[]'`/`'"x"'`/`'null'` (AttributeError) or a
+                    # deeply nested array (RecursionError) tore the socket down and sent
+                    # the node round the reconnect backoff. Skip the frame instead; a
+                    # frame we cannot parse has no id to answer to anyway.
+                    try:
+                        msg = json.loads(raw)
+                        if not isinstance(msg, dict):
+                            continue
+                        if "id" in msg:
+                            msg["id"] = _frame_id(msg)
+                        mtype = msg.get("type")
+                        if mtype in ("job", "shard_job", "embed"):
+                            # A job we cannot name is a job we cannot answer: the result
+                            # frame would carry a null id the gateway can never match,
+                            # and the work would be pure burn on the owner's machine.
+                            # Refused BEFORE a slot is taken, and reported.
+                            if not isinstance(msg.get("id"), (str, int)) or msg["id"] == "":
+                                await self._send(ws, {
+                                    "type": "error", "id": msg.get("id"),
+                                    "message": "job frame has no usable id",
+                                })
+                                continue
+                            # CONCURRENCY CAP. Taken here, synchronously, before the task
+                            # exists: a node advertises max_concurrency and must honour it,
+                            # or one caller can start unlimited generations on the owner's
+                            # machine. Over-cap is refused immediately so the gateway
+                            # re-plans onto a node with room, rather than queueing.
+                            if not self._acquire_slot():
+                                self.rejected_jobs += 1
+                                await self._send(ws, {
+                                    "type": "error", "id": msg.get("id"),
+                                    "message": (f"node at capacity "
+                                                f"({self.max_concurrency} concurrent jobs)"),
+                                })
+                                continue
+                            try:
+                                self._inflight.add(msg["id"])
+                                handler = (self._handle_job if mtype == "job"
+                                           else self._handle_embed if mtype == "embed"
+                                           else self._handle_shard_job)
+                                asyncio.create_task(handler(ws, msg))
+                            except BaseException:
+                                # Nothing between the acquire and the task may be able
+                                # to lose the slot — that is REAL 1's whole shape.
+                                self._inflight.discard(msg["id"])
+                                self._release_slot()
+                                raise
+                        elif mtype == "job_offer":
+                            # CAPTURE THE FLAG. The gateway posted a job; decide for OURSELVES
+                            # whether to race for it. Answering from our own live state is the
+                            # point of the design: the gateway's view of our load is always one
+                            # heartbeat stale, ours never is.
+                            #
+                            # Failures are LOGGED, never swallowed. A node that stops racing is
+                            # indistinguishable from a node that is busy, so a silent exception
+                            # here removes the machine from the network invisibly — which is
+                            # exactly what happened on 2026-08-21.
+                            try:
+                                await self._maybe_claim(ws, msg)
+                            except Exception as e:
+                                print(f"[node] could not race for {msg.get('id')}: "
+                                      f"{type(e).__name__}: {e}")
+
+                        elif mtype == "cancel":
+                            # §5A: the gateway's client bailed/steered — stop the stream job.
+                            # Only for a job actually running here: an id we have never
+                            # seen is either noise or an attempt to pre-cancel a FUTURE
+                            # job, and remembering every one of them was an unbounded set.
+                            jid = msg.get("id")
+                            if jid in self._inflight:
+                                self._cancel_jobs.add(jid)
+                        elif mtype == "registered":
+                            status = msg.get("status", "active")
+                            # Placement trust as the OPERATOR labelled us. Only ever lowers
+                            # or raises what this node will do locally (tool execution); it
+                            # is not something the node can claim for itself.
+                            self.trust = str(msg.get("trust") or "untrusted")
+                            note = (" — awaiting operator approval before it will "
+                                    "receive traffic") if status == "pending" else ""
+                            print(f"[{self.name}] registered as id={msg.get('id')} "
+                                  f"[{status}, trust={self.trust}]{note}")
+                            # THE join signal. A node `pending` operator approval is admitted but
+                            # gets no traffic, so it is not "serving" and must not read as such.
+                            _publish_link_state(
+                                "serving" if status == "active" else f"registered-{status}")
+                            # FILE 08-B2 — ANSWER THE ATTESTATION CHALLENGE, if this machine has
+                            # evidence to answer it with. An ordinary node has none and says nothing,
+                            # which is the correct behaviour: it simply never joins the TEE lane.
+                            await self._answer_tee_challenge(ws, str(msg.get("tee_challenge") or ""))
+                        elif mtype == "attested":
+                            # The gateway's verdict. Printed rather than swallowed because an operator
+                            # who has just brought up a confidential host has no other way to learn
+                            # whether their GPU is out of CC mode or the anchor is unpinned.
+                            if msg.get("lane"):
+                                print(f"[{self.name}] attested — serving the confidential (TEE) lane")
+                            elif msg.get("error"):
+                                print(f"[{self.name}] attestation was NOT accepted: {msg['error']}")
+                    except Exception as e:
+                        print(f"[{self.name}] ignoring unusable frame "
+                              f"({type(e).__name__}: {e})")
+                        continue
+            finally:
+                hb.cancel()
+                rot.cancel()
+                self._live_sockets[target] = max(0, self._live_sockets.get(target, 1) - 1)
+                self.connected = False
+
+    async def _rotate_when_due(self, url: str) -> None:
+        """Open a replacement socket before this one's deadline (F-20.3).
+
+        The replacement runs the ordinary connect loop with `rotate` set on its first register, so
+        it is subject to every admission check a fresh socket is. Once the gateway installs it,
+        this node's old socket is drained and closed with 4005, and `_run_one` recognises that as
+        "your replacement is live" rather than as a failure to retry.
+        """
+        try:
+            await asyncio.sleep(SOCKET_MAX_AGE_S)
+        except asyncio.CancelledError:
+            return
+        if self._stop:
+            return
+        print(f"[{self.name}] socket reached {SOCKET_MAX_AGE_S:.0f}s — opening its replacement "
+              f"before this one is closed")
+        task = asyncio.ensure_future(self._run_one(url, first_rotate=True))
+        self._rotations.add(task)
+        task.add_done_callback(self._rotations.discard)
+
+    def _start_gui(self) -> None:
+        if not self.gui_port:
+            return
+        try:
+            from . import gui
+            url = gui.start_gui(self.stats, self.stop, self.gui_port)
+            print(f"[{self.name}] dashboard: open {url} in your browser")
+        except Exception as e:
+            print(f"[{self.name}] dashboard could not start: {type(e).__name__}: {e}")
+
+    def _refresh_credential(self) -> bool:
+        """Re-read the credential from its source before dialling. True if it changed
+        (i.e. the owner dropped in a fresh one), so we can say so and retry at once."""
+        current = _load_credential(self._cred_args)
+        if current and current != self.credential:
+            self.credential = current
+            return True
+        return False
+
+    def gateway_urls(self) -> list:
+        """Which gateways this node keeps a socket open to (file 02-A3).
+
+        Two by default. One socket means one gateway restart takes the node off the network until
+        it reconnects; with two the restart is a routine event rather than an incident.
+
+        WHERE THE LIST COMES FROM (02-A3 decision table)
+        "Directory returns the region's gateway list sorted by RTT probe; node takes first two;
+        re-probe every 10 min." So the directory is asked first and the node measures the RTT
+        itself. `STEVAI_GATEWAY_URLS` remains an explicit override, and the single
+        `--gateway-url` is the last resort — a node must still come up if the directory is down.
+
+        REGION-SCOPED, ALWAYS. A node that dials another region's gateway advertises itself as
+        capacity in that region, and the directory keeps ONE gateway per node (last heartbeat
+        wins) — so the node flaps between regions and `/discover` intermittently finds nothing.
+        Measured live 2026-08-28 with two nodes wrongly armed across eu-west and us-east: 7 of
+        12 turns failed with "no network gateway with an active provider".
+        """
+        want = max(1, int(envvar("NODE_GATEWAYS", "2") or 2))
+
+        regional = self._directory_gateways()          # region-scoped by construction
+
+        override = envvar("GATEWAY_URLS", "").strip()
+        urls = [u.strip() for u in override.split(",") if u.strip()]
+        if urls and self._foreign_gateways():
+            # THE OVERRIDE IS NOT ALLOWED OUT OF ITS REGION. The docstring above has always said
+            # "REGION-SCOPED, ALWAYS" and this path did not enforce it: `STEVAI_GATEWAY_URLS` in
+            # the RunPod specs lists ALL THREE regions, so an eu-west node sorted them by RTT and
+            # dialled us-east.
+            #
+            # That is not a latency curiosity. The directory keeps ONE gateway per node, so the
+            # node registered under `gw-us-east-a` while declaring `region=eu-west`, and
+            # `/discover` for eu-west — which requires a node whose gateway belongs to an EU
+            # gateway — correctly found nothing. Measured 2026-09-28: TWENTY consecutive turns,
+            # zero landed, every one refused at `/discover`. It is also plainly wrong for data
+            # residency (plan 29): EU traffic served through a US gateway.
+            # DROP ONLY WHAT IS KNOWN TO BE SOMEONE ELSE'S REGION. "Not in my region's list" is
+            # not the same as "in another region": an operator pinning a brand-new or private
+            # gateway would be overruled by a control plane that has simply not heard of it yet,
+            # and an explicit pin must not be silently discarded. So the test is positive
+            # evidence of foreignness, taken from the directory's own region labels.
+            foreign = self._foreign_gateways()
+            kept = [u for u in urls if u not in foreign]
+            dropped = [u for u in urls if u in foreign]
+            if dropped:
+                print(f"[{self.name}] ignoring {len(dropped)} configured gateway(s) outside "
+                      f"region {self.region!r}: {', '.join(dropped)}")
+            urls = kept
+            if not urls:
+                print(f"[{self.name}] every configured gateway belongs to another region — "
+                      f"using region {self.region!r}'s list from the directory instead")
+                urls = list(regional)
+        elif not urls:
+            urls = regional
+        if not urls:
+            # Last resort: the directory is unreachable AND nothing was configured for this
+            # region. `--gateway-url` is region-specific by deployment, so it is the safe default.
+            urls = [self.gateway_url]
+
+        # Distinct gateways first, in RTT order — the nearest is primary, the next is standby.
+        seen, distinct = set(), []
+        for u in urls:
+            if u and u not in seen:
+                seen.add(u)
+                distinct.append(u)
+        if len(distinct) > 1:
+            # IN-REGION FIRST, THEN NEAREST. Ordering, not filtering: nothing is discarded, so an
+            # operator's explicit pin is still used and an unknown gateway is still kept — the two
+            # properties `test_dual_gateway_sockets` pins, and which an earlier attempt of mine
+            # broke by preferring the regional list outright.
+            #
+            # Pure RTT order is wrong for the thing that actually matters. Measured 2026-09-29: a
+            # node whose spec lists all three regions was placed far from Europe, sorted by RTT,
+            # and took asia-southeast1 + us-east1 — while DECLARING `region=eu-west`. Its capacity
+            # is eu-west, so the gateway correctly refused every turn ("No machine in 'us-east'
+            # can run qwen3-8b right now. It is available in eu-west"), `/discover` had nothing to
+            # hand out, and the node served nothing while looking perfectly healthy.
+            #
+            # `regional` is the directory's list for THIS node's region, so membership in it is
+            # positive evidence of being in-region — the same evidence `_foreign_gateways` uses,
+            # read the other way round. Latency still decides between two gateways that are both
+            # in region, which is what the RTT probe was for.
+            in_region = set(regional)
+            distinct.sort(key=lambda u: (0 if u in in_region else 1, self._rtt_s(u)))
+
+        # PADDING REMOVED 2026-09-28, AFTER IT COST 18 TURNS IN 20.
+        #
+        # The padding repeated one url to reach `want` sockets, reasoning that two sockets to a
+        # min-instances=2 service land on two different instances and give A3's redundancy. The
+        # reasoning is sound and the gateway forbids the thing it relies on: `gateway.py` allows
+        # ONE socket per node id PER INSTANCE, and a second closes the first with 4004 (there is a
+        # reclaim path for a silent socket, and a hard refusal otherwise). Whether the two sockets
+        # land on different instances is Cloud Run's choice, not ours — so padding is a coin flip
+        # between "redundancy" and "the node evicts itself".
+        #
+        # It landed on the wrong side. With NODE_GATEWAYS=2 and one region url, the live gateway
+        # log was a continuous alternation for a single node:
+        #     provider connected: stevai-warm-1 / provider gone: stevai-warm-1 / connected / gone …
+        # and 20 consecutive real turns produced 2 answers: 17 failed at `/discover` with "no
+        # gateway with an active provider", because the node's own two sockets kept knocking each
+        # other off and the directory's view of it was stale most of the time.
+        #
+        # So: ONE SOCKET PER DISTINCT GATEWAY, never the same gateway twice. A region that
+        # publishes one gateway gets one socket, and that is reported rather than papered over.
+        # The instance-coverage problem the padding was reaching for is REAL and is a gateway-side
+        # question (an instance holding no socket cannot serve), but it cannot be solved by making
+        # a node fight itself.
+        if len(distinct) < want:
+            print(f"[{self.name}] region publishes {len(distinct)} distinct gateway(s); "
+                  f"holding {len(distinct)} socket(s), not {want} — a second socket to the SAME "
+                  f"gateway would be refused as a duplicate of this node id")
+        return distinct[:want]
+
+    #: Re-probe cadence for the region's gateway list (02-A3: "re-probe every 10 min").
+    REPROBE_S = float(envvar("GATEWAY_REPROBE_S", "600"))
+
+    def _foreign_gateways(self) -> set:
+        """Gateway urls the directory says belong to a region that is NOT this node's.
+
+        Used to police `STEVAI_GATEWAY_URLS`, which the RunPod pod specs populate with ALL THREE
+        regions. An eu-west node sorted those by RTT and dialled us-east; the directory keeps one
+        gateway per node, so it registered under `gw-us-east-a` while declaring `region=eu-west`,
+        and `/discover` for eu-west found no provider — twenty consecutive turns, zero landed
+        (2026-09-28). It is also simply wrong for data residency (plan 29).
+
+        Empty on any failure, so a directory blink can never strand a node: unknown is treated as
+        allowed, and only POSITIVE evidence of another region removes a url.
+        """
+        now = time.time()
+        cached = getattr(self, "_foreign_cache", None)
+        if cached and (now - cached[0]) < self.REPROBE_S:
+            return set(cached[1])
+        base = envvar("DIRECTORY_URL", "").rstrip("/")
+        if not base:
+            return set()
+        try:
+            import urllib.request
+            with urllib.request.urlopen(f"{base}/gateways", timeout=8) as r:
+                data = json.loads(r.read() or b"{}")
+        except Exception as e:
+            print(f"[{self.name}] could not fetch the full gateway list "
+                  f"({type(e).__name__}); not policing the configured list")
+            return set()
+        mine = (self.region or "").strip()
+        out = {str(g.get("url") or "") for g in (data.get("gateways") or [])
+               if str(g.get("region") or "") != mine and g.get("url")}
+        self._foreign_cache = (now, set(out))
+        return out
+
+    def _directory_gateways(self) -> list:
+        """The node's own region's gateways, newest view, as provider WebSocket urls.
+
+        Cached for `REPROBE_S`; every failure returns [] so the caller falls back rather than
+        the node failing to start because the control plane blinked.
+        """
+        now = time.time()
+        cached = getattr(self, "_gw_cache", None)
+        if cached and (now - cached[0]) < self.REPROBE_S:
+            return list(cached[1])
+
+        base = envvar("DIRECTORY_URL", "").rstrip("/")
+        if not base:
+            return []
+        try:
+            import urllib.parse
+            import urllib.request
+            q = urllib.parse.urlencode({"region": self.region or ""})
+            with urllib.request.urlopen(f"{base}/gateways?{q}", timeout=8) as r:
+                data = json.loads(r.read() or b"{}")
+        except Exception as e:
+            print(f"[{self.name}] could not fetch the region gateway list ({type(e).__name__}); "
+                  f"falling back to the configured gateway")
+            return []
+
+        urls = []
+        for g in data.get("gateways") or []:
+            # Trust but verify: the directory filters by region, and so do we. A node must never
+            # hold a socket to another region's gateway, whatever the control plane returns.
+            if self.region and g.get("region") and g["region"] != self.region:
+                continue
+            ws = _provider_ws_url(str(g.get("url") or ""))
+            if ws:
+                urls.append(ws)
+        self._gw_cache = (now, list(urls))
+        return urls
+
+    def _rtt_s(self, ws_url: str) -> float:
+        """Round-trip time to a gateway, for primary/standby ordering. inf when unreachable, so
+        a dead gateway sorts last instead of becoming the primary."""
+        import urllib.request
+        health = ws_url.replace("wss://", "https://").replace("ws://", "http://")
+        health = health.split("/provider/ws")[0] + "/health"
+        t0 = time.time()
+        try:
+            urllib.request.urlopen(health, timeout=4).read(1)
+            return time.time() - t0
+        except Exception:
+            return float("inf")
+
+    def _check_own_build(self) -> None:
+        """FILE 22 — is the code this node is about to run the code we published?
+
+        ⚠️ REWRITTEN AFTER A REVIEW PROVED THE FIRST VERSION HANDED AN UNAUTHENTICATED ATTACKER A
+        FLEET KILL SWITCH, and turned a normal release into a crash-loop on billing pods.
+
+        The first version raised `REFUSING TO SERVE` on any signature that did not hold and on any digest
+        that differed. Two consequences, both worse than the gap it closed:
+
+        1. **Anyone who can answer `/node-package`** — DNS, a proxy, a compromised sibling — could reply
+           `{"signed": true, "signature": "garbage"}` and stop every node in the fleet. NO KEY REQUIRED.
+           `deploy/runpod/entrypoint.sh` restarts the agent forever, so that is an infinite crash-loop on
+           a rented GPU serving nothing, which is the exact failure the comment above that loop was
+           written about after it cost real money.
+        2. **A digest mismatch is the repo's own normal release order.** `config.py`'s
+           `extra_agent_digests` exists precisely because "a directory only ever knows the digest of the
+           code IT was built from, so any node-agent change locks out every still-running older node the
+           moment the directory deploys first". The old node is SUPPOSED to keep serving through the
+           rollout, and admission — which is authenticated — decides. My check refused instead.
+
+        So the asymmetry is now about WHAT WE CAN CONCLUDE, not about how bad the news sounds:
+
+        * **A signature that does not verify tells us nothing at all.** The answer is not from Frutebox, so
+          it is not evidence about this build — it is noise from whoever replied. Disregard it loudly and
+          carry on. Refusing would let an unauthenticated stranger decide whether we serve.
+        * **A verified manifest naming OUR version with a DIFFERENT digest is authenticated evidence that
+          this node is running code that was not published under that version.** That refuses.
+        * **A verified manifest naming a different version is a rollout.** Report the skew and carry on;
+          the gateway's admission check is the authenticated gate for that, and it already handles it.
+
+        The remaining gap, stated because it is not closed here: the digest covers four modules
+        (`attest._AGENT_FILES`), not the tarball, and nothing verifies the package before `install.sh`
+        runs. See `docs/as-built/05-known-gaps.md`.
+        """
+        base = envvar("DIRECTORY_URL", "").rstrip("/")
+        if not base:
+            # Nothing to compare against. A contributor laptop following the packaged `run-node.sh` sets
+            # only the gateway URL and reaches here — which is why this returns rather than refuses, and
+            # why the gateway's admission check remains the gate that actually matters.
+            return
+        strict = envvar("REQUIRE_SIGNED_PACKAGE", "").strip() == "1"
+        try:
+            import urllib.request
+            with urllib.request.urlopen(f"{base}/node-package", timeout=8) as r:
+                manifest = json.loads(r.read() or b"{}")
+        except Exception as e:
+            print(f"[{self.name}] could not fetch the package manifest ({type(e).__name__}); "
+                  f"the gateway still checks this build's digest at admission")
+            if strict:
+                raise
+            return
+
+        mine = attest.agent_digest()
+        if not manifest.get("signed"):
+            msg = (f"[{self.name}] the control plane serves an UNSIGNED package manifest — this build's "
+                   f"digest cannot be attributed to Frutebox, only to whoever answered")
+            if strict:
+                raise RuntimeError(msg + " (FRUTEBOX_REQUIRE_SIGNED_PACKAGE=1)")
+            print(msg)
+            return
+        try:
+            signed = packaging.verify_package_manifest(manifest)
+        except packaging.NoVerifyKeyHere as e:
+            # A DISTINCT TYPE, not `if "not set" in str(e)`. That substring test was one reworded message
+            # away from turning a refusal into a warning; this cannot drift.
+            msg = (f"[{self.name}] cannot verify the package manifest: {e}. This node has no verify key, "
+                   f"so the manifest is not evidence either way. The gateway's admission check is "
+                   f"unaffected.")
+            if strict:
+                raise RuntimeError(msg + " (FRUTEBOX_REQUIRE_SIGNED_PACKAGE=1)") from e
+            print(msg)
+            return
+        except packaging.PackageRejected as e:
+            # NOT A REFUSAL. An answer we cannot authenticate is not evidence about this build, and
+            # treating it as evidence is what made an unauthenticated reply a fleet-wide stop button.
+            msg = (f"[{self.name}] DISREGARDING the package manifest: {e}. It is not attributable to "
+                   f"Frutebox, so it says nothing about this build. The gateway's admission check is the "
+                   f"authenticated gate and is unaffected.")
+            if strict:
+                raise RuntimeError(msg + " (FRUTEBOX_REQUIRE_SIGNED_PACKAGE=1)") from e
+            print(msg)
+            return
+
+        if signed["agent_digest"] == mine:
+            print(f"[{self.name}] build verified against the signed manifest "
+                  f"(version {signed['version']}, {mine[:12]}…)")
+            return
+        if signed["version"] and signed["version"] != packaging.NODE_VERSION:
+            # A rollout, not a wrong build. Both versions are meant to be admissible for its length.
+            print(f"[{self.name}] the control plane publishes version {signed['version']} and this node "
+                  f"is {packaging.NODE_VERSION} — a rollout is in progress. Carrying on; admission "
+                  f"decides whether this digest is still accepted (see EXTRA_AGENT_DIGESTS).")
+            return
+        raise RuntimeError(
+            f"[{self.name}] REFUSING TO SERVE: the signed manifest names version "
+            f"{signed['version'] or '?'} — this node's own version — with agent digest "
+            f"{signed['agent_digest'][:12]}…, and this node computes {mine[:12]}…. That is signed "
+            f"evidence that this build was not published under this version. Re-fetch the package from "
+            f"{base}/node-package rather than continuing.")
+
+    async def run(self) -> None:
+        self._start_gui()
+        self._check_own_build()
+        urls = self.gateway_urls()
+        if len(urls) == 1:
+            await self._run_one(urls[0])
+            return
+        print(f"[{self.name}] holding {len(urls)} gateway sockets: {', '.join(urls)}")
+        # Concurrency is still capped by `max_concurrency` on the node as a whole, so two
+        # sockets cannot make the machine take twice the work — they only mean the job can
+        # arrive by either route.
+        await asyncio.gather(*(self._run_one(u) for u in urls))
+
+    async def _run_one(self, url: str, first_rotate: bool = False) -> None:
+        backoff = 1.0
+        rotate = first_rotate
+        while not self._stop:
+            try:
+                # Pick up a replacement credential written since the last attempt —
+                # this is how an expiry is actually resolved, rather than by retrying.
+                self._refresh_credential()
+                await self._serve_once(url, rotate=rotate)
+                rotate = False       # only the FIRST register of a replacement claims rotation
+                backoff = 1.0
+            except Exception as e:
+                code = _close_code(e)
+                # A REFUSED ROTATION IS NOT RE-ASSERTED. `rotate` is cleared only after a
+                # SUCCESSFUL register, so a replacement socket whose rotation was refused used to
+                # re-claim it on every retry — and the gateway refuses a rotation it cannot prove
+                # every single time, so the two spun against each other forever.
+                #
+                # MEASURED 2026-10-14 on the media node, which has no credential: the gateway
+                # logged "REFUSED rotation for 'frutebox-runpod-2': no verified credential" every
+                # ~30s, the node logged 17 consecutive 4003s, and it held a socket almost never.
+                # Media requests then failed as "no node is serving 'image'" — a node that cannot
+                # keep a socket is indistinguishable from absent capacity. Note the bug is only
+                # reachable WITHOUT a credential: the branch below handles a refusal for nodes
+                # that have one, and it was the `and self.credential` that let this fall through
+                # to the generic retry with the stale claim intact.
+                #
+                # Retrying as an ORDINARY socket is strictly better: it succeeds if the old socket
+                # has gone, and is refused as a duplicate if it has not — in which case the live
+                # socket is still serving and `_rotate_when_due` will come round again.
+                if code == _CLOSE_REJECTED and rotate:
+                    print(f"[{self.name}] rotation was refused; retrying as an ordinary socket")
+                    rotate = False
+                if code in (_CLOSE_CRED_EXPIRED, _CLOSE_REJECTED) and self.credential:
+                    # The gateway told us THIS credential is no good. Re-presenting it
+                    # on a 1s loop changes nothing and, before the gateway learned to
+                    # tell expiry from tampering, got honest nodes slashed off the
+                    # network in seconds. Say what to do, then wait properly.
+                    what = ("has EXPIRED" if code == _CLOSE_CRED_EXPIRED
+                            else "was REFUSED by the gateway")
+                    print(f"[{self.name}] node credential {what}. Get a fresh one "
+                          f"(the desktop app does this automatically; for this node ask "
+                          f"the operator / your dashboard for a new credential) and write "
+                          f"it to --credential-file or $FRUTEBOX_NODE_CREDENTIAL — it is "
+                          f"picked up on the next attempt, no restart needed. "
+                          f"Retrying in {_CRED_RETRY_S:.0f}s.")
+                    if self._stop:
+                        break
+                    # Poll the credential source while we wait so a fresh file is used
+                    # the moment it appears instead of after the full backoff.
+                    waited = 0.0
+                    while waited < _CRED_RETRY_S and not self._stop:
+                        await asyncio.sleep(min(5.0, _CRED_RETRY_S - waited))
+                        waited += 5.0
+                        if self._refresh_credential():
+                            print(f"[{self.name}] picked up a new credential — reconnecting")
+                            break
+                    backoff = 1.0
+                    continue
+                if code == _CLOSE_DUPLICATE:
+                    print(f"[{self.name}] the gateway already has a live connection for "
+                          f"id '{self.id}' — another process is sharing under this id. "
+                          f"Retrying in {backoff:.0f}s.")
+                elif code == _CLOSE_ROTATED:
+                    # Expected, and the whole point: this socket was replaced by one this node
+                    # opened itself, which is already serving. Reconnecting here would dial a
+                    # SECOND socket for an id that now has a live one and be refused as a
+                    # duplicate — so the old socket simply stands down, quietly.
+                    print(f"[{self.name}] socket rotated out; its replacement is serving")
+                    return
+                else:
+                    print(f"[{self.name}] disconnected ({type(e).__name__}: {e}); "
+                          f"retrying in {backoff:.0f}s")
+                _publish_link_state("disconnected")
+                # NO STAND-DOWN. This used to `return` when another socket for this url was
+                # registered, to avoid two tasks racing for one node id. It STRANDED THE URL, in
+                # production, on 2026-09-29: the rotation task hit a refusal and returned, the
+                # original had already stood down because the rotation was live at that moment,
+                # and nothing was left to redial eu-west. The gateway sat at providers=0 while
+                # the node reported itself healthy and served the other region.
+                #
+                # Racing is the cheaper failure. The gateway already enforces one socket per node
+                # id and refuses the loser with 4004, which this loop treats as an ordinary
+                # disconnect and retries with backoff. That is a little noise in a log. A URL
+                # nothing reconnects is a region with no capacity until someone notices.
+                if self._stop:
+                    break
+                await asyncio.sleep(backoff)
+                backoff = min(backoff * 2, 30.0)
+        print(f"[{self.name}] stopped sharing on {url}.")
+
+
+    async def _answer_tee_challenge(self, ws, challenge: str) -> None:
+        """FILE 08-B2 (node side) — present this machine's dual attestation, if it has one.
+
+        SILENT WHEN THERE IS NOTHING TO PRESENT, and that is the normal case: an ordinary node has no
+        evidence files, sends no frame, and never joins the confidential lane. The gateway's default is
+        already "not attested", so saying nothing is the safe answer rather than a missed opportunity.
+
+        NOTHING HERE CAN TAKE THE NODE DOWN. Every failure — no files, unreadable files, a socket that
+        rejects the frame — is caught and logged, because this node's ORDINARY traffic is unrelated to
+        the confidential lane and must not depend on it. The one thing that would be worse than not
+        joining the lane is a confidential-computing feature that breaks plain inference.
+
+        The challenge is echoed back inside the evidence, not beside it: the JWTs are produced by the
+        image's attestation tooling with the nonce baked into the signed report, which is the only form
+        that proves anything. A node cannot add a nonce to someone else's signed report, which is
+        exactly why the gateway insists on finding it there.
+        """
+        if not challenge:
+            return
+        cpu_path, gpu_path = Path(TEE_CPU_EVIDENCE), Path(TEE_GPU_EVIDENCE)
+        if not (cpu_path.exists() and gpu_path.exists()):
+            return                     # not a confidential host; nothing to say
+        try:
+            cpu = cpu_path.read_text(encoding="utf-8").strip()
+            gpu = gpu_path.read_text(encoding="utf-8").strip()
+        except OSError as e:
+            print(f"[{self.name}] TEE evidence present but unreadable ({type(e).__name__}); "
+                  f"staying off the confidential lane")
+            return
+        if not (cpu and gpu):
+            print(f"[{self.name}] TEE evidence is empty; staying off the confidential lane")
+            return
+        try:
+            await ws.send(json.dumps({
+                "type": "attestation", "cpu_tee": cpu, "gpu_tee": gpu,
+                # Which verification path the image used. "maa" means Azure vouched for the CPU report
+                # rather than us verifying it ourselves, and the gateway refuses it unless the
+                # hyperscaler sub-lane is switched on — so it is declared, never assumed.
+                "cpu_path": envvar("TEE_CPU_PATH", "self"),
+                "provider": envvar("TEE_PROVIDER", ""),
+                # NOT the binding. The gateway checks the nonce inside the signed reports; this is
+                # echoed only so an operator reading a log can match the frame to the challenge.
+                "challenge_echo": challenge[:16],
+            }))
+            print(f"[{self.name}] presented a TEE attestation for the gateway's challenge")
+        except Exception as e:
+            print(f"[{self.name}] could not send the TEE attestation ({type(e).__name__}); "
+                  f"ordinary traffic is unaffected")
+
+#: Where a confidential host leaves its attestation evidence for this agent to read.
+#:
+#: FILES, not a subprocess. Producing the evidence means running `snpguest`/`tdx-attest` and calling
+#: NVIDIA's attestation SDK, which needs the right driver, the right privileges and a network call — all
+#: of it specific to the confidential image, none of it appropriate inside a hot node agent. So the image
+#: writes the two JWTs and this reads them. That also keeps the node agent testable and means a broken
+#: attestation tool cannot take the node's ORDINARY traffic down with it.
+TEE_CPU_EVIDENCE = envvar("TEE_CPU_EVIDENCE", "/run/frutebox/cpu_tee.jwt")
+TEE_GPU_EVIDENCE = envvar("TEE_GPU_EVIDENCE", "/run/frutebox/gpu_tee.jwt")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(description="frutebox provider node")
+    p.add_argument("--name", default=None, help="Human-friendly node name")
+    p.add_argument("--id", default=None, help="Stable node id (defaults to random)")
+    p.add_argument("--region", default=CONFIG.default_region,
+                   choices=CONFIG.regions, help="Jurisdiction this hardware sits in")
+    p.add_argument("--compute-type", default=None, choices=["gpu", "cpu"],
+                   help="Hardware class. Omit to auto-detect (GPU if nvidia-smi "
+                        "reports VRAM, else CPU). GPUs are primary; CPUs are a "
+                        "slower fallback used when no GPU is available.")
+    p.add_argument("--memory-gb", type=float, default=0.0,
+                   help="Memory this node contributes (VRAM for GPU, RAM for CPU). "
+                        "Lets many small CPUs pool their RAM to hold a big model.")
+    p.add_argument("--cpu-cores", type=int, default=0,
+                   help="CPU core count (used to estimate CPU throughput).")
+    p.add_argument("--models", default="",
+                   help="Comma-separated models served (empty = serves anything)")
+    p.add_argument("--cost-per-1k", type=float, default=CONFIG.default_cost_per_1k,
+                   help="Energy+maintenance cost charged per 1000 tokens")
+    p.add_argument("--concurrency", type=int, default=0,
+                   help="Max simultaneous jobs this node will run — advertised AND "
+                        "enforced. 0 = the node-governance default "
+                        f"({CONFIG.node_max_concurrent_jobs}, $FRUTEBOX_NODE_MAX_CONCURRENT_JOBS).")
+    p.add_argument("--job-timeout-s", type=float, default=0.0,
+                   help="Hard wall-clock limit for one job on this machine; the job is "
+                        "cancelled and reported as an error. 0 = the node-governance "
+                        f"default ({CONFIG.node_job_timeout_s:.0f}s, $FRUTEBOX_NODE_JOB_TIMEOUT_S).")
+    p.add_argument("--gateway-url", default=None,
+                   help="Network gateway WebSocket URL to dial into")
+    p.add_argument("--credential", default="",
+                   help="Signed node credential issued for this node id (admits "
+                        "this machine without a manual operator approval). It is "
+                        "~4.5KB and secret-ish — prefer --credential-file.")
+    p.add_argument("--credential-file", default="",
+                   help="Path to a file containing the signed node credential "
+                        "(falls back to $FRUTEBOX_NODE_CREDENTIAL).")
+    p.add_argument("--gui-port", type=int, default=0,
+                   help="If set, serve the modern local dashboard on this port "
+                        "(e.g. --gui-port 8790 → http://127.0.0.1:8790)")
+    return p
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    if args.gateway_url is None:
+        args.gateway_url = envvar("GATEWAY_WS", "ws://127.0.0.1:8090/provider/ws")
+    node = ProviderNode(args)
+    try:
+        asyncio.run(node.run())
+    except KeyboardInterrupt:
+        print(f"\n[{node.name}] shutting down")
+
+
+if __name__ == "__main__":
+    main()