nvdc 0.2.2__tar.gz → 0.2.3__tar.gz

{nvdc-0.2.2 → nvdc-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nvdc
-Version: 0.2.2
+Version: 0.2.3
 Summary: Bring your GPU onto the network: one command turns a GPU into a verifiable, OpenAI-compatible inference node.
 Author: NVDC
 License: Apache-2.0
@@ -17,6 +17,8 @@ Requires-Dist: stripe>=9.0
 Provides-Extra: attestation
 Requires-Dist: nv-attestation-sdk>=2.7.0; extra == "attestation"
 Requires-Dist: nv-local-gpu-verifier>=2.7.0; extra == "attestation"
+Provides-Extra: mlx
+Requires-Dist: mlx-lm>=0.22; extra == "mlx"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 
@@ -127,8 +129,14 @@ client.chat.completions.create(model="llama3.1:8b",
 ## Models
 
 The catalog is pinned to the **Ollama** library (reliable, known sizes; Ollama
-also handles CUDA / Apple Metal / CPU offload). Each node reports its memory
-budget and the UI marks every model **Fits / Tight / Won't fit** against it:
+also handles CUDA / Apple Metal / CPU offload). Each catalog entry also maps to
+a canonical **MLX** artifact (4-bit, `mlx-community` on Hugging Face): run
+`nvdc serve --backend mlx` and the client downloads the weights into its own
+store (`~/.nvdc/models`), hashes them itself (full custody), re-verifies them
+before every go-live, and serves them via a managed `mlx_lm.server` —
+`pip install "nvdc[mlx]"` (Apple silicon; on Linux add `mlx[cuda]`).
+Each node reports its memory budget and the UI marks every model
+**Fits / Tight / Won't fit** against it:
 
 - unified-memory systems (DGX Spark / GB10, Apple Silicon) → budget = system RAM
 - dedicated-VRAM GPUs → budget = VRAM

{nvdc-0.2.2 → nvdc-0.2.3}/README.md

@@ -105,8 +105,14 @@ client.chat.completions.create(model="llama3.1:8b",
 ## Models
 
 The catalog is pinned to the **Ollama** library (reliable, known sizes; Ollama
-also handles CUDA / Apple Metal / CPU offload). Each node reports its memory
-budget and the UI marks every model **Fits / Tight / Won't fit** against it:
+also handles CUDA / Apple Metal / CPU offload). Each catalog entry also maps to
+a canonical **MLX** artifact (4-bit, `mlx-community` on Hugging Face): run
+`nvdc serve --backend mlx` and the client downloads the weights into its own
+store (`~/.nvdc/models`), hashes them itself (full custody), re-verifies them
+before every go-live, and serves them via a managed `mlx_lm.server` —
+`pip install "nvdc[mlx]"` (Apple silicon; on Linux add `mlx[cuda]`).
+Each node reports its memory budget and the UI marks every model
+**Fits / Tight / Won't fit** against it:
 
 - unified-memory systems (DGX Spark / GB10, Apple Silicon) → budget = system RAM
 - dedicated-VRAM GPUs → budget = VRAM

{nvdc-0.2.2 → nvdc-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nvdc"
-version = "0.2.2"
+version = "0.2.3"
 description = "Bring your GPU onto the network: one command turns a GPU into a verifiable, OpenAI-compatible inference node."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -19,6 +19,9 @@ dependencies = [
 
 [project.optional-dependencies]
 attestation = ["nv-attestation-sdk>=2.7.0", "nv-local-gpu-verifier>=2.7.0"]
+# Apple-silicon (and MLX-CUDA) inference with full client custody of weights.
+# On Linux, additionally install the CUDA flavor: pip install "mlx[cuda]"
+mlx = ["mlx-lm>=0.22"]
 dev = ["pytest>=8.0"]
 
 [project.scripts]

{nvdc-0.2.2 → nvdc-0.2.3}/src/nvdc/__init__.py

@@ -1,3 +1,3 @@
 """nvdc: bring your GPU onto the network as a verifiable, OpenAI-compatible inference node."""
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"

{nvdc-0.2.2 → nvdc-0.2.3}/src/nvdc/agent.py

@@ -21,7 +21,6 @@ from .gpu import detect_gpu, detect_gpus, detect_interconnect
 from .hardware import detect_hardware, machine_id as hw_machine_id
 from .inference import Backend, make_backend
 from .keys import Identity
-from .models import local_ollama_commit
 
 log = logging.getLogger("nvdc.agent")
 
@@ -215,15 +214,15 @@ class NodeAgent:
             )
         # Model identity commitment: the content digest of the artifact this
         # node will serve. Signed into every work receipt (nvdc-work-v2) and
-        # checked by the coordinator against the public registry.
-        if getattr(self.backend, "name", "") == "ollama":
-            self.model_commit = await local_ollama_commit(
-                self.model, getattr(self.backend, "base_url", "http://localhost:11434"))
-            if self.model_commit:
-                log.info("model commitment: %s -> %s", self.model, self.model_commit[:23] + "…")
-            else:
-                log.warning("could not derive a model commitment for '%s'; "
-                            "receipts will not carry a model identity", self.model)
+        # checked by the coordinator against public ground truth. For MLX the
+        # client itself hashed the weights it downloaded (full custody); for
+        # Ollama it is the content digest the local daemon reports.
+        self.model_commit = await self.backend.model_commitment(self.model)
+        if self.model_commit:
+            log.info("model commitment: %s -> %s", self.model, self.model_commit[:23] + "…")
+        elif getattr(self.backend, "name", "") in ("ollama", "mlx"):
+            log.warning("could not derive a model commitment for '%s'; "
+                        "receipts will not carry a model identity", self.model)
         profile = protocol.NodeProfile(
             name=self.name,
             models=[self.model],
@@ -243,6 +242,7 @@ class NodeAgent:
             cluster=self.cluster,
             private=self.private,
             model_commit=self.model_commit,
+            backend=getattr(self.backend, "name", "ollama"),
             agent_version=__version__,
         )
         await self._send(protocol.MSG_REGISTER, profile=protocol.node_profile_to_dict(profile))
@@ -392,4 +392,11 @@ async def serve(
         private=private,
         auto_update=auto_update,
     )
-    await agent.run_forever()
+    try:
+        await agent.run_forever()
+    finally:
+        # Backends that manage a server subprocess (MLX) must not orphan it
+        # when the agent exits (Ctrl-C, drain, upgrade-required, errors).
+        close = getattr(backend, "close", None)
+        if callable(close):
+            close()

{nvdc-0.2.2 → nvdc-0.2.3}/src/nvdc/catalog.py

@@ -25,37 +25,63 @@ class Model:
     size_gb: float     # published download size of this tag
     min_memory_gb: float  # recommended resident memory for hot inference
     description: str
+    # Hugging Face repo of the canonical MLX artifact for this model (4-bit,
+    # mlx-community). The coordinator derives MLX ground-truth digests from
+    # THIS mapping — never from a node-supplied repo. "" = no MLX artifact.
+    mlx_repo: str = ""
 
 
 # Curated, popular, reliably-available tags from the Ollama library.
 CATALOG: List[Model] = [
     Model("gpt-oss:20b", "GPT-OSS 20B", "gpt-oss", "20B", 14.0, 16.0,
-          "OpenAI's open-weight model; strong general reasoning, runs on a single 16GB+ GPU."),
+          "OpenAI's open-weight model; strong general reasoning, runs on a single 16GB+ GPU.",
+          mlx_repo="mlx-community/gpt-oss-20b-MXFP4-Q4"),
     Model("gpt-oss:120b", "GPT-OSS 120B", "gpt-oss", "120B", 65.0, 80.0,
-          "OpenAI's large open-weight model; needs ~80GB (high-end / unified-memory systems)."),
+          "OpenAI's large open-weight model; needs ~80GB (high-end / unified-memory systems).",
+          mlx_repo="mlx-community/gpt-oss-120b-MXFP4-Q4"),
     Model("llama3.1:8b", "Llama 3.1 8B", "llama", "8B", 4.9, 8.0,
-          "Meta Llama 3.1 8B Instruct; excellent quality-to-size, great default."),
+          "Meta Llama 3.1 8B Instruct; excellent quality-to-size, great default.",
+          mlx_repo="mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"),
     Model("llama3.1:70b", "Llama 3.1 70B", "llama", "70B", 43.0, 48.0,
-          "Meta Llama 3.1 70B Instruct; frontier-class quality, needs a large GPU."),
+          "Meta Llama 3.1 70B Instruct; frontier-class quality, needs a large GPU.",
+          mlx_repo="mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"),
+    Model("llama3.2:1b", "Llama 3.2 1B", "llama", "1B", 1.3, 2.0,
+          "Tiny Llama 3.2; runs almost anywhere, good for testing and edge.",
+          mlx_repo="mlx-community/Llama-3.2-1B-Instruct-4bit"),
     Model("llama3.2:3b", "Llama 3.2 3B", "llama", "3B", 2.0, 4.0,
-          "Small, fast Llama 3.2; good for low-memory GPUs and edge."),
+          "Small, fast Llama 3.2; good for low-memory GPUs and edge.",
+          mlx_repo="mlx-community/Llama-3.2-3B-Instruct-4bit"),
     Model("qwen2.5:7b", "Qwen2.5 7B", "qwen", "7B", 4.7, 8.0,
-          "Alibaba Qwen2.5 7B; strong multilingual and coding."),
+          "Alibaba Qwen2.5 7B; strong multilingual and coding.",
+          mlx_repo="mlx-community/Qwen2.5-7B-Instruct-4bit"),
     Model("qwen2.5:32b", "Qwen2.5 32B", "qwen", "32B", 20.0, 24.0,
-          "Qwen2.5 32B; high quality, needs a 24GB+ GPU."),
+          "Qwen2.5 32B; high quality, needs a 24GB+ GPU.",
+          mlx_repo="mlx-community/Qwen2.5-32B-Instruct-4bit"),
     Model("deepseek-r1:7b", "DeepSeek-R1 7B", "deepseek", "7B", 4.7, 8.0,
-          "Distilled reasoning model with visible chain-of-thought."),
+          "Distilled reasoning model with visible chain-of-thought.",
+          mlx_repo="mlx-community/DeepSeek-R1-Distill-Qwen-7B-4bit"),
     Model("deepseek-r1:70b", "DeepSeek-R1 70B", "deepseek", "70B", 43.0, 48.0,
           "Large distilled reasoning model."),
     Model("mistral:7b", "Mistral 7B", "mistral", "7B", 4.1, 6.0,
-          "Mistral 7B Instruct; fast, lightweight, widely used."),
+          "Mistral 7B Instruct; fast, lightweight, widely used.",
+          mlx_repo="mlx-community/Mistral-7B-Instruct-v0.3-4bit"),
     Model("gemma2:9b", "Gemma 2 9B", "gemma", "9B", 5.4, 9.0,
-          "Google Gemma 2 9B; balanced quality and size."),
+          "Google Gemma 2 9B; balanced quality and size.",
+          mlx_repo="mlx-community/gemma-2-9b-it-4bit"),
     Model("phi4:14b", "Phi-4 14B", "phi", "14B", 9.1, 12.0,
-          "Microsoft Phi-4; strong reasoning for its size."),
+          "Microsoft Phi-4; strong reasoning for its size.",
+          mlx_repo="mlx-community/phi-4-4bit"),
 ]
 
 
+def mlx_repo_for(model_id: str) -> str:
+    """The canonical MLX artifact repo for a catalog model id ("" if none)."""
+    for m in CATALOG:
+        if m.id == model_id:
+            return m.mlx_repo
+    return ""
+
+
 # Default reference market prices in USD per 1,000,000 output tokens. Used as a
 # fallback when no miner is currently online serving a given model. Loosely
 # tracks relative model size/cost; the live network price overrides these.
@@ -64,6 +90,7 @@ DEFAULT_PRICES = {
     "gpt-oss:120b": 0.60,
     "llama3.1:8b": 0.10,
     "llama3.1:70b": 0.40,
+    "llama3.2:1b": 0.03,
     "llama3.2:3b": 0.05,
     "qwen2.5:7b": 0.10,
     "qwen2.5:32b": 0.30,

{nvdc-0.2.2 → nvdc-0.2.3}/src/nvdc/cli.py

@@ -163,7 +163,10 @@ def build_parser() -> argparse.ArgumentParser:
     s.add_argument("--coordinator", "-c", default=DEFAULT_COORDINATOR_WS,
                    help=f"coordinator host/url (default: {DEFAULT_COORDINATOR_WS})")
     s.add_argument("--name", "-n", default=os.uname().nodename, help="node display name")
-    s.add_argument("--backend", "-b", default="ollama", choices=["ollama", "echo"])
+    s.add_argument("--backend", "-b", default="ollama",
+                   choices=["ollama", "mlx", "echo"],
+                   help="inference engine; 'mlx' downloads the canonical MLX "
+                        "artifact into the client's own store (full custody)")
     s.add_argument("--model", "-m", default="llama3.1:8b",
                    help="the single model this node commits to serving (hot-loaded)")
     s.add_argument("--ollama-url", default="http://localhost:11434")

{nvdc-0.2.2 → nvdc-0.2.3}/src/nvdc/coordinator.py

@@ -187,10 +187,13 @@ class Node:
         self.machine_id = profile.get("machine_id", "") or self.id
         self.cluster = profile.get("cluster", "") or ""
         # Model identity: the node's claimed artifact digest, and the verdict
-        # from checking that claim against the public registry at registration
+        # from checking that claim against public ground truth at registration
         # ("verified" | "mismatch" | "unknown").
         self.model_commit = str(profile.get("model_commit", "") or "")
         self.model_verdict = "unknown"
+        # Inference engine; ground truth differs per backend (ollama registry
+        # manifest vs Hugging Face LFS digests for the catalog's MLX repo).
+        self.backend = str(profile.get("backend", "") or "ollama")
         self.draining = False
         self.pending: Dict[str, PendingRequest] = {}
         self._send_lock = asyncio.Lock()
@@ -265,6 +268,7 @@ class Node:
             "price_per_mtok": round(self.price_per_mtok, 6),
             "price_per_mtok_input": round(self.price_per_mtok_input, 6),
             "private": self.private,
+            "backend": self.backend,
             "model_commit": self.model_commit,
             "model_verdict": self.model_verdict,
             "attested": self.attested,
@@ -511,24 +515,43 @@ def create_app(auth_token: str = "", require_attested: bool = False,
                 node.restore_stats(rec)
                 return
 
-    # Registry ground truth for model digests, cached so registration doesn't
-    # hammer registry.ollama.ai (tags are mutable, hence the TTL refresh).
+    # Ground truth for model digests, cached so registration doesn't hammer
+    # the upstream sources (tags/repos are mutable, hence the TTL refresh).
     _model_digest_cache: Dict[str, tuple] = {}
 
-    async def expected_model_digest(model: str) -> str:
-        hit = _model_digest_cache.get(model)
+    async def _cached(key: str, fetch) -> str:
+        hit = _model_digest_cache.get(key)
         if hit and time.time() - hit[1] < 6 * 3600:
             return hit[0]
-        digest = await model_identity.fetch_registry_digest(model)
-        _model_digest_cache[model] = (digest, time.time())
+        digest = await fetch()
+        _model_digest_cache[key] = (digest, time.time())
         return digest
 
+    async def expected_model_digest(node: Node) -> str:
+        """Authoritative digest for the node's model, per its backend."""
+        if node.backend == "mlx":
+            from .catalog import mlx_repo_for
+            repo = mlx_repo_for(node.model)
+            if not repo:
+                return ""  # non-catalog model: no canonical MLX ground truth
+
+            async def fetch_mlx() -> str:
+                digests = await model_identity.fetch_hf_weights_digests(repo)
+                return model_identity.weights_commitment(digests) if digests else ""
+
+            return await _cached("mlx:" + repo, fetch_mlx)
+        if node.backend in ("api", "echo"):
+            return ""  # no public artifact to anchor to
+        return await _cached(
+            "ollama:" + node.model,
+            lambda: model_identity.fetch_registry_digest(node.model))
+
     async def _verify_node_model(node: Node) -> None:
-        """Check the node's claimed artifact digest against the registry."""
+        """Check the node's claimed artifact digest against ground truth."""
         if not node.model_commit:
             return  # no claim made -> verdict stays "unknown"
         try:
-            expected = await expected_model_digest(node.model)
+            expected = await expected_model_digest(node)
         except Exception:
             expected = ""
         node.model_verdict = model_identity.model_verdict(node.model_commit, expected)

{nvdc-0.2.2 → nvdc-0.2.3}/src/nvdc/house.py

@@ -117,6 +117,7 @@ class HouseNode(Node):
             "owner_account": identity.account_id,
             "machine_id": "house-" + identity.account_id[-12:],
             "cluster": "house",
+            "backend": "api",
             "attestation": {
                 "supported": False, "verified": False, "mode": "none",
                 "reason": "house node: API-backed, no local GPU to attest",

nvdc-0.2.3/src/nvdc/inference.py

@@ -0,0 +1,350 @@
+"""Local inference backends exposed behind a tiny async interface.
+
+The node agent forwards an OpenAI ``/v1/chat/completions`` body to a backend and
+relays whatever the backend produces straight back through the tunnel, so the
+public API stays byte-for-byte OpenAI-compatible.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import atexit
+import json
+import logging
+import platform
+import socket
+import subprocess
+import sys
+import time
+import uuid
+from pathlib import Path
+from typing import Any, AsyncIterator, Dict, List, Optional
+
+import httpx
+
+log = logging.getLogger("nvdc.inference")
+
+
+class Backend:
+    name = "base"
+
+    async def list_models(self) -> List[str]:
+        raise NotImplementedError
+
+    async def warm(self, model: str) -> None:
+        """Load the model resident in memory for hot inference (best-effort)."""
+        return None
+
+    async def model_commitment(self, model: str) -> str:
+        """Content digest of the artifact this backend serves ("" if unknown).
+
+        Signed into every work receipt (nvdc-work-v2) and verified by the
+        coordinator against public ground truth.
+        """
+        return ""
+
+    async def chat_stream(self, body: Dict[str, Any]) -> AsyncIterator[str]:
+        """Yield raw SSE 'data: ...' lines (OpenAI streaming format)."""
+        raise NotImplementedError
+        yield  # pragma: no cover
+
+    async def chat_once(self, body: Dict[str, Any]) -> Dict[str, Any]:
+        """Return a full OpenAI chat.completion object."""
+        raise NotImplementedError
+
+
+class OllamaBackend(Backend):
+    name = "ollama"
+
+    def __init__(self, base_url: str = "http://localhost:11434", default_model: str = ""):
+        self.base_url = base_url.rstrip("/")
+        self.default_model = default_model
+
+    async def list_models(self) -> List[str]:
+        async with httpx.AsyncClient(timeout=10) as c:
+            r = await c.get(f"{self.base_url}/api/tags")
+            r.raise_for_status()
+            return [m["name"] for m in r.json().get("models", [])]
+
+    async def warm(self, model: str) -> None:
+        async with httpx.AsyncClient(timeout=None) as c:
+            await c.post(f"{self.base_url}/api/generate", json={
+                "model": model, "prompt": "ok", "stream": False, "keep_alive": -1,
+            })
+
+    async def model_commitment(self, model: str) -> str:
+        # Ollama owns its model store; the commitment is the content digest
+        # the (local) daemon reports for the tag it will serve.
+        from .models import local_ollama_commit
+        return await local_ollama_commit(model, self.base_url)
+
+    @staticmethod
+    def _error(status: int, text: str, model: str) -> str:
+        detail = text
+        try:
+            j = json.loads(text)
+            detail = (j.get("error") or j.get("message") or text)
+            if isinstance(detail, dict):
+                detail = detail.get("message", text)
+        except Exception:
+            pass
+        return f"ollama error {status} for model '{model}': {str(detail)[:400]}"
+
+    async def chat_once(self, body: Dict[str, Any]) -> Dict[str, Any]:
+        body = dict(body)
+        body["stream"] = False
+        async with httpx.AsyncClient(timeout=None) as c:
+            r = await c.post(f"{self.base_url}/v1/chat/completions", json=body)
+            if r.status_code >= 400:
+                raise RuntimeError(self._error(r.status_code, r.text, body.get("model", "")))
+            return r.json()
+
+    async def chat_stream(self, body: Dict[str, Any]) -> AsyncIterator[str]:
+        body = dict(body)
+        body["stream"] = True
+        async with httpx.AsyncClient(timeout=None) as c:
+            async with c.stream("POST", f"{self.base_url}/v1/chat/completions", json=body) as r:
+                if r.status_code >= 400:
+                    await r.aread()
+                    raise RuntimeError(self._error(r.status_code, r.text, body.get("model", "")))
+                async for line in r.aiter_lines():
+                    if line:
+                        yield line
+
+
+def ensure_mlx_lm(auto_install: bool = True) -> bool:
+    """True when mlx-lm is importable, installing it first when allowed.
+
+    Auto-install is attempted only on Apple silicon (where ``pip install
+    mlx-lm`` just works); on other platforms the operator picks the right
+    MLX flavor themselves (e.g. ``pip install mlx-lm 'mlx[cuda]'``).
+    """
+    try:
+        import mlx_lm  # noqa: F401
+        return True
+    except ImportError:
+        pass
+    if not (auto_install and sys.platform == "darwin" and platform.machine() == "arm64"):
+        return False
+    log.info("installing mlx-lm into this client's environment…")
+    proc = subprocess.run([sys.executable, "-m", "pip", "install", "mlx-lm"],
+                          capture_output=True, text=True, timeout=900)
+    if proc.returncode != 0:
+        log.error("mlx-lm install failed: %s", (proc.stderr or "")[-400:])
+        return False
+    try:
+        import mlx_lm  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def _free_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+class MlxBackend(Backend):
+    """Apple-silicon (and MLX-CUDA) backend with full client custody.
+
+    The client downloads the canonical MLX artifact from Hugging Face into its
+    own store, hashes the weights itself, re-verifies them before serving, and
+    runs ``mlx_lm.server`` (OpenAI-compatible) as a managed subprocess. The
+    model commitment signed into receipts describes bytes the client measured.
+    """
+
+    name = "mlx"
+
+    def __init__(self, model: str, mlx_repo: str = "", store=None,
+                 auto_install: bool = True):
+        from .catalog import mlx_repo_for
+        from .store import ModelStore
+        self.model = model
+        self.repo = mlx_repo or mlx_repo_for(model)
+        if not self.repo:
+            raise ValueError(
+                f"no canonical MLX artifact for '{model}' — pick a catalog "
+                f"model with an mlx_repo, or use --backend ollama")
+        if not ensure_mlx_lm(auto_install=auto_install):
+            raise ValueError(
+                "mlx-lm is not installed and could not be installed "
+                "automatically; run: pip install 'nvdc[mlx]' "
+                "(Linux/CUDA: pip install mlx-lm 'mlx[cuda]')")
+        self.store = store or ModelStore()
+        self.port = _free_port()
+        self._proc: Optional[subprocess.Popen] = None
+        self._commit = ""
+        self._model_dir: Optional[Path] = None
+
+    @property
+    def base_url(self) -> str:
+        return f"http://127.0.0.1:{self.port}"
+
+    async def list_models(self) -> List[str]:
+        return [self.model]
+
+    async def model_commitment(self, model: str) -> str:
+        return self._commit or self.store.commitment(self.model)
+
+    async def warm(self, model: str) -> None:
+        model_dir = await asyncio.to_thread(self.store.ensure, self.model, self.repo)
+        # Custody check: the commitment we are about to sign must describe
+        # the bytes on disk RIGHT NOW, not at download time.
+        if not await asyncio.to_thread(self.store.verify, self.model):
+            raise RuntimeError(
+                f"custody verification FAILED for {self.model}: weights on "
+                f"disk do not match this client's manifest — refusing to "
+                f"serve (delete {model_dir} to re-download)")
+        self._commit = self.store.commitment(self.model)
+        self._model_dir = model_dir
+        await self._start_server(model_dir)
+
+    async def _start_server(self, model_dir: Path) -> None:
+        if self._proc and self._proc.poll() is None:
+            return
+        log_path = model_dir / "server.log"
+        cmd = [sys.executable, "-m", "mlx_lm", "server",
+               "--model", str(model_dir), "--host", "127.0.0.1",
+               "--port", str(self.port)]
+        log.info("starting mlx_lm.server on port %d for %s", self.port, self.model)
+        with open(log_path, "ab") as lf:
+            self._proc = subprocess.Popen(cmd, stdout=lf, stderr=lf)
+        atexit.register(self.close)
+        deadline = time.time() + 300  # large models take a while to load
+        async with httpx.AsyncClient(timeout=5) as c:
+            while time.time() < deadline:
+                if self._proc.poll() is not None:
+                    tail = ""
+                    try:
+                        tail = log_path.read_bytes()[-600:].decode(errors="replace")
+                    except Exception:
+                        pass
+                    raise RuntimeError(f"mlx_lm.server exited at startup: {tail}")
+                try:
+                    r = await c.get(f"{self.base_url}/health")
+                    if r.status_code < 500:
+                        break
+                except Exception:
+                    pass
+                await asyncio.sleep(0.5)
+            else:
+                raise RuntimeError("mlx_lm.server did not become ready in time")
+        # Force the weights resident so the node is genuinely hot.
+        await self.chat_once({"model": self.model, "max_tokens": 1,
+                              "messages": [{"role": "user", "content": "ok"}]})
+
+    def close(self) -> None:
+        if self._proc and self._proc.poll() is None:
+            self._proc.terminate()
+            try:
+                self._proc.wait(timeout=5)
+            except Exception:
+                self._proc.kill()
+
+    @staticmethod
+    def _error(status: int, text: str, model: str) -> str:
+        detail = text
+        try:
+            j = json.loads(text)
+            detail = (j.get("error") or j.get("message") or text)
+            if isinstance(detail, dict):
+                detail = detail.get("message", text)
+        except Exception:
+            pass
+        return f"mlx error {status} for model '{model}': {str(detail)[:400]}"
+
+    def _dir(self) -> str:
+        return str(self._model_dir or self.store.dir_for(self.model))
+
+    async def chat_once(self, body: Dict[str, Any]) -> Dict[str, Any]:
+        body = dict(body)
+        body["stream"] = False
+        # mlx_lm.server resolves the request's model as a repo/path to load;
+        # translate the canonical network id to the local store directory and
+        # restore the canonical id in the response.
+        body["model"] = self._dir()
+        async with httpx.AsyncClient(timeout=None) as c:
+            r = await c.post(f"{self.base_url}/v1/chat/completions", json=body)
+            if r.status_code >= 400:
+                raise RuntimeError(self._error(r.status_code, r.text, self.model))
+            result = r.json()
+            if isinstance(result, dict) and result.get("model"):
+                result["model"] = self.model
+            return result
+
+    async def chat_stream(self, body: Dict[str, Any]) -> AsyncIterator[str]:
+        body = dict(body)
+        body["stream"] = True
+        body["model"] = self._dir()
+        local_dir = self._dir()
+        async with httpx.AsyncClient(timeout=None) as c:
+            async with c.stream("POST", f"{self.base_url}/v1/chat/completions", json=body) as r:
+                if r.status_code >= 400:
+                    await r.aread()
+                    raise RuntimeError(self._error(r.status_code, r.text, self.model))
+                async for line in r.aiter_lines():
+                    if line:
+                        yield line.replace(local_dir, self.model)
+
+
+class EchoBackend(Backend):
+    """Zero-dependency backend that streams back a canned response. Used to test
+    the network plumbing without downloading a model."""
+
+    name = "echo"
+
+    def __init__(self, model: str = "echo-1"):
+        self.model = model
+
+    async def list_models(self) -> List[str]:
+        return [self.model]
+
+    def _reply_text(self, body: Dict[str, Any]) -> str:
+        msgs = body.get("messages", [])
+        last = msgs[-1]["content"] if msgs else ""
+        return f"[echo] You said: {last}"
+
+    async def chat_once(self, body: Dict[str, Any]) -> Dict[str, Any]:
+        text = self._reply_text(body)
+        return {
+            "id": f"chatcmpl-{uuid.uuid4().hex[:24]}",
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "model": body.get("model", self.model),
+            "choices": [{
+                "index": 0,
+                "message": {"role": "assistant", "content": text},
+                "finish_reason": "stop",
+            }],
+            "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
+        }
+
+    async def chat_stream(self, body: Dict[str, Any]) -> AsyncIterator[str]:
+        cid = f"chatcmpl-{uuid.uuid4().hex[:24]}"
+        model = body.get("model", self.model)
+        created = int(time.time())
+
+        def frame(delta: Dict[str, Any], finish=None) -> str:
+            obj = {
+                "id": cid, "object": "chat.completion.chunk", "created": created,
+                "model": model,
+                "choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
+            }
+            return "data: " + json.dumps(obj, separators=(",", ":"))
+
+        yield frame({"role": "assistant"})
+        for word in self._reply_text(body).split(" "):
+            yield frame({"content": word + " "})
+        yield frame({}, finish="stop")
+        yield "data: [DONE]"
+
+
+def make_backend(kind: str, model: str = "", ollama_url: str = "http://localhost:11434") -> Backend:
+    if kind == "echo":
+        return EchoBackend()
+    if kind == "ollama":
+        return OllamaBackend(base_url=ollama_url, default_model=model)
+    if kind == "mlx":
+        return MlxBackend(model=model)
+    raise ValueError(f"unknown backend: {kind}")