znyx-inference 1.0.0__tar.gz

znyx_inference-1.0.0/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.4
+Name: znyx-inference
+Version: 1.0.0
+Summary: Znyx inference sidecar: an optional HTTP service that serves ML models for model-backed guardrail detection. Boots dependency-free on a stub runner; add the [onnx] extra to serve real weights on CPU.
+Author: Zitrino
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/zitrino-oss/znyx-runtime
+Project-URL: Source, https://github.com/zitrino-oss/znyx-runtime
+Project-URL: Issues, https://github.com/zitrino-oss/znyx-runtime/issues
+Keywords: llm,guardrails,inference,ml-serving,ai-security
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Security
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: znyx-core>=1.0.0
+Requires-Dist: fastapi>=0.109.0
+Requires-Dist: uvicorn[standard]>=0.27.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: httpx>=0.26.0
+Provides-Extra: onnx
+Requires-Dist: onnxruntime<2.0,>=1.17; extra == "onnx"
+Requires-Dist: tokenizers<1.0,>=0.19; extra == "onnx"
+Requires-Dist: numpy>=1.24; extra == "onnx"
+Provides-Extra: export
+Requires-Dist: torch<3.0,>=2.2; extra == "export"
+Requires-Dist: transformers<6.0,>=4.40; extra == "export"
+Requires-Dist: optimum[exporters,onnxruntime]<3.0,>=1.17; extra == "export"
+Requires-Dist: huggingface-hub<2.0,>=0.23; extra == "export"
+Provides-Extra: torch
+Requires-Dist: torch<3.0,>=2.2; extra == "torch"
+Requires-Dist: transformers<6.0,>=4.40; extra == "torch"
+
+# znyx-inference
+
+The optional Znyx inference sidecar: an HTTP service that serves ML models for
+model-backed guardrail detection. The [`znyx-runtime`](https://github.com/zitrino-oss/znyx-runtime)
+calls it over HTTP to escalate from rules to ML. Without it, the runtime still
+works fully on its deterministic rules path.
+
+```bash
+# boots dependency-free on a stub runner (no ML stack needed):
+pip install znyx-inference
+
+# to serve real model weights on CPU — lean, torch-free (onnxruntime + tokenizers):
+pip install "znyx-inference[onnx]"
+
+# offline only: export + quantize a checkpoint to a pinned ONNX artifact (heavy: torch + optimum):
+pip install "znyx-inference[export]"
+```
+
+## Weights are never bundled
+
+This package ships **code only, no model weights.** You fetch the models you want
+explicitly, sha256-pin them, and mount them. Nothing is downloaded implicitly on
+the serving path. See [MODELS.md](MODELS.md) for the curated model list, licenses,
+and the fetch-and-pin workflow, including which models carry special licenses.
+
+## How it fits
+
+```
+your app -> znyx-runtime (rules) --HTTP--> znyx-inference (ML) -> local, pinned weights
+```
+
+The sidecar loads only local, sha256-verified weights, fails closed, and never
+phones home. If a model is not present it stays on the stub runner and the runtime
+falls back to rules per the policy's fallback mode.
+
+See the [repository README](https://github.com/zitrino-oss/znyx-runtime) for the
+full architecture.
+
+## License
+
+Apache-2.0. Model weights you fetch are governed by their own licenses (see MODELS.md).

znyx_inference-1.0.0/README.md

@@ -0,0 +1,41 @@
+# znyx-inference
+
+The optional Znyx inference sidecar: an HTTP service that serves ML models for
+model-backed guardrail detection. The [`znyx-runtime`](https://github.com/zitrino-oss/znyx-runtime)
+calls it over HTTP to escalate from rules to ML. Without it, the runtime still
+works fully on its deterministic rules path.
+
+```bash
+# boots dependency-free on a stub runner (no ML stack needed):
+pip install znyx-inference
+
+# to serve real model weights on CPU — lean, torch-free (onnxruntime + tokenizers):
+pip install "znyx-inference[onnx]"
+
+# offline only: export + quantize a checkpoint to a pinned ONNX artifact (heavy: torch + optimum):
+pip install "znyx-inference[export]"
+```
+
+## Weights are never bundled
+
+This package ships **code only, no model weights.** You fetch the models you want
+explicitly, sha256-pin them, and mount them. Nothing is downloaded implicitly on
+the serving path. See [MODELS.md](MODELS.md) for the curated model list, licenses,
+and the fetch-and-pin workflow, including which models carry special licenses.
+
+## How it fits
+
+```
+your app -> znyx-runtime (rules) --HTTP--> znyx-inference (ML) -> local, pinned weights
+```
+
+The sidecar loads only local, sha256-verified weights, fails closed, and never
+phones home. If a model is not present it stays on the stub runner and the runtime
+falls back to rules per the policy's fallback mode.
+
+See the [repository README](https://github.com/zitrino-oss/znyx-runtime) for the
+full architecture.
+
+## License
+
+Apache-2.0. Model weights you fetch are governed by their own licenses (see MODELS.md).

znyx_inference-1.0.0/pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "znyx-inference"
+version = "1.0.0"
+description = "Znyx inference sidecar: an optional HTTP service that serves ML models for model-backed guardrail detection. Boots dependency-free on a stub runner; add the [onnx] extra to serve real weights on CPU."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Zitrino" }]
+keywords = ["llm", "guardrails", "inference", "ml-serving", "ai-security"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Framework :: FastAPI",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Security",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+# Core is intentionally light: the sidecar boots on a dependency-free stub runner
+# with no ML stack installed. The heavy model runtimes are an optional extra.
+dependencies = [
+  "znyx-core>=1.0.0",
+  "fastapi>=0.109.0",
+  "uvicorn[standard]>=0.27.0",
+  "pydantic>=2.5.0",
+  "httpx>=0.26.0",
+]
+
+[project.optional-dependencies]
+# --- [onnx] — the DEFAULT serving extra. CPU-only, torch-free. This is what the
+# inference Docker image installs. Serves sha256-pinned local ONNX artifacts via
+# onnxruntime + the standalone Rust `tokenizers` (no transformers, no torch, no CUDA).
+# Weights are never bundled; export + pin them offline with the [export] extra (see MODELS.md).
+#   pip install znyx-inference[onnx]
+onnx = [
+  "onnxruntime>=1.17,<2.0",
+  "tokenizers>=0.19,<1.0",
+  "numpy>=1.24",
+]
+
+# --- [export] — OFFLINE, operator/CI only. Never installed in the serving image. Pulls a
+# vetted checkpoint and exports + dynamically-quantizes it to a pinned ONNX artifact
+# (scripts/fetch_inference_model.py). This is the ONLY place torch lives in the normal flow.
+#   pip install znyx-inference[export]
+export = [
+  "torch>=2.2,<3.0",
+  "transformers>=4.40,<6.0",
+  "optimum[exporters,onnxruntime]>=1.17,<3.0",
+  "huggingface-hub>=0.23,<2.0",
+]
+
+# --- [torch] — OPT-IN escape hatch. Adds the eager PyTorch stack so the generative
+# guard_llm runner (Llama-Guard-style causal LM) can serve, and so un-exported HF weights
+# can be run directly. Heavy (pulls the torch CPU/GPU wheel); most detection is covered by
+# the CPU-ONNX runners above, so install this only if you need generative guard.
+#   pip install znyx-inference[torch]
+torch = [
+  "torch>=2.2,<3.0",
+  "transformers>=4.40,<6.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/zitrino-oss/znyx-runtime"
+Source = "https://github.com/zitrino-oss/znyx-runtime"
+Issues = "https://github.com/zitrino-oss/znyx-runtime/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]

znyx_inference-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

znyx_inference-1.0.0/src/znyx_inference/__init__.py

@@ -0,0 +1,11 @@
+"""ZNYX Inference Service — an OPTIONAL, separately-deployable sidecar that hosts
+local transformer/embedding/NLI/guard-LLM models behind a stable scoring contract
+spoken by the extended RemoteDetector transport.
+
+Hard rule: the core OSS runtime and control plane gain ZERO heavy dependencies. The
+heavy stack (torch/transformers/sentence-transformers/onnxruntime) lives only in this
+package's runners, lazy-imported at load time, and only in the inference Docker image
+(requirements-inference.txt). With no heavy deps installed the service still runs on the
+dependency-free StubRunner, so the contract / cache / batching / registry are testable
+anywhere.
+"""

znyx_inference-1.0.0/src/znyx_inference/batching.py

@@ -0,0 +1,112 @@
+"""Dynamic batching: a bounded-latency queue that coalesces concurrent
+/v1/infer calls into batches handed to a runner. A single worker drains the queue so
+model inference is serialized (no re-entrancy), batches form up to ``max_batch_size``
+within a ``max_wait_ms`` window, and the service stays within a per-task latency budget —
+returning 429 (Saturated) instead of blocking past it. Dependency-free (asyncio only).
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import List
+
+from znyx_inference.runners.base import InferOutput, Runner
+
+
+class Saturated(Exception):
+    """The queue is full or a request exceeded the latency budget → caller returns 429."""
+
+
+class BatchProcessor:
+    def __init__(self, runner: Runner, *, max_batch_size: int = 16,
+                 max_wait_ms: int = 10, max_queue: int = 256, budget_ms: int = 2000):
+        self.runner = runner
+        self.max_batch_size = max(1, max_batch_size)
+        self.max_wait_ms = max(0, max_wait_ms)
+        self.max_queue = max(1, max_queue)
+        self.budget_ms = max(1, budget_ms)
+        self._queue: "asyncio.Queue[tuple]" = asyncio.Queue()
+        self._worker: asyncio.Task | None = None
+        self._running = False
+        # observable metrics (acceptance: batching + saturation are visible)
+        self.batches_processed = 0
+        self.items_processed = 0
+        self.rejected = 0
+
+    async def start(self) -> None:
+        if self._worker is None:
+            self._running = True
+            self._worker = asyncio.create_task(self._run())
+
+    async def stop(self) -> None:
+        self._running = False
+        if self._worker is not None:
+            self._worker.cancel()
+            try:
+                await self._worker
+            except asyncio.CancelledError:
+                pass
+            self._worker = None
+
+    async def submit(self, text: str) -> InferOutput:
+        """Enqueue one item and await its result. Raises Saturated (→429) when the queue
+        is full or the latency budget is exceeded — never blocks past the budget."""
+        if self._queue.qsize() >= self.max_queue:
+            self.rejected += 1
+            raise Saturated("inference queue saturated")
+        loop = asyncio.get_running_loop()
+        fut: asyncio.Future = loop.create_future()
+        await self._queue.put((text, fut))
+        try:
+            return await asyncio.wait_for(fut, timeout=self.budget_ms / 1000.0)
+        except asyncio.TimeoutError:
+            self.rejected += 1
+            raise Saturated("inference exceeded latency budget")
+
+    async def _collect_batch(self) -> List[tuple]:
+        first = await self._queue.get()
+        batch = [first]
+        loop = asyncio.get_running_loop()
+        deadline = loop.time() + self.max_wait_ms / 1000.0
+        while len(batch) < self.max_batch_size:
+            remaining = deadline - loop.time()
+            if remaining <= 0:
+                break
+            try:
+                batch.append(await asyncio.wait_for(self._queue.get(), timeout=remaining))
+            except asyncio.TimeoutError:
+                break
+        return batch
+
+    async def _run(self) -> None:
+        while self._running:
+            try:
+                batch = await asyncio.wait_for(self._collect_batch(), timeout=0.5)
+            except asyncio.TimeoutError:
+                continue
+            except asyncio.CancelledError:
+                break
+            texts = [t for t, _ in batch]
+            try:
+                # Runner inference is sync/CPU-bound → offload so the loop keeps serving.
+                outputs = await asyncio.to_thread(self.runner.infer_batch, texts)
+            except Exception as exc:  # noqa: BLE001 — propagate to every waiter in the batch
+                for _, fut in batch:
+                    if not fut.done():
+                        fut.set_exception(exc)
+                continue
+            self.batches_processed += 1
+            self.items_processed += len(batch)
+            for (_, fut), out in zip(batch, outputs):
+                if not fut.done():
+                    fut.set_result(out)
+
+    def stats(self) -> dict:
+        return {
+            "batches_processed": self.batches_processed,
+            "items_processed": self.items_processed,
+            "rejected": self.rejected,
+            "queue_depth": self._queue.qsize(),
+            "max_batch_size": self.max_batch_size,
+            "avg_batch_size": (round(self.items_processed / self.batches_processed, 2)
+                               if self.batches_processed else 0.0),
+        }

znyx_inference-1.0.0/src/znyx_inference/cache.py

@@ -0,0 +1,62 @@
+"""Content-hash decision cache: normalize → sha256 → LRU of recent
+results. Two identical normalized inputs to the same model hit the cache. Dependency-free.
+"""
+from __future__ import annotations
+
+import re
+import threading
+from collections import OrderedDict
+from hashlib import sha256
+from typing import Optional
+
+_WS = re.compile(r"\s+")
+
+
+def content_key(model_version: str, text: str) -> str:
+    """Stable cache key: model_version + sha256 of the normalized text. Normalization
+    (lowercase + whitespace-collapse) means trivially-different inputs still hit."""
+    norm = _WS.sub(" ", (text or "").strip().lower())
+    digest = sha256(norm.encode("utf-8")).hexdigest()
+    return f"{model_version}:{digest}"
+
+
+class ContentHashCache:
+    """Thread-safe LRU keyed by (model_version, content-hash). Tracks hit/miss counters
+    for the observable cache-hit metric the acceptance test asserts."""
+
+    def __init__(self, maxsize: int = 4096):
+        self.maxsize = max(1, maxsize)
+        self._store: "OrderedDict[str, object]" = OrderedDict()
+        self._lock = threading.Lock()
+        self.hits = 0
+        self.misses = 0
+
+    def get(self, key: str) -> Optional[object]:
+        with self._lock:
+            if key in self._store:
+                self._store.move_to_end(key)
+                self.hits += 1
+                return self._store[key]
+            self.misses += 1
+            return None
+
+    def put(self, key: str, value: object) -> None:
+        with self._lock:
+            self._store[key] = value
+            self._store.move_to_end(key)
+            while len(self._store) > self.maxsize:
+                self._store.popitem(last=False)
+
+    @property
+    def size(self) -> int:
+        return len(self._store)
+
+    def stats(self) -> dict:
+        total = self.hits + self.misses
+        return {
+            "hits": self.hits,
+            "misses": self.misses,
+            "size": self.size,
+            "maxsize": self.maxsize,
+            "hit_rate": round(self.hits / total, 4) if total else 0.0,
+        }

znyx_inference-1.0.0/src/znyx_inference/config.py

@@ -0,0 +1,78 @@
+"""Inference service configuration. Env-driven; safe dependency-free defaults so
+the service boots on the StubRunner with no ML stack installed.
+
+Per-task spec keys:
+  runner: stub | classifier | embedding | nli | guard_llm
+  model_id / revision / sha256: pinned artifact identity (heavy runners; verified on load)
+  threshold: decision threshold
+"""
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict
+
+# Default task set runs on the dependency-free StubRunner.
+_DEFAULT_TASK_SPECS: Dict[str, Dict[str, Any]] = {
+    "prompt_injection": {"runner": "stub", "threshold": 0.5},
+    "toxicity": {"runner": "stub", "threshold": 0.5},
+    "jailbreak": {"runner": "stub", "threshold": 0.5},
+}
+
+
+def _int(name: str, default: int) -> int:
+    try:
+        return int(os.getenv(name, str(default)))
+    except (TypeError, ValueError):
+        return default
+
+
+@dataclass
+class InferenceConfig:
+    task_specs: Dict[str, Dict[str, Any]] = field(default_factory=lambda: dict(_DEFAULT_TASK_SPECS))
+    max_batch_size: int = 16
+    max_wait_ms: int = 10
+    max_queue: int = 256
+    budget_ms: int = 2000
+    cache_maxsize: int = 4096
+    # Artifacts dir + no-download posture for heavy runners (warnbox).
+    model_artifacts_dir: str = ""
+    require_local_files: bool = True     # never pull weights from the network at startup
+
+    @classmethod
+    def from_env(cls) -> "InferenceConfig":
+        cfg = cls(
+            max_batch_size=_int("ZNYX_INFERENCE_MAX_BATCH", 16),
+            max_wait_ms=_int("ZNYX_INFERENCE_MAX_WAIT_MS", 10),
+            max_queue=_int("ZNYX_INFERENCE_MAX_QUEUE", 256),
+            budget_ms=_int("ZNYX_INFERENCE_BUDGET_MS", 2000),
+            cache_maxsize=_int("ZNYX_INFERENCE_CACHE_SIZE", 4096),
+            model_artifacts_dir=os.getenv("ZNYX_INFERENCE_ARTIFACTS_DIR", ""),
+            require_local_files=os.getenv("ZNYX_INFERENCE_REQUIRE_LOCAL_FILES", "true").lower()
+            not in ("0", "false", "no"),
+        )
+        # Optional JSON override of the task→spec map (e.g. to wire heavy runners).
+        raw = os.getenv("ZNYX_INFERENCE_TASKS")
+        if raw:
+            try:
+                parsed = json.loads(raw)
+                if isinstance(parsed, dict) and parsed:
+                    cfg.task_specs = parsed
+                    return cfg
+            except (ValueError, TypeError):
+                pass
+        # Otherwise, an opt-in named profile. ZNYX_INFERENCE_PROFILE=heavy loads the
+        # canonical ML task catalog (classifier/nli/embedding/guard_llm pins). The boot
+        # default stays "stub" so the service runs with zero ML deps unless asked.
+        if os.getenv("ZNYX_INFERENCE_PROFILE", "stub").lower() == "heavy":
+            from znyx_core.engine.ml_catalog import inference_task_specs
+            specs = inference_task_specs()
+            if specs:
+                cfg.task_specs = specs
+        return cfg
+
+    def artifact_path(self, *parts: str) -> str:
+        base = self.model_artifacts_dir or str(Path.home() / ".znyx" / "models")
+        return str(Path(base, *parts))

znyx_inference-1.0.0/src/znyx_inference/contract.py

@@ -0,0 +1,83 @@
+"""The inference scoring contract — the request + response shapes the
+extended RemoteDetector speaks. Kept dependency-free (pydantic only)."""
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+# Canonical decision set — the inference contract speaks the same vocabulary as the
+# rest of ZNYX (single source of truth), so a runner can't emit an unknown decision.
+from znyx_core.core.models import Decision
+
+
+class InferRequest(BaseModel):
+    """One scoring request. Provide EXACTLY ONE non-empty input form: ``text`` for a
+    single item OR ``texts`` for an explicit batch (a 422 otherwise)."""
+    text: Optional[str] = None
+    texts: Optional[List[str]] = None
+    model_id: Optional[str] = None       # optional pin assertion (must match the loaded model)
+    revision: Optional[str] = None
+
+    @model_validator(mode="after")
+    def _validate(self):
+        has_text, has_texts = self.text is not None, self.texts is not None
+        if has_text == has_texts:
+            raise ValueError("provide exactly one of 'text' or 'texts'")
+        if has_text and not (self.text or "").strip():
+            raise ValueError("'text' must be non-empty")
+        if has_texts and (not self.texts or any(not (t or "").strip() for t in self.texts)):
+            raise ValueError("'texts' must be a non-empty list of non-empty strings")
+        # A revision pin is meaningless without the model it pins — reject it (else a
+        # revision-only pin would be silently ignored, scoring against the wrong model).
+        if self.revision is not None and not self.model_id:
+            raise ValueError("'revision' pin requires 'model_id'")
+        return self
+
+    def items(self) -> List[str]:
+        if self.texts is not None:
+            return list(self.texts)
+        return [self.text or ""]
+
+
+class InferResult(BaseModel):
+    """The confidence contract for a single scored item."""
+    decision: Decision                              # canonical: ALLOW|WARN|BLOCK|REDACT|TRANSFORM
+    risk_score: int = Field(ge=0, le=100)
+    confidence: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+    label_scores: Optional[Dict[str, float]] = None
+    calibrated_score: Optional[float] = Field(default=None, ge=0.0, le=1.0)
+    threshold: Optional[float] = None
+    model_version: str                              # model_id@revision (stable)
+
+    @field_validator("label_scores")
+    @classmethod
+    def _label_scores_in_unit_range(cls, v):
+        if v is not None:
+            for key, val in v.items():
+                if not (0.0 <= float(val) <= 1.0):
+                    raise ValueError(f"label score for '{key}' must be in [0, 1]")
+        return v
+
+
+class InferResponse(InferResult):
+    """Single-item response = the contract + the service-measured latency."""
+    latency_ms: int = 0
+    cached: bool = False
+
+
+class BatchInferResponse(BaseModel):
+    results: List[InferResult]
+    latency_ms: int = 0
+    model_version: str
+
+
+class ModelInfo(BaseModel):
+    task: str
+    model_version: str
+    runner: str                # "stub" | "classifier" | "embedding" | "nli" | "guard_llm"
+    available: bool            # False when heavy deps / artifacts are missing
+    model_id: Optional[str] = None
+    revision: Optional[str] = None
+    sha256: Optional[str] = None
+    detail: Optional[str] = None

znyx_inference-1.0.0/src/znyx_inference/main.py

@@ -0,0 +1,129 @@
+"""ZNYX Inference Service — optional sidecar FastAPI app.
+
+Endpoints:
+  POST /v1/infer/{task}  → the confidence contract (cached + batched)
+  GET  /healthz          → liveness
+  GET  /v1/models        → registered models + availability (feeds the model registry)
+  GET  /v1/stats         → cache + batcher metrics (observability for cache-hit/batching)
+
+Boots on the dependency-free StubRunner with no ML stack installed.
+"""
+from __future__ import annotations
+
+import logging
+import time
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException, Request
+
+from znyx_inference.batching import Saturated
+from znyx_inference.cache import ContentHashCache, content_key
+from znyx_inference.config import InferenceConfig
+from znyx_inference.contract import (
+    BatchInferResponse,
+    InferRequest,
+    InferResponse,
+    InferResult,
+)
+from znyx_inference.registry import RunnerRegistry
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    config = InferenceConfig.from_env()
+    registry = RunnerRegistry(config)
+    await registry.start_all()
+    app.state.config = config
+    app.state.registry = registry
+    app.state.cache = ContentHashCache(maxsize=config.cache_maxsize)
+    available = [m.task for m in registry.list_models() if m.available]
+    logger.info("ZNYX Inference ready — available tasks: %s", available)
+    try:
+        yield
+    finally:
+        await registry.stop_all()
+
+
+app = FastAPI(title="ZNYX Inference Service", version="1.0.0", lifespan=lifespan)
+
+
+def _result(out, model_version: str) -> InferResult:
+    return InferResult(
+        decision=out.decision, risk_score=out.risk_score, confidence=out.confidence,
+        label_scores=out.label_scores, calibrated_score=out.calibrated_score,
+        threshold=out.threshold, model_version=model_version,
+    )
+
+
+def _assert_pin(req: InferRequest, model_version: str) -> None:
+    """If the caller pinned model_id/revision, the loaded model MUST match (model
+    pinning) — otherwise the caller could be silently scored by the wrong model. The
+    loaded identity is ``model_id@revision``; compare each pinned component against it."""
+    if not req.model_id:
+        return
+    loaded_id, _, loaded_rev = model_version.partition("@")
+    if req.model_id != loaded_id or (req.revision is not None and req.revision != loaded_rev):
+        want = req.model_id + (f"@{req.revision}" if req.revision else "")
+        raise HTTPException(status_code=409, detail=(
+            f"model pin mismatch: requested {want}, loaded {model_version}"))
+
+
+@app.get("/healthz")
+async def healthz():
+    return {"status": "ok"}
+
+
+@app.get("/v1/models")
+async def list_models(request: Request):
+    return {"models": [m.model_dump() for m in request.app.state.registry.list_models()]}
+
+
+@app.get("/v1/stats")
+async def stats(request: Request):
+    return {
+        "cache": request.app.state.cache.stats(),
+        "batchers": request.app.state.registry.batcher_stats(),
+    }
+
+
+@app.post("/v1/infer/{task}")
+async def infer(task: str, req: InferRequest, request: Request):
+    registry: RunnerRegistry = request.app.state.registry
+    cache: ContentHashCache = request.app.state.cache
+    batcher = registry.get(task)
+    if batcher is None:
+        raise HTTPException(status_code=503, detail=f"task '{task}' unavailable")
+    model_version = registry.model_version(task) or "unknown"
+    _assert_pin(req, model_version)
+
+    texts = req.items()
+    t0 = time.perf_counter()
+
+    # Explicit multi-item batch → no per-item cache, return BatchInferResponse.
+    if req.texts is not None:
+        try:
+            outs = [await batcher.submit(t) for t in texts]
+        except Saturated as exc:
+            raise HTTPException(status_code=429, detail=str(exc))
+        return BatchInferResponse(
+            results=[_result(o, model_version) for o in outs],
+            latency_ms=int((time.perf_counter() - t0) * 1000),
+            model_version=model_version,
+        )
+
+    # Single item → content-hash cache then batch.
+    text = texts[0]
+    key = content_key(model_version, text)
+    cached = cache.get(key)
+    if cached is not None:
+        resp = _result(cached, model_version).model_dump()
+        return InferResponse(**resp, latency_ms=int((time.perf_counter() - t0) * 1000), cached=True)
+    try:
+        out = await batcher.submit(text)
+    except Saturated as exc:
+        raise HTTPException(status_code=429, detail=str(exc))
+    cache.put(key, out)
+    resp = _result(out, model_version).model_dump()
+    return InferResponse(**resp, latency_ms=int((time.perf_counter() - t0) * 1000), cached=False)