frutebox-node 0.1.0__py3-none-any.whl

frutebox/inference.py

@@ -0,0 +1,624 @@
+"""Real model execution on a provider's hardware — via a local Ollama server.
+
+Every node runs an actual open-weight model (gemma3 / llama / qwen / …) through
+Ollama. **There is no mock.** If the engine is unreachable or the model isn't
+installed, the job fails loudly and the gateway re-routes — a node never returns
+a fabricated answer.
+
+The network's default model alias (``stevai-default``) maps to a concrete
+installed model via ``STEVAI_OLLAMA_MODEL`` (default ``gemma3:4b``).
+
+Every request built here goes through :mod:`frutebox.engine_guard`, which owns the
+engine surface (inference endpoints only — never model management), the model
+allowlist, payload sanitisation and the generation budget. Read that module's
+docstring for exactly what is and is not contained.
+"""
+from __future__ import annotations
+
+import json
+import os
+from typing import AsyncIterator, Dict, List, Optional, Tuple
+
+# The transport itself lives in engine_guard (the single chokepoint); httpx stays
+# imported here because `inference.httpx` is the module-level handle tests patch,
+# and patching it is patching the same module object engine_guard uses.
+import httpx  # noqa: F401
+
+from . import engine_guard
+from .engine_guard import EngineGuardError  # noqa: F401  (re-exported for callers)
+from .envvars import env as envvar
+from . import deltasync
+
+
+def _with_scheme(host: str) -> str:
+    # The Ollama CLI convention is a bare "127.0.0.1:11434" — httpx needs a scheme,
+    # so normalize rather than crash every job with UnsupportedProtocol.
+    h = (host or "").strip()
+    return h if h.startswith(("http://", "https://")) else f"http://{h}"
+
+
+OLLAMA_HOST = _with_scheme(os.environ.get("OLLAMA_HOST",
+                           envvar("OLLAMA_HOST", "http://127.0.0.1:11434")))
+# The concrete model a node runs when the request uses the network's alias.
+DEFAULT_MODEL = envvar("OLLAMA_MODEL", "gemma3:4b")
+_ALIASES = {"", "stevai-default"}
+_TIMEOUT_S = float(envvar("OLLAMA_TIMEOUT_S", "300"))
+
+# Inference engine on THIS node. "ollama" (single-file models) or "llamacpp" (sharded GGUF —
+# e.g. Kimi K2.x/K3, which Ollama cannot serve). A Kimi node sets STEVAI_ENGINE=llamacpp; the agent
+# then talks OpenAI /v1 to a local llama.cpp server instead of Ollama's /api/chat. The call is
+# node-local (127.0.0.1), so there is no RunPod-proxy timeout — a long reasoning turn is fine.
+ENGINE = envvar("ENGINE", "ollama").lower()
+LLAMACPP_HOST = envvar("LLAMACPP_HOST", "http://127.0.0.1:8080")
+# Kimi K2.7-Code reasons for many tokens before answering — give it room and a long local timeout.
+_LLAMACPP_TIMEOUT_S = float(envvar("LLAMACPP_TIMEOUT_S", "2400"))
+_LLAMACPP_MAX_TOKENS = int(envvar("LLAMACPP_MAX_TOKENS", "20000"))
+
+
+def estimate_tokens(text: str) -> int:
+    # ~4 characters per token is a decent rule of thumb; only used as a fallback
+    # when Ollama doesn't report exact counts.
+    return max(1, round(len(text or "") / 4))
+
+
+#: The tags a model uses when it writes its monologue INLINE instead of on its own channel. Lower-case
+#: only: the scan is case-insensitive, and keeping the pairs here means adding a dialect is one line.
+_THINK_TAGS: "Tuple[Tuple[str, str], ...]" = (("<think>", "</think>"), ("<thinking>", "</thinking>"))
+
+#: The longest closing tag, so a streaming splitter knows how many characters it must hold back to avoid
+#: cutting a tag in half across two deltas.
+_LONGEST_TAG = max(len(t) for pair in _THINK_TAGS for t in pair)
+
+
+class ReasoningGate:
+    """The turn's effort budget, applied to a STREAM of reasoning deltas.
+
+    ONE OBJECT RATHER THAN A CHECK AT EACH YIELD. There are three reasoning yield sites per streaming
+    function and two streaming functions, plus the final aggregate that gets stored — six places that
+    would each have to remember the running total. The bug that shape produces is a stream and an
+    aggregate that disagree about what the model said, which is unfalsifiable after the fact. So the
+    gate is the only thing that counts, and `text()` returns EXACTLY what it forwarded: the stored
+    reasoning is the forwarded reasoning, by construction.
+    """
+
+    def __init__(self, effort: str = ""):
+        self.budget = engine_guard.reasoning_char_budget(effort)
+        self.effort = effort
+        self.used = 0
+        self.clamped = False
+        self._out: "List[str]" = []
+
+    def allow(self, delta: str) -> str:
+        """The part of `delta` that may be forwarded — "" once the budget is spent."""
+        if not delta:
+            return ""
+        if self.budget is None:
+            self.used += len(delta)
+            self._out.append(delta)
+            return delta
+        room = self.budget - self.used
+        if room <= 0:
+            self.clamped = True
+            return ""
+        if len(delta) <= room:
+            self.used += len(delta)
+            self._out.append(delta)
+            return delta
+        self.clamped = True
+        self.used = self.budget
+        head = delta[:room]
+        self._out.append(head)
+        return head
+
+    def text(self) -> str:
+        return "".join(self._out)
+
+
+def clamp_reasoning(text: str, effort: str) -> "tuple[str, bool]":
+    """Apply the turn's effort budget to its reasoning. Returns `(text, was_clamped)`.
+
+    WHAT THIS BOUNDS, AND WHAT IT DOES NOT. It bounds the reasoning the caller receives, the
+    governance store keeps and the §6A post-gate has to screen — on every engine, including one that
+    ignores its own level knob. It does NOT reduce what the engine SPENT: measured 2026-10-11, Ollama
+    0.35 accepts `think: "low"` on qwen3:4b and produces byte-identical output to `think: true`, and
+    `options.num_predict` bounds the spend only by truncating the turn before it answers. The saving
+    on a trivial turn comes from ROUTING it to a non-reasoning model (8 tokens against 196, measured),
+    which is the other tooth of the same tier. Claiming this cap saves tokens would be the kind of
+    knob-that-does-nothing it exists to defend against.
+
+    `effort` of "" means no budget was stated — today's behaviour, unchanged.
+    """
+    budget = engine_guard.reasoning_char_budget(effort)
+    if budget is None or len(text) <= budget:
+        return text, False
+    if budget == 0:
+        return "", True
+    # Cut at a boundary rather than mid-word, so a clamped monologue reads as truncated rather than
+    # corrupted. The marker is NOT added: this text goes through the same §6A screen as any other
+    # reasoning, and injecting our own sentence into a channel the model owns would make the screen
+    # judge our words as the model's.
+    cut = text.rfind(" ", 0, budget)
+    return text[:cut if cut > budget // 2 else budget], True
+
+
+def split_inline_reasoning(text: str) -> "Tuple[str, str]":
+    """(content, reasoning) — pull an INLINE monologue out of an answer. THE BACKSTOP.
+
+    Thinking is always on now (`gateway.THINK_ALWAYS`), and a model that honours it puts its monologue on
+    its own channel. Some do not: measured 2026-10-03, qwen3:4b under `think: false` returned 0 characters
+    of `message.thinking` and a 5083-character answer beginning "Okay, so I need to prove..." — the
+    monologue relocated into the text the user reads. This function is what keeps `content` clean when a
+    model does that anyway.
+
+    DELIBERATELY ONLY TAGGED BLOCKS. A monologue with no markers around it is not distinguishable from an
+    answer that happens to begin "Okay, so..." — a heuristic on prose would eventually eat somebody's real
+    first paragraph, which is far worse than leaving one monologue in place. If a model emits an untagged
+    monologue, that is reported, not guessed at.
+
+    An UNCLOSED opening tag means the answer was cut off mid-thought, so everything after it is reasoning
+    and the content is whatever preceded it. Returning the half-thought as the answer would show the user
+    the one thing this exists to hide.
+    """
+    if not text or "<" not in text:
+        return text, ""
+    low = text.lower()
+    content_parts: "List[str]" = []
+    reasoning_parts: "List[str]" = []
+    i = 0
+    while i < len(text):
+        nxt = None
+        for open_tag, close_tag in _THINK_TAGS:
+            at = low.find(open_tag, i)
+            if at != -1 and (nxt is None or at < nxt[0]):
+                nxt = (at, open_tag, close_tag)
+        if nxt is None:
+            content_parts.append(text[i:])
+            break
+        at, open_tag, close_tag = nxt
+        content_parts.append(text[i:at])
+        end = low.find(close_tag, at + len(open_tag))
+        if end == -1:
+            reasoning_parts.append(text[at + len(open_tag):])
+            i = len(text)
+            break
+        reasoning_parts.append(text[at + len(open_tag):end])
+        i = end + len(close_tag)
+    # The content is re-joined and stripped of the whitespace the removed block leaves behind, so an
+    # answer does not arrive with a blank first line for a reason the reader cannot see.
+    return "".join(content_parts).strip(), "".join(reasoning_parts).strip()
+
+
+class InlineReasoningSplitter:
+    """`split_inline_reasoning` for a STREAM, where a tag can straddle two deltas.
+
+    The web app streams, so this is the half that users actually meet. A naive per-delta split would miss
+    `<thi` + `nk>` arriving as two chunks and then leak the whole monologue into the answer — so this holds
+    back up to one tag's worth of trailing text until it knows whether it is the start of one.
+    """
+
+    def __init__(self) -> None:
+        self._buf = ""
+        self._in_think = False
+        self._close = ""
+
+    def feed(self, text: str) -> "Tuple[str, str]":
+        """(content, reasoning) safe to emit NOW. Whatever might still be half a tag is retained."""
+        self._buf += text
+        content, reasoning = self._drain(final=False)
+        return content, reasoning
+
+    def finish(self) -> "Tuple[str, str]":
+        """Flush everything held back. An unclosed block ends up as reasoning, per the rule above."""
+        content, reasoning = self._drain(final=True)
+        return content, reasoning
+
+    def _drain(self, *, final: bool) -> "Tuple[str, str]":
+        out_c: "List[str]" = []
+        out_r: "List[str]" = []
+        while self._buf:
+            low = self._buf.lower()
+            if self._in_think:
+                end = low.find(self._close)
+                if end == -1:
+                    keep = 0 if final else min(len(self._buf), _LONGEST_TAG - 1)
+                    emit = self._buf[:len(self._buf) - keep]
+                    out_r.append(emit)
+                    self._buf = self._buf[len(self._buf) - keep:]
+                    break
+                out_r.append(self._buf[:end])
+                self._buf = self._buf[end + len(self._close):]
+                self._in_think = False
+                continue
+            nxt = None
+            for open_tag, close_tag in _THINK_TAGS:
+                at = low.find(open_tag)
+                if at != -1 and (nxt is None or at < nxt[0]):
+                    nxt = (at, open_tag, close_tag)
+            if nxt is None:
+                keep = 0 if final else min(len(self._buf), _LONGEST_TAG - 1)
+                emit = self._buf[:len(self._buf) - keep]
+                out_c.append(emit)
+                self._buf = self._buf[len(self._buf) - keep:]
+                break
+            at, open_tag, close_tag = nxt
+            out_c.append(self._buf[:at])
+            self._buf = self._buf[at + len(open_tag):]
+            self._in_think, self._close = True, close_tag
+        return "".join(out_c), "".join(out_r)
+
+
+def resolve_model(model: str) -> str:
+    """Map the network alias / catalog id to a concrete installed model, then CLAMP
+    it to what this node advertises.
+
+    Order: network alias → the node's default; catalog id (e.g. "llama-3.2-1b")
+    → its real Ollama tag ("llama3.2:1b"); anything else passes through as-is.
+    Without the catalog mapping Ollama 404s ("model not found") on catalog ids.
+
+    The clamp is the security part: previously ANY string the gateway sent was asked
+    of the engine verbatim, and an unauthenticated engine treats "a model I do not
+    have" as "a model I should go and download". `engine_guard.check_model` raises
+    instead, so the gateway re-plans onto a node that really serves it.
+    """
+    if (model or "") in _ALIASES:
+        return engine_guard.check_model(model, DEFAULT_MODEL)
+    from .manifest import OLLAMA_TAGS
+    return engine_guard.check_model(model, OLLAMA_TAGS.get(model, model))
+
+
+# The small embedding model a node runs for RAG retrieval (§5.2). Separate from the chat
+# model — a tiny model (nomic-embed-text ≈ 275 MB, bge-m3, …) so retrieval is cheap.
+EMBED_MODEL = envvar("EMBED_MODEL", "nomic-embed-text")
+_EMBED_TIMEOUT_S = float(envvar("EMBED_TIMEOUT_S", "60"))
+
+
+async def embed(texts: List[str], model: Optional[str] = None) -> List[List[float]]:
+    """Embed a batch of texts via the local Ollama server's /api/embed endpoint.
+
+    Returns one vector per input, in order. Raises on transport/engine errors so the caller
+    can fall back (e.g. to keyword retrieval) rather than silently returning nothing. Only the
+    Ollama engine serves embeddings here; a llama.cpp chat node has no embed endpoint, so a
+    caller on such a node should fall back."""
+    if not texts:
+        return []
+    # Through the SAME model clamp as chat. This was the one engine call that skipped
+    # it: a caller (or a future signature change that lets a job name the embedder)
+    # could put any string in `model`, and an unauthenticated engine treats a model it
+    # does not have as one to go and download onto the owner's disk.
+    want = model or EMBED_MODEL
+    mdl = engine_guard.check_model(want, want)
+    data = await engine_guard.post_json(
+        OLLAMA_HOST, engine_guard.EMBED_OLLAMA,
+        {"model": mdl, "input": [str(t or "") for t in texts]}, _EMBED_TIMEOUT_S)
+    # /api/embed returns {"embeddings": [[...], ...]}; be tolerant of the older singular field.
+    embs = data.get("embeddings")
+    if embs is None and "embedding" in data:
+        embs = [data["embedding"]]
+    return embs or []
+
+
+async def _run_ollama(
+    model: str, messages: List[Dict], tools: "Optional[List[Dict]]" = None,
+    tool_choice: "Optional[object]" = None, think: "Optional[bool]" = None,
+    think_effort: str = "",
+) -> Dict:
+    real_model = resolve_model(model)
+    # Sanitised, bounded copies — the job's own dicts never reach the engine. This is
+    # what drops unknown message fields (e.g. Ollama's `images`) and scrubs tool schemas.
+    messages = engine_guard.sanitize_messages(messages)
+    choice = engine_guard.sanitize_tool_choice(tool_choice)
+    payload: Dict = {"model": real_model, "messages": messages, "stream": False,
+                     "options": engine_guard.ollama_options()}
+    # Thinking is opt-OUT per turn, decided by the orchestrator's difficulty triage. Ollama
+    # ignores `think` on models that do not reason, so this is safe across a mixed fleet.
+    want_think = engine_guard.sanitize_think(think)
+    if want_think is not None:
+        payload["think"] = want_think
+    # tool_choice "none" means "never call a tool" — enforce it by withholding the tools
+    # entirely (backend-independent), rather than trusting the engine to honor the flag.
+    # For "auto" (the default) we just send the tools; for "required"/a named function we
+    # also forward tool_choice so an engine that honors it (Ollama native support is
+    # partial) can, and engines that ignore it fall back to normal auto behavior.
+    safe_tools = engine_guard.sanitize_tools(tools, tool_choice=choice)
+    if safe_tools:
+        payload["tools"] = safe_tools  # Ollama /api/chat natively supports tool-calling (agentic mode)
+        if choice and choice != "auto":
+            payload["tool_choice"] = choice
+    data = await engine_guard.post_json(
+        OLLAMA_HOST, engine_guard.CHAT_OLLAMA, payload, _TIMEOUT_S)
+    msg = data.get("message", {}) or {}
+    content = engine_guard.OutputMeter().add(msg.get("content", "") or "")
+    # THE THINKING CHANNEL, on the NON-STREAMING path. `stream_ollama` has always read
+    # `message.thinking` and yielded it as a `reasoning` event; this function never looked, so a
+    # non-streaming turn lost the channel at the very first parse — before the node, before the
+    # gateway. MEASURED 2026-10-03 against qwen3:4b: 2979 characters of `message.thinking` arrive
+    # beside a clean 2016-character answer, and every one of them was being discarded here.
+    reasoning = msg.get("thinking") or ""
+    # THE BACKSTOP. Thinking is always on, and a model that honours it puts the monologue on the line
+    # above. One that emits it INLINE instead gets it pulled out here, so `content` is clean either way
+    # and the governance channel gets the thinking rather than losing it. Any already-separate channel
+    # wins the ordering: its text came first.
+    content, inline = split_inline_reasoning(content)
+    if inline:
+        reasoning = f"{reasoning}\n{inline}".strip() if reasoning else inline
+    # THE TURN'S EFFORT BUDGET, applied AFTER the inline backstop on purpose: a model that monologues
+    # inline must not escape the cap by putting its thinking in the wrong channel.
+    reasoning, _clamped = clamp_reasoning(reasoning, engine_guard.sanitize_think_effort(think_effort))
+    tool_calls = msg.get("tool_calls") or []  # present when the model wants to call a tool
+    prompt_tokens = data.get("prompt_eval_count") or estimate_tokens(
+        " ".join(m.get("content", "") for m in messages if isinstance(m.get("content"), str)))
+    completion_tokens = data.get("eval_count") or estimate_tokens(content)
+    return {
+        "content": content,
+        "reasoning": reasoning,
+        "tool_calls": tool_calls,
+        "prompt_tokens": int(prompt_tokens),
+        "tokens": int(completion_tokens),
+        "backend": f"ollama:{real_model}",
+    }
+
+
+async def _run_llamacpp(
+    model: str, messages: List[Dict], tools: "Optional[List[Dict]]" = None,
+    tool_choice: "Optional[object]" = None, think_effort: str = "",
+) -> Dict:
+    """Non-streaming turn against a local llama.cpp server (OpenAI /v1/chat/completions). Used for
+    sharded-GGUF models (Kimi K2.x/K3) that Ollama cannot serve. No mock — raises on engine failure."""
+    real_model = resolve_model(model)
+    messages = engine_guard.sanitize_messages(messages)
+    tc = engine_guard.sanitize_tool_choice(tool_choice)
+    # 03-F2, the COMPUTE half of the saving. `cache_prompt` makes llama.cpp reuse the KV
+    # cache for the shared prefix instead of re-evaluating 19 turns of context on turn 20.
+    # `deltasync` owned this option and nothing called it, so the reuse was never asked
+    # for — the transfer half (delta-sync proper) is still unwired, but this half costs a
+    # single field and is the one that saves GPU time rather than bytes.
+    #
+    # It goes HERE and not in `ollama_options()`: `cache_prompt` is a llama.cpp field, and
+    # Ollama would silently ignore it — wiring it there would look done and do nothing.
+    payload: Dict = {"model": real_model, "messages": messages, "stream": False,
+                     "temperature": 0.0,
+                     **deltasync.engine_cache_options(),
+                     "max_tokens": engine_guard.llamacpp_max_tokens(_LLAMACPP_MAX_TOKENS)}
+    safe_tools = engine_guard.sanitize_tools(tools, tool_choice=tc)
+    if safe_tools:
+        payload["tools"] = safe_tools
+        payload["tool_choice"] = tc if (tc and tc != "auto") else "auto"
+    data = await engine_guard.post_json(
+        LLAMACPP_HOST, engine_guard.CHAT_OPENAI, payload, _LLAMACPP_TIMEOUT_S)
+    first = (data.get("choices") or [{}])[0]
+    msg = first.get("message", {}) or {}
+    content = engine_guard.OutputMeter().add(msg.get("content") or "")
+    # Same omission as the Ollama branch above, same fix. `stream_llamacpp` reads
+    # `delta.reasoning_content`; non-streamed it arrives as `message.reasoning_content`.
+    reasoning = msg.get("reasoning_content") or ""
+    content, inline = split_inline_reasoning(content)   # same backstop as the Ollama branch
+    if inline:
+        reasoning = f"{reasoning}\n{inline}".strip() if reasoning else inline
+    # The turn's effort budget, applied after the inline backstop for the same reason as the Ollama
+    # branch: a model that monologues inline must not escape the cap by using the wrong channel.
+    reasoning, _ = clamp_reasoning(reasoning, engine_guard.sanitize_think_effort(think_effort))
+    tool_calls = msg.get("tool_calls") or []
+    usage = data.get("usage") or {}
+    prompt_tokens = usage.get("prompt_tokens") or estimate_tokens(
+        " ".join(m.get("content", "") for m in messages if isinstance(m.get("content"), str)))
+    completion_tokens = usage.get("completion_tokens") or estimate_tokens(content)
+    return {
+        "content": content, "reasoning": reasoning, "tool_calls": tool_calls,
+        "prompt_tokens": int(prompt_tokens), "tokens": int(completion_tokens),
+        "backend": f"llamacpp:{real_model}",
+    }
+
+
+async def stream_llamacpp(
+    model: str, messages: List[Dict], tools: "Optional[List[Dict]]" = None,
+    tool_choice: "Optional[object]" = None, think: "Optional[bool]" = None,
+    think_effort: str = "",
+) -> "AsyncIterator[Dict]":
+    """Stream a local llama.cpp turn (OpenAI /v1 SSE). Yields the reasoning channel
+    (delta.reasoning_content) then the answer (delta.content), matching stream_ollama's shape."""
+    real_model = resolve_model(model)
+    messages = engine_guard.sanitize_messages(messages)
+    tc = engine_guard.sanitize_tool_choice(tool_choice)
+    # 03-F2, the COMPUTE half of the saving. `cache_prompt` makes llama.cpp reuse the KV
+    # cache for the shared prefix instead of re-evaluating 19 turns of context on turn 20.
+    # `deltasync` owned this option and nothing called it, so the reuse was never asked
+    # for — the transfer half (delta-sync proper) is still unwired, but this half costs a
+    # single field and is the one that saves GPU time rather than bytes.
+    #
+    # It goes HERE and not in `ollama_options()`: `cache_prompt` is a llama.cpp field, and
+    # Ollama would silently ignore it — wiring it there would look done and do nothing.
+    payload: Dict = {"model": real_model, "messages": messages, "stream": True,
+                     "temperature": 0.0,
+                     **deltasync.engine_cache_options(),
+                     "max_tokens": engine_guard.llamacpp_max_tokens(_LLAMACPP_MAX_TOKENS)}
+    safe_tools = engine_guard.sanitize_tools(tools, tool_choice=tc)
+    if safe_tools:
+        payload["tools"] = safe_tools
+        payload["tool_choice"] = tc if (tc and tc != "auto") else "auto"
+    content_parts: List[str] = []
+    _gate = ReasoningGate(engine_guard.sanitize_think_effort(think_effort))
+    splitter = InlineReasoningSplitter()        # same backstop as the Ollama stream
+    tool_calls: List = []
+    # Bounds the WHOLE turn (reasoning included) whatever the engine does with max_tokens.
+    meter = engine_guard.OutputMeter()
+    async with engine_guard.stream(
+            LLAMACPP_HOST, engine_guard.CHAT_OPENAI, payload, _LLAMACPP_TIMEOUT_S) as r:
+        async for line in r.aiter_lines():
+            line = (line or "").strip()
+            if not line.startswith("data:"):
+                continue
+            body = line[5:].strip()
+            if body == "[DONE]":
+                break
+            try:
+                delta = (json.loads(body).get("choices") or [{}])[0].get("delta", {}) or {}
+            except Exception:
+                continue
+            th = delta.get("reasoning_content")
+            if th:
+                meter.add(th)
+                _th = _gate.allow(th)
+                if _th:
+                    yield {"type": "reasoning", "text": _th}
+            c = delta.get("content")
+            if c:
+                meter.add(c)
+                # A tag can straddle two deltas, so route through the splitter rather than per-chunk.
+                c_out, r_out = splitter.feed(c)
+                if r_out:
+                    _ro = _gate.allow(r_out)
+                    if _ro:
+                        yield {"type": "reasoning", "text": _ro}
+                if c_out:
+                    content_parts.append(c_out)
+                    yield {"type": "content", "text": c_out}
+            if delta.get("tool_calls"):
+                tool_calls = delta["tool_calls"]
+    _c, _r = splitter.finish()                  # flush whatever was held back as a possible tag
+    if _r:
+        _rf = _gate.allow(_r)
+        if _rf:
+            yield {"type": "reasoning", "text": _rf}
+    if _c:
+        content_parts.append(_c)
+        yield {"type": "content", "text": _c}
+    content = "".join(content_parts)
+    yield {
+        # FROM THE GATE, not from a second accumulator: what is stored is exactly what was
+        # forwarded. Two lists would be two chances to disagree about what the model said.
+        "type": "done", "content": content, "reasoning": _gate.text(),
+        "tool_calls": tool_calls,
+        "prompt_tokens": estimate_tokens(
+            " ".join(m.get("content", "") for m in messages if isinstance(m.get("content"), str))),
+        "tokens": estimate_tokens(content), "backend": f"llamacpp:{real_model}",
+    }
+
+
+def stream_turn(
+    model: str, messages: List[Dict], tools: "Optional[List[Dict]]" = None,
+    tool_choice: "Optional[object]" = None, think: "Optional[bool]" = None,
+    think_effort: str = "",
+) -> "AsyncIterator[Dict]":
+    """Stream via THIS node's engine — llama.cpp for sharded-GGUF nodes, else Ollama.
+
+    `think_effort` IS FORWARDED, and it is worth saying so in a docstring because for one commit it
+    was not: this function accepted the budget and dropped it, so the whole streaming path — which is
+    what chat uses — ran unbounded while every signature said otherwise. A parameter that is accepted
+    and ignored is the exact defect this budget exists to avoid, and it reached the inside of the
+    feature. Caught by deriving the test's targets from the node's call sites instead of listing them.
+    """
+    if ENGINE == "llamacpp":
+        return stream_llamacpp(model, messages, tools, tool_choice, think,
+                               think_effort=think_effort)
+    return stream_ollama(model, messages, tools, tool_choice, think, think_effort=think_effort)
+
+
+async def stream_ollama(
+    model: str, messages: List[Dict], tools: "Optional[List[Dict]]" = None,
+    tool_choice: "Optional[object]" = None, think: "Optional[bool]" = None,
+    think_effort: str = "",
+) -> "AsyncIterator[Dict]":
+    """Stream a real Ollama /api/chat turn (§5A/§5B foundation). Yields event dicts as
+    tokens arrive — the model's separate REASONING channel first, then the answer:
+        {"type": "reasoning", "text": <delta>}   # Ollama `thinking` field (think=True)
+        {"type": "content",   "text": <delta>}   # answer tokens
+        {"type": "done", "content", "reasoning", "tool_calls", "prompt_tokens",
+                         "tokens", "backend"}      # final aggregate (for metering/history)
+    No mock: streams straight from the engine; raises on failure so the caller re-routes."""
+    real_model = resolve_model(model)
+    messages = engine_guard.sanitize_messages(messages)
+    choice = engine_guard.sanitize_tool_choice(tool_choice)
+    payload: Dict = {"model": real_model, "messages": messages, "stream": True,
+                     "options": engine_guard.ollama_options()}
+    if think is not None:
+        payload["think"] = think  # Ollama 0.30 native reasoning channel
+    safe_tools = engine_guard.sanitize_tools(tools, tool_choice=choice)
+    if safe_tools:
+        payload["tools"] = safe_tools
+        if choice and choice != "auto":
+            payload["tool_choice"] = choice
+    content_parts: List[str] = []
+    _gate = ReasoningGate(engine_guard.sanitize_think_effort(think_effort))
+    tool_calls: List = []
+    prompt_tokens = completion_tokens = 0
+    meter = engine_guard.OutputMeter()
+    splitter = InlineReasoningSplitter()
+    async with engine_guard.stream(
+            OLLAMA_HOST, engine_guard.CHAT_OLLAMA, payload, _TIMEOUT_S) as r:
+        async for line in r.aiter_lines():
+            if not line or not line.strip():
+                continue
+            data = json.loads(line)
+            msg = data.get("message", {}) or {}
+            th = msg.get("thinking")
+            if th:
+                meter.add(th)
+                _th = _gate.allow(th)
+                if _th:
+                    yield {"type": "reasoning", "text": _th}
+            c = msg.get("content")
+            if c:
+                meter.add(c)
+                # THE BACKSTOP, STREAMED. A tag can straddle two deltas, so the splitter holds back up
+                # to one tag's worth of text rather than letting `<thi` + `nk>` through as answer text.
+                # Emitting the monologue as `content` would put it on the user's screen, which is the
+                # whole failure being defended against.
+                c_out, r_out = splitter.feed(c)
+                if r_out:
+                    _ro = _gate.allow(r_out)
+                    if _ro:
+                        yield {"type": "reasoning", "text": _ro}
+                if c_out:
+                    content_parts.append(c_out)
+                    yield {"type": "content", "text": c_out}
+            if msg.get("tool_calls"):
+                tool_calls = msg["tool_calls"]
+            if data.get("done"):
+                prompt_tokens = data.get("prompt_eval_count") or estimate_tokens(
+                    " ".join(m.get("content", "") for m in messages if isinstance(m.get("content"), str)))
+                completion_tokens = data.get("eval_count") or estimate_tokens("".join(content_parts))
+    # FLUSH. The splitter holds back a possible partial tag, so without this the last few characters of
+    # an answer would simply never arrive — and an unclosed block would be left dangling.
+    _c, _r = splitter.finish()
+    if _r:
+        _rf = _gate.allow(_r)
+        if _rf:
+            yield {"type": "reasoning", "text": _rf}
+    if _c:
+        content_parts.append(_c)
+        yield {"type": "content", "text": _c}
+    yield {
+        "type": "done",
+        "content": "".join(content_parts),
+        "reasoning": _gate.text(),
+        "tool_calls": tool_calls,
+        "prompt_tokens": int(prompt_tokens),
+        "tokens": int(completion_tokens),
+        "backend": f"ollama:{real_model}",
+    }
+
+
+async def run_inference(
+    model: str,
+    messages: List[Dict],
+    region: Optional[str] = None,
+    load: float = 0.0,
+    compute_type: str = "gpu",
+    shard: "Optional[Tuple[int, int]]" = None,
+    tools: "Optional[List[Dict]]" = None,
+    tool_choice: "Optional[object]" = None,
+    think: "Optional[bool]" = None,
+    think_effort: str = "",
+) -> Dict:
+    """Run the real model and return {content, tokens, prompt_tokens, backend}.
+
+    In a RAM-pool (shard set), only the lead (shard 0) computes the answer — an
+    Ollama run can't be split across nodes, so non-lead members return
+    participation-only rather than duplicating work or faking output. Raises on
+    any engine failure so the caller re-routes instead of returning a fake."""
+    if shard is not None and shard[0] != 0:
+        return {"content": "", "tool_calls": [], "prompt_tokens": 0, "tokens": 0, "backend": "pool-member"}
+    if ENGINE == "llamacpp":
+        return await _run_llamacpp(model, messages, tools, tool_choice, think_effort=think_effort)
+    return await _run_ollama(model, messages, tools, tool_choice, think,
+                             think_effort=think_effort)