caudate-cli 0.1.0__py3-none-any.whl

api/caudate_middleware.py

@@ -0,0 +1,618 @@
+"""CaudateMiddleware — gives Caudate her full body inside /v1/messages.
+
+When Claude Code (or any Anthropic-format client) hits Cognos via
+the compat endpoint, this middleware sits between the request and the
+underlying LLM and engages every Caudate capability:
+
+  1. **State capture**   — recent message text + extracted images + mood
+  2. **Prediction**       — tool / tier / think / value head outputs
+  3. **Vocab growth**     — registers any new Claude-Code tools
+  4. **Hint injection**   — at WHISPER+, prepends Caudate's hint to system
+  5. **Tier override**    — at ADVISOR+, her prediction wins (already wired
+                            via Router.set_caudate; we just call into it)
+  6. **Thinking gate**    — at CONTROLLER+, modulates request shape
+  7. **Reward derivation**— response shape → heuristic reward
+  8. **Episode storage**  — every call lands in episodic memory
+  9. **Replay + autotrain** — observer accumulates samples, retrains in BG
+ 10. **Cleanup**          — temp images deleted after the call
+
+Per-request hot path keeps the latency cost small; the heavy stuff
+(vision encode, episodic write) happens off the critical path where
+possible.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import logging
+import os
+import re
+import tempfile
+import time
+import uuid
+from contextlib import contextmanager
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+# Hard cap on how many images we extract per request. CLIP/InternVL2
+# are GPU-bound; running 50 images would stall a turn.
+_MAX_IMAGES_PER_TURN = 4
+
+
+class CaudateMiddleware:
+    """Wraps an LLM call with Caudate's full prediction/learning loop."""
+
+    def __init__(self, agent: Any):
+        self.agent = agent
+        self.caudate = getattr(agent, "caudate", None)
+
+    # ------------------------------------------------------------------
+    # Public hooks called from anthropic_compat
+    # ------------------------------------------------------------------
+
+    def has_caudate(self) -> bool:
+        return self.caudate is not None
+
+    def begin_turn(
+        self,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None,
+        model_source: str = "unknown",
+    ) -> "_TurnContext":
+        """Open a turn-context that will be passed back at end_turn().
+
+        - Builds the textual state Caudate needs.
+        - Materializes any base64 images to temp files for vision encoding.
+        - Calls caudate.on_turn_start to get a Prediction.
+        - Registers Claude Code's tool names in Caudate's vocab.
+        - Tags the resulting sample with `model_source` so future
+          architectural phases can branch per teacher model
+          (CAUDATE_EVOLUTION.md, Phase 1).
+        """
+        ctx = _TurnContext(self.agent, messages)
+        ctx.model_source = model_source
+        if self.caudate is None:
+            return ctx
+
+        # Grow vocab with whatever tools Claude Code passed
+        if tools:
+            for t in tools:
+                fn = t.get("function") or t   # accept either shape
+                name = fn.get("name") or t.get("name")
+                if name:
+                    self.caudate.replay  # touch to keep it alive
+                    self.caudate.scorer  # ditto
+                    # ToolVocab.add is idempotent
+                    try: self.caudate.advisor.vocab.add(name)
+                    except Exception: pass
+
+        # Build the textual state from recent messages
+        recent_text = _flatten_messages_to_text(messages, limit=self.caudate.cfg.msg_window)
+
+        # Extract images
+        image_paths = _extract_images_to_temp(messages, _MAX_IMAGES_PER_TURN)
+        ctx.temp_image_paths = image_paths
+
+        # Mood
+        mood = _read_mood(self.agent)
+
+        # Predict (logs prediction internally, also caches as last_prediction)
+        try:
+            pred = self.caudate.on_turn_start(
+                recent_messages=recent_text,
+                mood=mood,
+                image_paths=image_paths,
+                model_source=model_source,
+            )
+            ctx.prediction = pred
+        except Exception as e:
+            logger.debug(f"Caudate.on_turn_start failed: {e}")
+
+        return ctx
+
+    def maybe_inject_hint(
+        self,
+        messages: list[dict[str, Any]],
+        ctx: "_TurnContext",
+    ) -> list[dict[str, Any]]:
+        """At WHISPER+, prepend Caudate's hint to the system message."""
+        if self.caudate is None or ctx.prediction is None:
+            return messages
+        if not self.caudate.can_whisper():
+            return messages
+        if ctx.prediction.tool_confidence < self.caudate.cfg.advisor_min_confidence:
+            return messages
+
+        hint = _build_hint_block(ctx.prediction, self.caudate.policy.level.label)
+        # Find or create system message
+        out = list(messages)
+        if out and out[0].get("role") == "system":
+            head = out[0].get("content", "")
+            if isinstance(head, list):
+                head = " ".join(b.get("text", "") for b in head if isinstance(b, dict))
+            out[0] = {"role": "system", "content": f"{head}\n\n{hint}"}
+        else:
+            out = [{"role": "system", "content": hint}, *out]
+        return out
+
+    def maybe_disable_thinking(self, ctx: "_TurnContext") -> bool:
+        """CONTROLLER+: returns True if thinking should be DISABLED for this call."""
+        if self.caudate is None or ctx.prediction is None:
+            return False
+        if not self.caudate.can_control():
+            return False
+        return ctx.prediction.think < 0.3
+
+    def observe_response_text(self, ctx: "_TurnContext", text: str) -> None:
+        """Record the visible text of the assistant reply (for episode)."""
+        ctx.response_text = (ctx.response_text or "") + (text or "")
+
+    def observe_thinking(self, ctx: "_TurnContext", text: str) -> None:
+        """Record reasoning-channel text. Thinking is valuable signal for
+        tool inference even if it never reaches visible content."""
+        ctx.thinking_text = (ctx.thinking_text or "") + (text or "")
+
+    def observe_tool_use(self, ctx: "_TurnContext", tool_name: str) -> None:
+        """One tool call emitted by the LLM."""
+        if self.caudate is None:
+            return
+        try:
+            self.caudate.on_tool_use(tool_name)
+        except Exception as e:
+            logger.debug(f"Caudate.on_tool_use failed: {e}")
+        ctx.tools_used.append(tool_name)
+
+    def observe_arbitration(
+        self,
+        ctx: "_TurnContext",
+        fast_text: str,
+        slow_text: str,
+        fast_score: float,
+        slow_score: float,
+        winner: str,
+        fast_model: str = "",
+        slow_model: str = "",
+    ) -> None:
+        """Record a dual-brain arbitration event on this turn.
+
+        Both drafts + their heuristic scores are stored on the turn
+        context; end_turn writes them into the episodic record so
+        later we can train a preference head on (state, draft_a,
+        draft_b, picked) tuples. This is the data substrate for
+        Phase 4 of CAUDATE_EVOLUTION.md (the conductor).
+        """
+        ctx.arbitration = {
+            "fast": {"text": fast_text[:2000], "score": fast_score, "model": fast_model},
+            "slow": {"text": slow_text[:2000], "score": slow_score, "model": slow_model},
+            "winner": winner,
+        }
+        try:
+            logger.info(
+                f"[arbitration] fast({fast_model})={fast_score:.2f} "
+                f"slow({slow_model})={slow_score:.2f}  →  {winner}"
+            )
+        except Exception:
+            pass
+
+    def end_turn(self, ctx: "_TurnContext", error: bool = False) -> None:
+        """Close the turn — derive reward, push samples, write episode.
+
+        Special case: if the LLM didn't actually call a tool but its
+        text indicates an intended tool ("I'll use Bash to..."), record
+        the *intended* tool as the target with a low reward. This way
+        Caudate learns the corrective action ("for this prompt, the
+        right tool was Bash even though gemma4 stalled") rather than
+        mimicking the failure mode.
+        """
+        try:
+            # Stalling rescue: infer the *intended* tool from three
+            # signals, in priority order:
+            #   1. visible response text (model said "I'll use Bash")
+            #   2. thinking-channel text (model reasoned "use Bash")
+            #   3. user's prompt itself (user asked "what's in folder?"
+            #      which clearly maps to Bash regardless of model's reply)
+            # The user-prompt fallback catches cases where the model
+            # refused entirely ("I can't see your files") — Caudate
+            # should still learn that the right tool was Bash, with a
+            # low reward signaling the model failed.
+            inferred_tool: str | None = None
+            if (not ctx.tools_used
+                    and self.caudate is not None
+                    and self.caudate._pending is not None):
+                combined_text = (
+                    (ctx.response_text or "") + "\n"
+                    + (ctx.thinking_text or "")
+                )
+                if combined_text.strip():
+                    inferred_tool = _infer_intended_tool(combined_text)
+                if inferred_tool is None:
+                    user_text = _last_user_text(ctx.messages)
+                    if user_text:
+                        inferred_tool = _infer_intended_tool(user_text)
+                if inferred_tool:
+                    # Inject into observer's pending state so the sample
+                    # is built with this target instead of '<none>'.
+                    self.caudate._pending.chosen_tools.append(inferred_tool)
+                    ctx.tools_used.append(inferred_tool)
+                    ctx.inferred_from_stall = True
+
+            reward = _derive_reward(ctx, error)
+            if self.caudate is not None:
+                # Pass turn-outcome signals so observer can label the
+                # Tier 1 / Tier 2 heads (refusal, code, stall, latency,
+                # difficulty, etc.) from what actually happened.
+                elapsed_s = max(0.0, time.time() - ctx.started_at)
+                self.caudate.on_turn_end(
+                    reward=reward,
+                    response_text=ctx.response_text or "",
+                    inferred_from_stall=getattr(ctx, "inferred_from_stall", False),
+                    elapsed_s=elapsed_s,
+                    completion_tokens=getattr(ctx, "completion_tokens", None),
+                )
+            self._write_episode(ctx, reward)
+
+            if inferred_tool:
+                logger.info(
+                    f"Caudate stall-rescue: inferred tool={inferred_tool!r} "
+                    f"from stalling text, reward={reward:.2f}"
+                )
+        except Exception as e:
+            logger.debug(f"Caudate.end_turn failed: {e}")
+        finally:
+            for p in ctx.temp_image_paths:
+                try: os.unlink(p)
+                except Exception: pass
+
+    # ------------------------------------------------------------------
+
+    def _write_episode(self, ctx: "_TurnContext", reward: float) -> None:
+        """Land each turn in episodic memory so the meta-learner can use it
+        and the consolidator picks it up at training time."""
+        episodic = getattr(getattr(self.agent, "loop", None), "episodic", None)
+        if episodic is None:
+            return
+        try:
+            from core.schemas import Episode, ToolResult, ToolResultStatus
+            user_text = _last_user_text(ctx.messages)[:400]
+            for tool in ctx.tools_used or ["<reply>"]:
+                ep = Episode(
+                    goal_id="claude-code",
+                    task_id=str(uuid.uuid4()),
+                    action=user_text or "(no user text)",
+                    tool_name=(tool if tool != "<reply>" else None),
+                    tool_args={},
+                    result=ToolResult(
+                        tool_name=(tool if tool != "<reply>" else "Respond"),
+                        status=(ToolResultStatus.SUCCESS if reward > 0.5
+                                else ToolResultStatus.ERROR),
+                        output=(ctx.response_text or "")[:400],
+                    ),
+                )
+                episodic.store(ep)
+        except Exception as e:
+            logger.debug(f"episode write failed: {e}")
+
+        # Dual-brain arbitration record — append to a dedicated JSONL
+        # so a future preference-learning trainer can pick it up. We
+        # keep it separate from the normal replay buffer because the
+        # schema is different (carries TWO drafts + scores + winner,
+        # not a single target).
+        if ctx.arbitration is not None:
+            try:
+                import json as _json, time as _time
+                from pathlib import Path as _P
+                p = _P("data/nn/arbitrations.jsonl")
+                p.parent.mkdir(parents=True, exist_ok=True)
+                with p.open("a") as f:
+                    f.write(_json.dumps({
+                        "ts": _time.time(),
+                        "user_text": _last_user_text(ctx.messages)[:600],
+                        "model_source": ctx.model_source,
+                        "reward": reward,
+                        "arbitration": ctx.arbitration,
+                    }) + "\n")
+            except Exception as e:
+                logger.debug(f"arbitration write failed: {e}")
+
+
+# ---- TurnContext ---------------------------------------------------
+
+
+class _TurnContext:
+    """Lives across a single /v1/messages call. Carries Caudate's
+    prediction and the bookkeeping needed to score it at the end."""
+
+    def __init__(self, agent: Any, messages: list[dict[str, Any]]):
+        self.agent = agent
+        self.messages = messages
+        self.prediction = None
+        self.temp_image_paths: list[str] = []
+        self.tools_used: list[str] = []
+        self.response_text: str = ""
+        # Thinking-channel text (gemma4 / kimi reasoning). Counted as
+        # signal for tool inference but NOT as the visible reply.
+        self.thinking_text: str = ""
+        self.started_at: float = time.time()
+        # True when the LLM stalled (text-only, "I will use X..." but no
+        # actual tool_use emitted) and we inferred the intended tool from
+        # its text. This flag tunes the reward.
+        self.inferred_from_stall: bool = False
+        # Phase 1 of evolution roadmap: which model produced this turn's
+        # response. Set by begin_turn(); flows into ConversationSample.
+        self.model_source: str = "unknown"
+        # Dual-brain arbitration record (Phase 4 / pattern 2). When the
+        # arbiter ran, this captures both drafts + scores so end_turn
+        # can log them for preference-learning training data.
+        # Shape: {"fast": {text, score}, "slow": {text, score},
+        #         "winner": "fast"|"slow"}
+        self.arbitration: dict[str, Any] | None = None
+
+
+# ---- Helpers --------------------------------------------------------
+
+
+def _flatten_messages_to_text(messages: list[dict[str, Any]], limit: int) -> list[str]:
+    """Render the recent conversation as plain text for Caudate's encoder.
+
+    Important: do NOT silently drop tool_calls / tool_use blocks. They
+    carry the structured intent of the previous turn (e.g. the question
+    that AskUserQuestion was asking). Without them, Caudate sees only
+    the user's picked answer with no idea what was being asked.
+    """
+    import json as _json
+    out: list[str] = []
+    for m in messages[-limit:]:
+        role = m.get("role", "?")
+        c = m.get("content")
+        chunks: list[str] = []
+        if isinstance(c, list):
+            for b in c:
+                if not isinstance(b, dict):
+                    continue
+                btype = b.get("type")
+                if btype == "text":
+                    chunks.append(b.get("text", ""))
+                elif btype == "image_url":
+                    chunks.append("[image]")
+                elif btype == "tool_use":
+                    name = b.get("name", "")
+                    inp = b.get("input") or {}
+                    try:
+                        inp_text = _json.dumps(inp, ensure_ascii=False)[:400]
+                    except Exception:
+                        inp_text = str(inp)[:400]
+                    chunks.append(f"[tool_use {name}({inp_text})]")
+                elif btype == "tool_result":
+                    rc = b.get("content")
+                    if isinstance(rc, list):
+                        rc = " ".join(
+                            x.get("text", "") for x in rc
+                            if isinstance(x, dict) and x.get("type") == "text"
+                        )
+                    chunks.append(f"[tool_result {str(rc)[:200]}]")
+                elif btype == "thinking":
+                    chunks.append(f"[thinking {b.get('thinking', '')[:200]}]")
+            text = " ".join(p for p in chunks if p)
+        else:
+            text = str(c) if c else ""
+        # OpenAI-shape assistant messages put tool calls in a sibling
+        # field (m["tool_calls"]) rather than inside content blocks.
+        # Render them so Caudate sees the structured intent.
+        tcs = m.get("tool_calls") or []
+        for tc in tcs:
+            fn = tc.get("function") or {}
+            name = fn.get("name", "")
+            args = fn.get("arguments") or ""
+            text = f"{text} [tool_call {name}({str(args)[:400]})]".strip()
+        if text:
+            out.append(f"{role}: {text[:400]}")
+    return out
+
+
+def _extract_images_to_temp(
+    messages: list[dict[str, Any]], cap: int,
+) -> list[str]:
+    """Pull base64 images out of message content blocks, write to temp PNGs.
+
+    Caudate's vision encoder takes file paths, not raw bytes. So we
+    materialize any image_url data: URLs to short-lived tempfiles, return
+    paths. Caller deletes them after the turn.
+    """
+    paths: list[str] = []
+    for m in messages:
+        c = m.get("content")
+        if not isinstance(c, list):
+            continue
+        for b in c:
+            if not isinstance(b, dict):
+                continue
+            if len(paths) >= cap:
+                return paths
+            url = None
+            if b.get("type") == "image_url":
+                url = (b.get("image_url") or {}).get("url")
+            elif b.get("type") == "image":
+                src = b.get("source") or {}
+                if src.get("type") == "base64":
+                    media = src.get("media_type", "image/png").split("/")[-1]
+                    data = src.get("data", "")
+                    try:
+                        raw = base64.b64decode(data)
+                    except Exception:
+                        continue
+                    fd, p = tempfile.mkstemp(suffix=f".{media}", prefix="caudate-img-")
+                    try:
+                        os.write(fd, raw); paths.append(p)
+                    finally:
+                        os.close(fd)
+                    continue
+            if url and url.startswith("data:"):
+                try:
+                    head, _, b64 = url.partition(",")
+                    media = head.split(";")[0].split("/")[-1] or "png"
+                    raw = base64.b64decode(b64)
+                except Exception:
+                    continue
+                fd, p = tempfile.mkstemp(suffix=f".{media}", prefix="caudate-img-")
+                try:
+                    os.write(fd, raw); paths.append(p)
+                finally:
+                    os.close(fd)
+    return paths
+
+
+def _read_mood(agent: Any) -> list[float]:
+    """Read 4 mood floats off the personality engine if available."""
+    p = getattr(agent, "personality", None)
+    if p is None:
+        return [0.5, 0.5, 0.5, 0.5]
+    try:
+        m = p.mood
+        # MoodState fields: confidence, curiosity, frustration, satisfaction
+        return [
+            float(getattr(m, "confidence", 0.5)),
+            float(getattr(m, "curiosity", 0.5)),
+            float(getattr(m, "frustration", 0.5)),
+            float(getattr(m, "satisfaction", 0.5)),
+        ]
+    except Exception:
+        return [0.5, 0.5, 0.5, 0.5]
+
+
+def _last_user_text(messages: list[dict[str, Any]]) -> str:
+    for m in reversed(messages):
+        if m.get("role") == "user":
+            c = m.get("content")
+            if isinstance(c, str):
+                return c
+            if isinstance(c, list):
+                for b in c:
+                    if isinstance(b, dict) and b.get("type") == "text":
+                        return b.get("text", "")
+    return ""
+
+
+def _build_hint_block(pred: Any, level: str) -> str:
+    if level == "controller":
+        preface = "## Caudate (your trained action-selection net) recommends:"
+    elif level == "advisor":
+        preface = "## Caudate (advisor) suggests, based on prior sessions:"
+    else:
+        preface = "## Caudate (whispering — still learning) thinks you might want:"
+    return (
+        f"{preface}\n"
+        f"  next tool:  {pred.tool}      (confidence {pred.tool_confidence:.2f})\n"
+        f"  routing:    {pred.tier}      (confidence {pred.tier_confidence:.2f})\n"
+        f"  thinking:   {'helpful' if pred.think >= 0.5 else 'not needed'} "
+        f"(p={pred.think:.2f})\n"
+        f"  expected reward: {pred.value:.2f}\n"
+        f"You may disagree — Caudate ({level}) hasn't seen this exact context, "
+        f"only patterns from past turns."
+    )
+
+
+# ---- Stalling detection + intended-tool inference ------------------
+# These power Caudate's "rescue" learning: when an LLM (gemma4 in
+# particular) generates intent without action, we extract the action it
+# *should* have taken so Caudate's next prediction is corrective rather
+# than mimetic.
+
+# Regex that catches "I will / I'll / Let me / I should / I need to / I
+# am going to / first I" preambles followed by an action verb. Matches
+# only at the START of a sentence so we don't false-positive on
+# "I would not..." or "I'll be honest" (those end without action plans).
+_STALLING_PHRASES = re.compile(
+    r"\b(?:I\s+will|I'll|let\s+me|I\s+should|I\s+need\s+to|"
+    r"I\s+am\s+going\s+to|first(?:,)?\s+I'?l?l?|"
+    r"I'll\s+start\s+by|I\s+need\s+to)\s+",
+    re.IGNORECASE,
+)
+
+# Mapping: text fragments → Cognos tool name (case-insensitive).
+# Cognos tools: Bash, Read, Write, Edit, Glob, Grep, WebSearch,
+# WebFetch, PythonExec, Think, Respond, Agent, Draw.
+_TOOL_HINTS: list[tuple[re.Pattern, str]] = [
+    (re.compile(r"\b(?:use\s+)?bash\b|`?ls\b|`?cat\b|shell\s+command",        re.I), "Bash"),
+    (re.compile(r"\b(?:use\s+)?glob\b|find\s+files?\s+match|file\s+pattern",  re.I), "Glob"),
+    (re.compile(r"\b(?:use\s+)?grep\b|search\s+(?:for|the|in)|search\s+code", re.I), "Grep"),
+    (re.compile(r"\b(?:use\s+)?read\b|read\s+(?:the\s+)?file|inspect\s+file", re.I), "Read"),
+    (re.compile(r"\b(?:use\s+)?write\b|write\s+(?:to|a)\s+file|create\s+file", re.I), "Write"),
+    (re.compile(r"\b(?:use\s+)?edit\b|modify\s+(?:the\s+)?file|change\s+code", re.I), "Edit"),
+    (re.compile(r"\b(?:use\s+)?websearch\b|search\s+(?:the\s+)?web|google",   re.I), "WebSearch"),
+    (re.compile(r"\b(?:use\s+)?webfetch\b|fetch\s+(?:the\s+)?(?:url|page)",   re.I), "WebFetch"),
+    (re.compile(r"\b(?:use\s+)?python(?:exec)?\b|run\s+python|execute\s+python", re.I), "PythonExec"),
+    (re.compile(r"\b(?:use\s+)?(?:the\s+)?think\b|think\s+(?:about|step)",    re.I), "Think"),
+    (re.compile(r"\b(?:use\s+)?(?:agent|subagent)\b|spawn\s+(?:an?\s+)?agent", re.I), "Agent"),
+    (re.compile(r"\b(?:use\s+)?draw(?:ing)?\b|generate\s+(?:an?\s+)?(?:image|picture|diagram)|create\s+(?:an?\s+)?(?:image|picture)", re.I), "Draw"),
+    # Generic exploratory prompts → Bash. These catch user-side language
+    # that maps clearly to a filesystem-inspection turn even when the
+    # model didn't suggest a tool.
+    (re.compile(r"list\s+(?:the\s+)?files?|"
+                r"what'?s\s+in\s+(?:\w+\s+)?(?:dir|folder|directory)|"
+                r"what\s+is\s+in\s+(?:\w+\s+)?(?:dir|folder|directory)|"
+                r"show\s+(?:me\s+)?(?:the\s+)?files?|"
+                r"(?:check|see)\s+(?:the\s+)?(?:dir|folder|directory|files?)|"
+                r"(?:what'?s|what\s+is|tell\s+me\s+what'?s|tell\s+me\s+what\s+is)\s+"
+                r"(?:working|happening|going\s+on|here)\s+"
+                r"(?:in\s+(?:\w+\s+)?(?:dir|folder|directory|repo|project)|here)", re.I), "Bash"),
+]
+
+
+def _is_stalling(text: str) -> bool:
+    """Did the model say 'I will X' without actually emitting a tool call?"""
+    return bool(text) and bool(_STALLING_PHRASES.search(text))
+
+
+def _infer_intended_tool(text: str) -> str | None:
+    """If the model's text describes a tool action — whether stalling
+    ("I will use Bash"), refusing ("I can't run code; try `ls`"), or
+    advising ("you should grep for that") — return the matching Cognos
+    tool name. Returns None if no clear intent.
+
+    Used to *rescue* Caudate's training data: instead of recording
+    target_tool='<none>' whenever the model didn't call a tool, we
+    record the tool that the response *describes*, so Caudate learns
+    the corrective action rather than the failure pattern.
+    """
+    if not text:
+        return None
+    for pattern, tool in _TOOL_HINTS:
+        if pattern.search(text):
+            return tool
+    return None
+
+
+def _derive_reward(ctx: _TurnContext, error: bool) -> float:
+    """Reward signal for Caudate's training.
+
+      - upstream LLM error                    → 0.20  (clearly bad)
+      - real tool call emitted (not inferred) → 0.70  (engaged)
+      - inferred tool from stalling text      → 0.25  (model failed
+                                                       — rescue training)
+      - text-only reply, no stalling          → 0.60  (legitimate answer)
+      - empty reply                           → 0.40  (gave up)
+
+    The 0.25 reward on stalling-with-inference is the key change: it
+    teaches Caudate that the predicted action was correct (so she should
+    predict it again next time) but the *outcome* was bad (so the LLM
+    needs the hint). High enough that her tool-prediction head learns
+    the right answer; low enough that her value-prediction head learns
+    "this turn went poorly."
+    """
+    if error:
+        return 0.20
+    if ctx.inferred_from_stall:
+        return 0.25
+    if ctx.tools_used:
+        return 0.70
+    text = (ctx.response_text or "")
+    if _is_stalling(text):
+        # Stalling text but no inference matched (vague intent like
+        # "I'll think about it"). Penalize but no rescue available.
+        return 0.30
+    if text.strip():
+        return 0.60
+    return 0.40