premanmcp 1.1.14 → 1.1.16

package/bin/eval_harness.py

@@ -309,11 +309,19 @@ def _turn_raw(ref, raw_result):
 
     Carried on ``raw`` because assert-ai stores it verbatim and owns every other
     field of the artifact; the backend reads it back per turn
-    (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §7.2).
+    (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §7.2). A
+    proxied agent's token counts become model-call nodes here, as spans would
+    (docs/superpowers/specs/2026-10-08-proxy-tokens-ts-design.md §5).
     """
     raw = {"callable": ref}
     capture = raw_result.get("tool_capture")
     if isinstance(capture, dict):
+        try:
+            built = _proxy_nodes(capture)
+            if built is not None:
+                _add_nodes(raw, *built)
+        except Exception:
+            capture.pop("model_calls", None)
         raw["tool_capture"] = _session._sanitize_endpoint_value(capture)
     return raw
 
@@ -334,6 +342,79 @@ def _attach_capture(result, payload):
             return
 
 
+def _span_tokens_in(events) -> bool:
+    return any(isinstance(e, dict) and isinstance((e.get("raw") or {}).get("_tokens"), dict) for e in events or [])
+
+
+def _proxy_nodes(capture, spans_have_tokens=False):
+    """The proxy's token records for one turn, as the model-call nodes spans would give.
+
+    Pops ``model_calls`` so the stored capture never carries them. All or
+    nothing: one call without counts leaves the turn with none and
+    ``tokens_missing`` saying why, because a partial sum reads as a complete
+    number. Returns (nodes, totals) or None.
+    """
+    if not isinstance(capture, dict):
+        return None
+    calls = capture.pop("model_calls", None)
+    if not calls or spans_have_tokens:
+        return None
+    if not isinstance(calls, list) or not all(isinstance(c, dict) for c in calls):
+        capture["tokens_missing"] = "a model call's token counts could not be read"
+        return None
+    gap = next((c for c in calls if not (isinstance(c.get("input"), int) and isinstance(c.get("output"), int))), None)
+    if gap is not None:
+        capture["tokens_missing"] = gap.get("missing") or "a model call's token counts could not be read"
+        return None
+    nodes = [
+        {
+            "view": ["target", "combined"],
+            "actor": "target",
+            "edit": {"type": "add_message", "message": {"role": "assistant", "content": ""}},
+            "raw": {
+                "_node": c.get("kind") or "model",
+                "_model": c.get("model") or "",
+                "_tokens": {"input": c["input"], "output": c["output"]},
+                "_latency_ms": c.get("ms"),
+                "_source": "proxy",
+            },
+        }
+        for c in calls
+    ]
+    totals = {
+        "total_tokens": {"input": sum(c["input"] for c in calls), "output": sum(c["output"] for c in calls)},
+        "total_latency_ms": round(sum(c["ms"] for c in calls if isinstance(c.get("ms"), (int, float))), 1),
+        "llm_call_count": len(calls),
+    }
+    return nodes, totals
+
+
+def _add_nodes(raw, nodes, totals):
+    raw["trace_events"] = list(raw.get("trace_events") or []) + nodes
+    meta = dict(raw.get("trace_metadata") or {"nodes_visited": [], "tools_called": []})
+    meta.update(totals)
+    raw["trace_metadata"] = meta
+
+
+def _attach_proxy_nodes(result, payload):
+    """Token nodes for a traced turn whose spans carried no tokens (a proxied agent)."""
+    capture = payload.get("tool_capture") if isinstance(payload, dict) else None
+    if not isinstance(capture, dict):
+        return
+    try:
+        raw = result.raw if isinstance(result.raw, dict) else None
+        built = _proxy_nodes(capture, spans_have_tokens=_span_tokens_in((raw or {}).get("trace_events")))
+        if built is None or raw is None:
+            return
+        _add_nodes(raw, *built)
+        for message in reversed(result.interaction_messages):
+            if message.get("role") == "assistant" and isinstance(message.get("raw"), dict):
+                _add_nodes(message["raw"], *built)
+                return
+    except Exception:
+        capture.pop("model_calls", None)  # never fail the turn, never store the raw records
+
+
 async def _callable_run_turn(self, messages: list[Message]) -> TurnResult:
     user_text = ""
     for msg in reversed(messages):
@@ -570,6 +651,18 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
         # LangGraph message trace is replaced only when both count the same calls.
         capture = payload.get("tool_capture") if isinstance(payload, dict) else None
         span_trace = next((t for t in reversed(holder) if t), None)
+        if (
+            span_trace is None
+            and isinstance(capture, dict)
+            and capture.get("source") == "decorated"
+            and capture.get("state") == "recorded"
+            and capture.get("trace") is None
+        ):
+            # Every watched tool records itself, so no span is a confirmed
+            # zero, not a turn whose calls carry no trace.
+            capture["trace"] = {"links": "full", "known": ["step", "parent", "started", "ended"],
+                                "unknown": {}, "calls": []}
+            return
         if not isinstance(capture, dict) or span_trace is None:
             return
         own = capture.get("trace")
@@ -620,6 +713,7 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
             _use_span_trace(payload, holder)
         except Exception:
             pass  # a trace that cannot be chosen must never fail the turn
+        _attach_proxy_nodes(result, payload)
         _attach_capture(result, payload)
         if payload is None or any(m.get("role") == "tool" for m in result.interaction_messages):
             return result  # nothing reported, or the spans already carry the tools

package/bin/eval_target.js

@@ -207,7 +207,7 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
   return searchForEntry({ projectPath, args, log });
 }
 
-/** Shapes whose wrapper records tool calls: PY_CAPTURE, `_graph_calls`, or the
+/** Shapes whose wrapper records tool calls: PY_DECORATED, `_graph_calls`, or the
  * recording proxy (PY_PROXY) in eval_templates.js. */
 const CAPTURE_SHAPES = new Set(["py_history", "py_plain", "py_class", "other", "langgraph", "node_bridge", "cli_subprocess"]);
 
@@ -228,14 +228,19 @@ export function verifyEntry(saved) {
   if (saved.sha256 && digest !== saved.sha256) return null;
 
   // True for a wrapper the CLI generated with tool capture wired in (every
-  // shape but `node_bridge`, `cli_subprocess` and `langgraph` defines `_TOOLS`
-  // at module scope -- see PY_CAPTURE in eval_templates.js; `langgraph` reads
-  // calls from the graph's messages in `_graph_calls`; `node_bridge` and
-  // `cli_subprocess` load the recording proxy in `_tool_proxy`). A wrapper
-  // approved before that change has no such line and keeps matching its
-  // digest, so this is read from the file rather than the approval record.
+  // shape but `node_bridge`, `cli_subprocess` and `langgraph` defines
+  // `_DECORATED` at module scope -- see PY_DECORATED in eval_templates.js;
+  // `_TOOLS` is still recognised for wrappers approved before decorators;
+  // `langgraph` reads calls from the graph's messages in `_graph_calls`;
+  // `node_bridge` and `cli_subprocess` load the recording proxy in
+  // `_tool_proxy`). A wrapper approved before that change has no such line and
+  // keeps matching its digest, so this is read from the file rather than the
+  // approval record.
   const capturesTools =
-    /^_TOOLS = /m.test(source) || /^def _graph_calls\(/m.test(source) || /^def _tool_proxy\(/m.test(source);
+    /^_DECORATED = /m.test(source) ||
+    /^_TOOLS = /m.test(source) ||
+    /^def _graph_calls\(/m.test(source) ||
+    /^def _tool_proxy\(/m.test(source);
 
   // Every file the approval produced, not only the wrapper.
   //