premanmcp 1.1.13 → 1.1.14

package/bin/eval_harness.py

@@ -482,6 +482,106 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
     _traced_open = _Traced.open
     _traced_run_turn = _Traced.run_turn
 
+    import contextvars
+
+    # Filled by _spans_to_events_once with the trace its spans give; read by
+    # the run_turn that set it (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §7.5).
+    _span_trace = contextvars.ContextVar("preman_span_trace", default=None)
+
+    def _trace_from_spans(spans):
+        """step / parent / times for the turn's TOOL spans. 'event' is a span's
+        position among them, the order assert-ai turns them into tool events."""
+        tools = [s for s in spans if s.kind == "TOOL"]
+        if not tools:
+            return None
+        if any(getattr(s, "convention", None) == "gen_ai" for s in spans):
+            # assert-ai builds gen_ai tool events from the model's own tool-call
+            # list, not one per TOOL span, so 'event' would not line up.
+            return None
+        by_id = {(s.trace_id, s.span_id): s for s in spans}
+
+        def depth(span):
+            n, hop = 0, by_id.get((span.trace_id, span.parent_span_id))
+            while hop is not None and n < len(spans):
+                n, hop = n + 1, by_id.get((hop.trace_id, hop.parent_span_id))
+            return n
+
+        # Start time, then depth: a call and one it makes can start in the same
+        # nanosecond, and the outer one must be numbered (and seen) first.
+        order = sorted(tools, key=lambda s: (s.start_time_ns, depth(s)))
+        cid = {(s.trace_id, s.span_id): "c%d" % (i + 1) for i, s in enumerate(order)}
+        key_of = {v: k for k, v in cid.items()}
+        base = min(s.start_time_ns for s in spans)
+        llm = sorted((s for s in spans if s.kind == "LLM"), key=lambda s: s.end_time_ns)
+        unknown, parents, steps = {}, {}, {}
+        # "The reply that finished last before the tool started" holds for one
+        # agent loop; two model calls overlapping in time could pick the wrong one.
+        if any(a.start_time_ns < b.end_time_ns and b.start_time_ns < a.end_time_ns
+               for i, a in enumerate(llm) for b in llm[i + 1:]):
+            unknown["step"] = "two model calls in this turn overlapped, so which one asked for each tool is unclear"
+        for s in order:
+            key = (s.trace_id, s.span_id)
+            hop = by_id.get((s.trace_id, s.parent_span_id))
+            while hop is not None and hop.kind != "TOOL":
+                hop = by_id.get((hop.trace_id, hop.parent_span_id))
+            if hop is None and _dangling(s, by_id):
+                unknown["parent"] = "a tool span's parent was not among this turn's spans"
+            parents[key] = cid.get((hop.trace_id, hop.span_id)) if hop is not None else None
+            if parents[key] is not None:
+                steps[key] = steps[key_of[parents[key]]]  # a nested call shares its parent's step
+                continue
+            asked = [m for m in llm if m.end_time_ns <= s.start_time_ns]
+            if not asked:
+                unknown.setdefault("step", "a tool ran before any model reply in this turn")
+            steps[key] = (asked[-1].trace_id, asked[-1].span_id) if asked else None
+        numbering = {}
+        calls = []
+        for s in order:
+            key = (s.trace_id, s.span_id)
+            a = s.attributes
+            call = {
+                "id": cid[key],
+                "tool": str(a.get("tool.name", s.name)),  # the name assert-ai stores
+                "event": tools.index(s),
+                "started": round((s.start_time_ns - base) / 1e9, 3),
+                "ended": round((s.end_time_ns - base) / 1e9, 3),
+            }
+            if "step" not in unknown:
+                call["step"] = numbering.setdefault(steps[key], len(numbering) + 1)
+            if "parent" not in unknown:
+                call["parent"] = parents[key]
+            calls.append(call)
+        known = [f for f in ("step", "parent") if f not in unknown] + ["started", "ended"]
+        return {"links": "full", "known": known, "unknown": unknown, "calls": calls}
+
+    def _dangling(span, by_id):
+        # True when some span above this one is missing from the turn's spans.
+        hop = span
+        while hop.parent_span_id:
+            up = by_id.get((hop.trace_id, hop.parent_span_id))
+            if up is None:
+                return True
+            hop = up
+        return False
+
+    def _use_span_trace(payload, holder):
+        # Spans see LangChain / LangGraph tools the wrapper cannot, and give
+        # parent as well as step. A wrapper trace of plain functions is kept; a
+        # LangGraph message trace is replaced only when both count the same calls.
+        capture = payload.get("tool_capture") if isinstance(payload, dict) else None
+        span_trace = next((t for t in reversed(holder) if t), None)
+        if not isinstance(capture, dict) or span_trace is None:
+            return
+        own = capture.get("trace")
+        if own is None:
+            capture["trace"] = span_trace
+        elif (
+            capture.get("source") == "langgraph"
+            and isinstance(own, dict)
+            and len(own.get("calls") or []) == len(span_trace["calls"])
+        ):
+            capture["trace"] = span_trace
+
     async def _traced_open_unpacking(self):
         await _traced_open(self)
         agent = self._callable
@@ -505,6 +605,8 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
 
     async def _traced_run_turn_with_evidence(self, messages):
         self._preman_payload = None
+        holder = []
+        _span_trace.set(holder)
         try:
             result = await _traced_run_turn(self, messages)
         except Exception as exc:
@@ -514,6 +616,10 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
             raise classified from exc
 
         payload, self._preman_payload = self._preman_payload, None
+        try:
+            _use_span_trace(payload, holder)
+        except Exception:
+            pass  # a trace that cannot be chosen must never fail the turn
         _attach_capture(result, payload)
         if payload is None or any(m.get("role") == "tool" for m in result.interaction_messages):
             return result  # nothing reported, or the spans already carry the tools
@@ -579,7 +685,14 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
                         and a.start_time_ns <= b.start_time_ns and b.end_time_ns <= a.end_time_ns
                         and (a.start_time_ns, a.end_time_ns) != (b.start_time_ns, b.end_time_ns)):
                     outer.add((a.trace_id, a.span_id))
-        return _spans_to_events([s for s in spans if (s.trace_id, s.span_id) not in outer])
+        kept = [s for s in spans if (s.trace_id, s.span_id) not in outer]
+        holder = _span_trace.get()
+        if holder is not None:
+            try:
+                holder.append(_trace_from_spans(kept))
+            except Exception:
+                pass  # a trace that cannot be built must never fail the turn
+        return _spans_to_events(kept)
 
     # Resolved at call time: otel_session imported the name into its own module.
     _otel_session._spans_to_events = _spans_to_events_once

package/bin/eval_target.js

@@ -207,8 +207,9 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
   return searchForEntry({ projectPath, args, log });
 }
 
-/** Shapes the tool-capture wrapper applies to (PY_CAPTURE in eval_templates.js). */
-const CAPTURE_SHAPES = new Set(["py_history", "py_plain", "py_class", "other"]);
+/** Shapes whose wrapper records tool calls: PY_CAPTURE, `_graph_calls`, or the
+ * recording proxy (PY_PROXY) in eval_templates.js. */
+const CAPTURE_SHAPES = new Set(["py_history", "py_plain", "py_class", "other", "langgraph", "node_bridge", "cli_subprocess"]);
 
 /**
  * The approved shim, if it is still the file that was approved.
@@ -227,11 +228,14 @@ export function verifyEntry(saved) {
   if (saved.sha256 && digest !== saved.sha256) return null;
 
   // True for a wrapper the CLI generated with tool capture wired in (every
-  // shape but `node_bridge`, `cli_subprocess` and `langgraph` defines this at
-  // module scope -- see PY_CAPTURE in eval_templates.js). A wrapper approved
-  // before that change has no such line and keeps matching its digest, so this
-  // is read from the file rather than the approval record.
-  const capturesTools = /^_TOOLS = /m.test(source);
+  // shape but `node_bridge`, `cli_subprocess` and `langgraph` defines `_TOOLS`
+  // at module scope -- see PY_CAPTURE in eval_templates.js; `langgraph` reads
+  // calls from the graph's messages in `_graph_calls`; `node_bridge` and
+  // `cli_subprocess` load the recording proxy in `_tool_proxy`). A wrapper
+  // approved before that change has no such line and keeps matching its
+  // digest, so this is read from the file rather than the approval record.
+  const capturesTools =
+    /^_TOOLS = /m.test(source) || /^def _graph_calls\(/m.test(source) || /^def _tool_proxy\(/m.test(source);
 
   // Every file the approval produced, not only the wrapper.
   //