premanmcp 1.1.12 → 1.1.14

package/bin/eval.js

@@ -1988,13 +1988,16 @@ async function runWithToken(
     // import path, so the harness reads this and inserts it before handing off.
     process.env.PREMAN_EVAL_SYS_PATH = target.sysPath;
   }
-  // Opt-in for now (docs/EVAL_AGENT_TRACING.md). Kept only for an agent that
-  // runs inside the harness's Python process; a node bridge or a subprocess
-  // is invisible to in-process tracing, and tracing it would only replace the
-  // adapter's reported tool calls with nothing.
-  if (process.env.PREMAN_EVAL_TRACE === "1" && !tracesInProcess(target)) {
-    say(`tracing is not available for this agent (${target.language || "?"} / ${target.shape || "?"}); running untraced`);
-    process.env.PREMAN_EVAL_TRACE = "0";
+  // On unless PREMAN_EVAL_TRACE=0 (docs/EVAL_AGENT_TRACING.md). Kept only for an
+  // agent that runs inside the harness's Python process; a node bridge or a
+  // subprocess is invisible to in-process tracing, and tracing it would only
+  // replace the adapter's reported tool calls with nothing.
+  const traceAsked = process.env.PREMAN_EVAL_TRACE;
+  if (traceAsked !== "0") {
+    process.env.PREMAN_EVAL_TRACE = tracesInProcess(target) ? "1" : "0";
+    if (traceAsked === "1" && !tracesInProcess(target)) {
+      say(`tracing is not available for this agent (${target.language || "?"} / ${target.shape || "?"}); running untraced`);
+    }
   }
 
   if (adapter.translated) {

package/bin/eval_entry.js

@@ -52,7 +52,10 @@ export const ENTRY_FILE = `${ENTRY_MODULE}.py`;
 export const BRIDGE_FILE = "agent_bridge.mjs";
 
 const SEARCH_TIMEOUT_MS = 120_000;
-const PROBE_TIMEOUT_MS = 60_000;
+// One real turn, after importing the customer's agent. GPUScraper alone takes
+// 37s to import from /mnt/c (2,227 modules), so 60s failed on the import plus
+// one model call. PREMAN_PROBE_TIMEOUT_S overrides for slower repos.
+const PROBE_TIMEOUT_MS = (Number(process.env.PREMAN_PROBE_TIMEOUT_S) || 180) * 1000;
 const MAX_TREE_PATHS = 4_000;
 const MAX_TREE_BYTES = 120_000;
 const MAX_SHORTLIST = 12;
@@ -392,7 +395,11 @@ export async function validateShim(file, { projectPath, log }) {
     cwd: projectPath,
   });
   if (ran.error?.code === "ETIMEDOUT") {
-    return { ok: false, why: "it did not answer within 60 seconds" };
+    return {
+      ok: false,
+      why: `it did not answer within ${PROBE_TIMEOUT_MS / 1000} seconds (importing the agent ` +
+        `counts; set PREMAN_PROBE_TIMEOUT_S to allow longer)`,
+    };
   }
   const line = String(ran.stdout || "")
     .split("\n")

package/bin/eval_harness.py

@@ -304,6 +304,36 @@ def _is_adapter_payload(value) -> bool:
     return isinstance(value, dict) and isinstance(value.get("response"), str)
 
 
+def _turn_raw(ref, raw_result):
+    """The turn's raw record, plus the wrapper's capture outcome when it sent one.
+
+    Carried on ``raw`` because assert-ai stores it verbatim and owns every other
+    field of the artifact; the backend reads it back per turn
+    (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §7.2).
+    """
+    raw = {"callable": ref}
+    capture = raw_result.get("tool_capture")
+    if isinstance(capture, dict):
+        raw["tool_capture"] = _session._sanitize_endpoint_value(capture)
+    return raw
+
+
+def _attach_capture(result, payload):
+    """Carry the wrapper's tool_capture onto a traced turn.
+
+    A traced turn's stored record is rebuilt by assert-ai from spans
+    (trace_events / trace_metadata / span_validation), so the record
+    ``_turn_raw`` writes on the untraced path never reaches it.
+    """
+    capture = payload.get("tool_capture") if isinstance(payload, dict) else None
+    if not isinstance(capture, dict):
+        return
+    for message in reversed(result.interaction_messages):
+        if message.get("role") == "assistant" and isinstance(message.get("raw"), dict):
+            message["raw"]["tool_capture"] = _session._sanitize_endpoint_value(capture)
+            return
+
+
 async def _callable_run_turn(self, messages: list[Message]) -> TurnResult:
     user_text = ""
     for msg in reversed(messages):
@@ -380,7 +410,7 @@ async def _callable_run_turn(self, messages: list[Message]) -> TurnResult:
     response = _session._normalize_connector_response({
         "text": _session._sanitize_response_text(raw_result["response"]),
         "events": _session._sanitize_endpoint_value(raw_result.get("events")),
-        "raw": {"callable": self._callable_ref},
+        "raw": _turn_raw(self._callable_ref, raw_result),
     })
     interaction_messages = _session._serialize_connector_interaction_messages(
         user_text=user_text,
@@ -452,6 +482,106 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
     _traced_open = _Traced.open
     _traced_run_turn = _Traced.run_turn
 
+    import contextvars
+
+    # Filled by _spans_to_events_once with the trace its spans give; read by
+    # the run_turn that set it (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §7.5).
+    _span_trace = contextvars.ContextVar("preman_span_trace", default=None)
+
+    def _trace_from_spans(spans):
+        """step / parent / times for the turn's TOOL spans. 'event' is a span's
+        position among them, the order assert-ai turns them into tool events."""
+        tools = [s for s in spans if s.kind == "TOOL"]
+        if not tools:
+            return None
+        if any(getattr(s, "convention", None) == "gen_ai" for s in spans):
+            # assert-ai builds gen_ai tool events from the model's own tool-call
+            # list, not one per TOOL span, so 'event' would not line up.
+            return None
+        by_id = {(s.trace_id, s.span_id): s for s in spans}
+
+        def depth(span):
+            n, hop = 0, by_id.get((span.trace_id, span.parent_span_id))
+            while hop is not None and n < len(spans):
+                n, hop = n + 1, by_id.get((hop.trace_id, hop.parent_span_id))
+            return n
+
+        # Start time, then depth: a call and one it makes can start in the same
+        # nanosecond, and the outer one must be numbered (and seen) first.
+        order = sorted(tools, key=lambda s: (s.start_time_ns, depth(s)))
+        cid = {(s.trace_id, s.span_id): "c%d" % (i + 1) for i, s in enumerate(order)}
+        key_of = {v: k for k, v in cid.items()}
+        base = min(s.start_time_ns for s in spans)
+        llm = sorted((s for s in spans if s.kind == "LLM"), key=lambda s: s.end_time_ns)
+        unknown, parents, steps = {}, {}, {}
+        # "The reply that finished last before the tool started" holds for one
+        # agent loop; two model calls overlapping in time could pick the wrong one.
+        if any(a.start_time_ns < b.end_time_ns and b.start_time_ns < a.end_time_ns
+               for i, a in enumerate(llm) for b in llm[i + 1:]):
+            unknown["step"] = "two model calls in this turn overlapped, so which one asked for each tool is unclear"
+        for s in order:
+            key = (s.trace_id, s.span_id)
+            hop = by_id.get((s.trace_id, s.parent_span_id))
+            while hop is not None and hop.kind != "TOOL":
+                hop = by_id.get((hop.trace_id, hop.parent_span_id))
+            if hop is None and _dangling(s, by_id):
+                unknown["parent"] = "a tool span's parent was not among this turn's spans"
+            parents[key] = cid.get((hop.trace_id, hop.span_id)) if hop is not None else None
+            if parents[key] is not None:
+                steps[key] = steps[key_of[parents[key]]]  # a nested call shares its parent's step
+                continue
+            asked = [m for m in llm if m.end_time_ns <= s.start_time_ns]
+            if not asked:
+                unknown.setdefault("step", "a tool ran before any model reply in this turn")
+            steps[key] = (asked[-1].trace_id, asked[-1].span_id) if asked else None
+        numbering = {}
+        calls = []
+        for s in order:
+            key = (s.trace_id, s.span_id)
+            a = s.attributes
+            call = {
+                "id": cid[key],
+                "tool": str(a.get("tool.name", s.name)),  # the name assert-ai stores
+                "event": tools.index(s),
+                "started": round((s.start_time_ns - base) / 1e9, 3),
+                "ended": round((s.end_time_ns - base) / 1e9, 3),
+            }
+            if "step" not in unknown:
+                call["step"] = numbering.setdefault(steps[key], len(numbering) + 1)
+            if "parent" not in unknown:
+                call["parent"] = parents[key]
+            calls.append(call)
+        known = [f for f in ("step", "parent") if f not in unknown] + ["started", "ended"]
+        return {"links": "full", "known": known, "unknown": unknown, "calls": calls}
+
+    def _dangling(span, by_id):
+        # True when some span above this one is missing from the turn's spans.
+        hop = span
+        while hop.parent_span_id:
+            up = by_id.get((hop.trace_id, hop.parent_span_id))
+            if up is None:
+                return True
+            hop = up
+        return False
+
+    def _use_span_trace(payload, holder):
+        # Spans see LangChain / LangGraph tools the wrapper cannot, and give
+        # parent as well as step. A wrapper trace of plain functions is kept; a
+        # LangGraph message trace is replaced only when both count the same calls.
+        capture = payload.get("tool_capture") if isinstance(payload, dict) else None
+        span_trace = next((t for t in reversed(holder) if t), None)
+        if not isinstance(capture, dict) or span_trace is None:
+            return
+        own = capture.get("trace")
+        if own is None:
+            capture["trace"] = span_trace
+        elif (
+            capture.get("source") == "langgraph"
+            and isinstance(own, dict)
+            and len(own.get("calls") or []) == len(span_trace["calls"])
+        ):
+            capture["trace"] = span_trace
+
     async def _traced_open_unpacking(self):
         await _traced_open(self)
         agent = self._callable
@@ -475,6 +605,8 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
 
     async def _traced_run_turn_with_evidence(self, messages):
         self._preman_payload = None
+        holder = []
+        _span_trace.set(holder)
         try:
             result = await _traced_run_turn(self, messages)
         except Exception as exc:
@@ -484,6 +616,11 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
             raise classified from exc
 
         payload, self._preman_payload = self._preman_payload, None
+        try:
+            _use_span_trace(payload, holder)
+        except Exception:
+            pass  # a trace that cannot be chosen must never fail the turn
+        _attach_capture(result, payload)
         if payload is None or any(m.get("role") == "tool" for m in result.interaction_messages):
             return result  # nothing reported, or the spans already carry the tools
 
@@ -548,7 +685,14 @@ if os.environ.get("PREMAN_EVAL_TRACE") == "1":
                         and a.start_time_ns <= b.start_time_ns and b.end_time_ns <= a.end_time_ns
                         and (a.start_time_ns, a.end_time_ns) != (b.start_time_ns, b.end_time_ns)):
                     outer.add((a.trace_id, a.span_id))
-        return _spans_to_events([s for s in spans if (s.trace_id, s.span_id) not in outer])
+        kept = [s for s in spans if (s.trace_id, s.span_id) not in outer]
+        holder = _span_trace.get()
+        if holder is not None:
+            try:
+                holder.append(_trace_from_spans(kept))
+            except Exception:
+                pass  # a trace that cannot be built must never fail the turn
+        return _spans_to_events(kept)
 
     # Resolved at call time: otel_session imported the name into its own module.
     _otel_session._spans_to_events = _spans_to_events_once

package/bin/eval_target.js

@@ -160,7 +160,15 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
   const saved = savedEntry(projectPath);
   if (saved) {
     const verified = verifyEntry(saved);
-    if (verified) return verified;
+    if (verified) {
+      if (CAPTURE_SHAPES.has(verified.shape) && !verified.capturesTools) {
+        log(
+          "this agent's wrapper was approved before PreMan recorded tool calls; " +
+            "run `preman eval target` again to record them"
+        );
+      }
+      return verified;
+    }
     log(
       `The approved entry point ${saved.file} has changed since it was approved, ` +
         `so it is being re-checked rather than trusted.`
@@ -199,6 +207,10 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
   return searchForEntry({ projectPath, args, log });
 }
 
+/** Shapes whose wrapper records tool calls: PY_CAPTURE, `_graph_calls`, or the
+ * recording proxy (PY_PROXY) in eval_templates.js. */
+const CAPTURE_SHAPES = new Set(["py_history", "py_plain", "py_class", "other", "langgraph", "node_bridge", "cli_subprocess"]);
+
 /**
  * The approved shim, if it is still the file that was approved.
  *
@@ -215,6 +227,16 @@ export function verifyEntry(saved) {
   const digest = createHash("sha256").update(source).digest("hex");
   if (saved.sha256 && digest !== saved.sha256) return null;
 
+  // True for a wrapper the CLI generated with tool capture wired in (every
+  // shape but `node_bridge`, `cli_subprocess` and `langgraph` defines `_TOOLS`
+  // at module scope -- see PY_CAPTURE in eval_templates.js; `langgraph` reads
+  // calls from the graph's messages in `_graph_calls`; `node_bridge` and
+  // `cli_subprocess` load the recording proxy in `_tool_proxy`). A wrapper
+  // approved before that change has no such line and keeps matching its
+  // digest, so this is read from the file rather than the approval record.
+  const capturesTools =
+    /^_TOOLS = /m.test(source) || /^def _graph_calls\(/m.test(source) || /^def _tool_proxy\(/m.test(source);
+
   // Every file the approval produced, not only the wrapper.
   //
   // A `node_bridge` target is two files: the Python wrapper and the `.mjs` it
@@ -246,6 +268,7 @@ export function verifyEntry(saved) {
     sysPath,
     sha256: digest,
     files: extra,
+    capturesTools,
     shape: saved.shape || "",
     language: saved.language,
     framework: saved.framework,