premanmcp 1.1.11 → 1.1.13

package/bin/eval_harness.py

@@ -73,10 +73,39 @@ edit made under .venv/.
    kept on ``self`` between the call and its use can be overwritten by another
    turn. A replacement holds the payload in a local.
 
+6. A traced callable target -- opt-in, ``PREMAN_EVAL_TRACE=1``, set by the CLI
+   only for an agent that runs in this process (docs/EVAL_AGENT_TRACING.md).
+   With ``target.trace`` in the spec, assert-ai uses ``OTelTracedSession``,
+   which workaround 5 never reaches. Three things are done for it:
+
+   (a) PreMan's own model calls -- tester, judges, rubric -- run with tracing
+       suppressed. assert-ai files spans under a turn by time window, so with
+       ``concurrency`` above 1 another case's tester call lands in this agent's
+       trace. Their cost is still counted, by assert-ai's own accounting, into
+       ``metrics.json``.
+   (b) The traced session gets workaround 5's treatment: an adapter's
+       ``{"response", "events"}`` is unpacked instead of ``str()``'d into the
+       transcript, the tool calls it reports are kept when the spans carry
+       none, and a provider error is classified rather than crashing the case.
+       The reply is kept on the session between the call and its use, which (5)
+       explains is unsafe for ``CallableSession`` and is safe here: the
+       inference stage builds one ``OTelTracedSession`` per case and runs its
+       turns one after another.
+   (c) A model call is counted once. A framework's plugin (litellm, LangChain,
+       LlamaIndex) and the provider SDK's plugin each record the same call with
+       the same tokens, so a turn's spans are filtered before assert-ai
+       flattens them: a model-call span with another model-call span beneath
+       it is dropped, keeping the innermost -- the SDK's, which carries the
+       provider's response id and model name. "Beneath" is the parent link
+       where the plugin records one (litellm); LangChain and LlamaIndex do not,
+       so there it is a span with the same tokens inside its time window.
+
 Workarounds (1)-(4) go away upstream: (1) with a workspace-scoped API key, (2),
 (3) and (4) with a fix in AleWang16/ASSERT plus a bumped rev in pyproject.toml.
 (5) goes away when ``CallableSession`` serialises tool evidence the way
-``HTTPEndpointSession`` already does.
+``HTTPEndpointSession`` already does. (6a) goes away when assert-ai stops
+tracing its own calls, (6b) with (5), (6c) when assert-ai's
+``_spans_to_events`` drops nested model-call spans itself.
 
 This file exists twice. ``preman-mcp/bin/eval_harness.py`` is a byte-identical
 copy, shipped in the npm package so a run leased to a paired device goes through
@@ -275,6 +304,36 @@ def _is_adapter_payload(value) -> bool:
     return isinstance(value, dict) and isinstance(value.get("response"), str)
 
 
+def _turn_raw(ref, raw_result):
+    """The turn's raw record, plus the wrapper's capture outcome when it sent one.
+
+    Carried on ``raw`` because assert-ai stores it verbatim and owns every other
+    field of the artifact; the backend reads it back per turn
+    (docs/superpowers/specs/2026-09-30-tool-call-capture-design.md §7.2).
+    """
+    raw = {"callable": ref}
+    capture = raw_result.get("tool_capture")
+    if isinstance(capture, dict):
+        raw["tool_capture"] = _session._sanitize_endpoint_value(capture)
+    return raw
+
+
+def _attach_capture(result, payload):
+    """Carry the wrapper's tool_capture onto a traced turn.
+
+    A traced turn's stored record is rebuilt by assert-ai from spans
+    (trace_events / trace_metadata / span_validation), so the record
+    ``_turn_raw`` writes on the untraced path never reaches it.
+    """
+    capture = payload.get("tool_capture") if isinstance(payload, dict) else None
+    if not isinstance(capture, dict):
+        return
+    for message in reversed(result.interaction_messages):
+        if message.get("role") == "assistant" and isinstance(message.get("raw"), dict):
+            message["raw"]["tool_capture"] = _session._sanitize_endpoint_value(capture)
+            return
+
+
 async def _callable_run_turn(self, messages: list[Message]) -> TurnResult:
     user_text = ""
     for msg in reversed(messages):
@@ -351,7 +410,7 @@ async def _callable_run_turn(self, messages: list[Message]) -> TurnResult:
     response = _session._normalize_connector_response({
         "text": _session._sanitize_response_text(raw_result["response"]),
         "events": _session._sanitize_endpoint_value(raw_result.get("events")),
-        "raw": {"callable": self._callable_ref},
+        "raw": _turn_raw(self._callable_ref, raw_result),
     })
     interaction_messages = _session._serialize_connector_interaction_messages(
         user_text=user_text,
@@ -392,5 +451,140 @@ for _root in os.environ.get("PREMAN_EVAL_SYS_PATH", "").split(os.pathsep):
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-_cli = next(e for e in entry_points(group="console_scripts") if e.name == "assert-ai")
-sys.exit(_cli.load()())
+from eval_generation_debug import install as _install_generation_debug
+from eval_generation_status import install as _install_generation_status
+
+_install_generation_debug()
+_install_generation_status()
+
+# Workaround 6 -- see the module docstring.
+if os.environ.get("PREMAN_EVAL_TRACE") == "1":
+    import functools
+    import inspect
+
+    try:
+        from openinference.instrumentation import suppress_tracing as _suppress_tracing
+    except ImportError:  # no plugin installed: nothing would record PreMan's calls anyway
+        _suppress_tracing = None
+
+    if _suppress_tracing is not None:
+        _with_retries = model_client._with_retries
+
+        async def _with_retries_untraced(call_fn, *args, **kwargs):
+            with _suppress_tracing():
+                return await _with_retries(call_fn, *args, **kwargs)
+
+        # Rebound as a module attribute: every call site resolves it at call time.
+        model_client._with_retries = _with_retries_untraced
+
+    from assert_ai.core.otel_session import OTelTracedSession as _Traced
+
+    _traced_open = _Traced.open
+    _traced_run_turn = _Traced.run_turn
+
+    async def _traced_open_unpacking(self):
+        await _traced_open(self)
+        agent = self._callable
+
+        def _unpack(result):
+            if _is_adapter_payload(result):
+                self._preman_payload = result
+                return _session._sanitize_response_text(result["response"])
+            return result
+
+        if inspect.iscoroutinefunction(agent):
+            @functools.wraps(agent)
+            async def _agent(*args, **kwargs):
+                return _unpack(await agent(*args, **kwargs))
+        else:
+            @functools.wraps(agent)
+            def _agent(*args, **kwargs):
+                return _unpack(agent(*args, **kwargs))
+
+        self._callable = _agent
+
+    async def _traced_run_turn_with_evidence(self, messages):
+        self._preman_payload = None
+        try:
+            result = await _traced_run_turn(self, messages)
+        except Exception as exc:
+            classified = _classify_llm_error(exc)
+            if classified is exc:
+                raise
+            raise classified from exc
+
+        payload, self._preman_payload = self._preman_payload, None
+        _attach_capture(result, payload)
+        if payload is None or any(m.get("role") == "tool" for m in result.interaction_messages):
+            return result  # nothing reported, or the spans already carry the tools
+
+        response = _session._normalize_connector_response({
+            "text": result.text,
+            "events": _session._sanitize_endpoint_value(payload.get("events")),
+            "raw": {"callable": self._callable_ref},
+        })
+        reported = [
+            m
+            for m in _session._serialize_connector_interaction_messages(
+                user_text=result.interaction_messages[0].get("content", ""), response=response
+            )
+            if m.get("role") == "tool" or (m.get("role") == "assistant" and m.get("tool_calls"))
+        ]
+        result.interaction_messages[1:1] = reported
+        result.tool_traces = [
+            _session.ToolTrace(
+                tool_name=event.tool_name or "tool",
+                tool_args=event.tool_args or {},
+                tool_result=event.content,
+            )
+            for event in (response.events or [])
+            if event.role == "tool_result"
+        ]
+        return result
+
+    _Traced.open = _traced_open_unpacking
+    _Traced.run_turn = _traced_run_turn_with_evidence
+
+    from assert_ai.core import otel_session as _otel_session
+
+    _spans_to_events = _otel_session._spans_to_events
+
+    def _span_tokens(span):
+        a = span.attributes
+        return (a.get("llm.token_count.prompt", a.get("gen_ai.usage.input_tokens")),
+                a.get("llm.token_count.completion", a.get("gen_ai.usage.output_tokens")))
+
+    def _spans_to_events_once(spans):
+        by_id = {(s.trace_id, s.span_id): s for s in spans}
+        llm = [s for s in spans if s.kind == "LLM"]
+        outer = set()
+        for span in llm:
+            seen, ancestors = set(), []
+            parent = by_id.get((span.trace_id, span.parent_span_id))
+            while parent is not None and parent is not span and parent.span_id not in seen:
+                seen.add(parent.span_id)
+                if parent.kind == "LLM":
+                    ancestors.append((parent.trace_id, parent.span_id))
+                parent = by_id.get((parent.trace_id, parent.parent_span_id))
+            if parent is not span:  # a chain that loops back to this span proves no nesting
+                outer.update(ancestors)
+        # LangChain and LlamaIndex never make their span the current context, so
+        # the SDK's span is not linked beneath it: match by tokens and time instead.
+        for a in llm:
+            tokens = _span_tokens(a)
+            if None in tokens:
+                continue
+            for b in llm:
+                if (b is not a and _span_tokens(b) == tokens
+                        and a.start_time_ns <= b.start_time_ns and b.end_time_ns <= a.end_time_ns
+                        and (a.start_time_ns, a.end_time_ns) != (b.start_time_ns, b.end_time_ns)):
+                    outer.add((a.trace_id, a.span_id))
+        return _spans_to_events([s for s in spans if (s.trace_id, s.span_id) not in outer])
+
+    # Resolved at call time: otel_session imported the name into its own module.
+    _otel_session._spans_to_events = _spans_to_events_once
+
+
+if __name__ == "__main__":
+    _cli = next(e for e in entry_points(group="console_scripts") if e.name == "assert-ai")
+    sys.exit(_cli.load()())

package/bin/eval_start.js

@@ -0,0 +1,203 @@
+/**
+ * Starting an eval from a terminal: check, quote, ask, run, report.
+ *
+ * Separate from `eval.js` because the two halves are opposite roles wearing one
+ * word. `eval.js` is the machine volunteering itself as a worker -- redeem a
+ * token, pair a repository, wait for queued work. This is a person asking for
+ * their agent to be measured. They share plumbing and share nothing else, and
+ * keeping them in one file is what made `eval run` and `test --agent` read as
+ * synonyms when they are opposites.
+ *
+ * Nothing here words a card, a price or a refusal. The server does that, and
+ * this prints what comes back. Two surfaces composing the same sentence from
+ * the same numbers is how they end up disagreeing about what somebody is about
+ * to pay for -- and the one that disagrees is the one they approved.
+ */
+
+import {
+  callBackendJson,
+  cliInvocation,
+  describeFailure,
+  promptText,
+  resolveApiKey,
+} from "./shared.js";
+
+/** Ran, and the pass rate cleared the threshold (or none was asked for). */
+export const EXIT_OK = 0;
+/** Ran, and the pass rate did not clear `--fail-on`. */
+export const EXIT_BELOW_THRESHOLD = 1;
+/**
+ * Nothing ran: not ready, declined, refused, out of budget.
+ *
+ * Distinct from 1 because `preman eval && deploy` has to tell "the agent
+ * behaved" apart from "we never found out", and today both are a non-zero
+ * exit with no way to know which happened.
+ */
+export const EXIT_DID_NOT_RUN = 2;
+
+/**
+ * The account's four states, as the server sees them.
+ *
+ * Read rather than worked out here. The CLI has enough local knowledge to guess
+ * -- it can see whether a daemon is running on this box -- and a guess that
+ * disagrees with what the dashboard says about the same machine is worse than
+ * no answer, because the two are usually read minutes apart by one person.
+ */
+export async function accountReadiness(args, key) {
+  const found = await callBackendJson(args, "GET", "/eval-sessions/runner/readiness", {
+    token: key,
+  });
+  if (!found.ok) {
+    throw new Error(
+      `could not read whether a run could start (${found.status_code}): ` +
+        `${describeFailure(found)}`
+    );
+  }
+  return found;
+}
+
+/**
+ * What this selection costs, where it would run, and how long it usually takes.
+ *
+ * `surface: "cli"` changes exactly one clause -- the sentence about where the
+ * result will appear -- because a terminal has no conversation for the worker
+ * to append to, and that is the one promise on the card that is not true
+ * everywhere it is read.
+ */
+export async function quoteRun(args, key, { behaviorIds = [], endpoint = "", deviceId = "" } = {}) {
+  const quoted = await callBackendJson(args, "POST", "/eval-sessions/quote", {
+    token: key,
+    json: {
+      behavior_ids: behaviorIds,
+      endpoint: endpoint || null,
+      device_id: deviceId || null,
+      surface: "cli",
+    },
+  });
+  if (!quoted.ok) {
+    throw new Error(
+      `could not price this run (${quoted.status_code}): ${describeFailure(quoted)}`
+    );
+  }
+  return quoted;
+}
+
+/**
+ * Show the card and get an answer.
+ *
+ * Default no. A prompt that proceeds on a bare Enter is a prompt that spends
+ * money when somebody leans on the keyboard, and every other confirmation in
+ * this CLI that guards an irreversible charge reads the same way.
+ */
+export async function confirmSpend(
+  quote,
+  { assumeYes = false, say = () => {}, ask = promptText } = {}
+) {
+  say("");
+  say(quote.summary);
+  say("");
+  if (assumeYes) {
+    say("Starting: --yes was given.");
+    return true;
+  }
+  const answer = String((await ask("Start this run? [y/N] ")) || "").trim().toLowerCase();
+  return answer === "y" || answer === "yes";
+}
+
+/**
+ * Say why a run cannot start, in the server's words, and stop.
+ *
+ * One next step, printed as given. `readiness` already guarantees a single
+ * command per state; picking a second one to add here is how a refusal turns
+ * into a list, and a list reads as "it is broken" rather than "do this".
+ */
+export function reportNotReady(readiness, { say = () => {} } = {}) {
+  say(readiness.note);
+  if (readiness.next_step) {
+    say("");
+    say(`  ${readiness.next_step}`);
+  }
+  return EXIT_DID_NOT_RUN;
+}
+
+/**
+ * The fraction of cases that passed, or null when the summary cannot be read.
+ *
+ * Null rather than zero. A run whose summary did not parse has not told us it
+ * failed, and treating silence as failure would break a build over a reporting
+ * bug.
+ */
+export function passRate(summary) {
+  const cases = Number(summary?.cases);
+  const passed = Number(summary?.passed);
+  if (!Number.isFinite(cases) || !Number.isFinite(passed) || cases <= 0) return null;
+  return passed / cases;
+}
+
+/**
+ * Which exit code a finished run deserves.
+ *
+ * `--fail-on` defaults to off so that adding this to an existing pipeline
+ * cannot start failing builds that were passing yesterday. Opting in is the
+ * whole point of a gate.
+ */
+export function exitCodeFor(summary, { failOn = null } = {}) {
+  if (failOn === null || failOn === undefined || failOn === "") return EXIT_OK;
+  const rate = passRate(summary);
+  if (rate === null) return EXIT_OK;
+  const threshold = Number(failOn);
+  if (!Number.isFinite(threshold)) return EXIT_OK;
+  // A threshold given as a percentage is the likelier reading of `--fail-on 80`
+  // than a ratio of 80, and nobody means "eighty times as many passes as cases".
+  const wanted = threshold > 1 ? threshold / 100 : threshold;
+  return rate + 1e-9 >= wanted ? EXIT_OK : EXIT_BELOW_THRESHOLD;
+}
+
+/**
+ * Which behaviours this run will cover, resolved against what is switched on.
+ *
+ * Naming the ones that are on beats "not found": the usual cause of a miss is a
+ * behaviour that exists but is switched off, which is a different fix from a
+ * typo.
+ */
+export async function chooseBehaviors(args, key, { only = "" } = {}) {
+  const catalog = await callBackendJson(args, "GET", "/eval-behaviors", { token: key });
+  if (!catalog.ok) {
+    throw new Error(
+      `could not read what this account measures (${catalog.status_code}): ` +
+        `${describeFailure(catalog)}`
+    );
+  }
+  const enabled = (catalog.behaviors || []).filter((entry) => entry.enabled);
+  const wanted = String(only || "").trim();
+  const chosen = wanted
+    ? enabled.filter((entry) => entry.id === wanted || entry.name === wanted)
+    : enabled;
+
+  if (wanted && chosen.length === 0) {
+    const names = enabled.map((entry) => entry.id).join(", ") || "nothing";
+    throw new Error(`${wanted} is not switched on. Currently on: ${names}`);
+  }
+  if (chosen.length === 0) {
+    throw new Error(
+      "nothing is switched on for this account, so there is nothing to measure. " +
+        "Turn a behaviour on under Agent eval → Settings, or describe one in your " +
+        "own words there."
+    );
+  }
+  return chosen;
+}
+
+/**
+ * The key, or a message naming the command that gets one.
+ */
+export function requireKey(args) {
+  const key = resolveApiKey(args);
+  if (!key) {
+    throw new Error(
+      `no PreMan API key on this machine, so there is no account to read the ` +
+        `switched-on behaviours from. Run \`${cliInvocation()} login\`.`
+    );
+  }
+  return key;
+}

package/bin/eval_target.js

@@ -160,7 +160,15 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
   const saved = savedEntry(projectPath);
   if (saved) {
     const verified = verifyEntry(saved);
-    if (verified) return verified;
+    if (verified) {
+      if (CAPTURE_SHAPES.has(verified.shape) && !verified.capturesTools) {
+        log(
+          "this agent's wrapper was approved before PreMan recorded tool calls; " +
+            "run `preman eval target` again to record them"
+        );
+      }
+      return verified;
+    }
     log(
       `The approved entry point ${saved.file} has changed since it was approved, ` +
         `so it is being re-checked rather than trusted.`
@@ -199,6 +207,9 @@ export async function discoverTarget(args, { log = () => {}, allowSearch = true
   return searchForEntry({ projectPath, args, log });
 }
 
+/** Shapes the tool-capture wrapper applies to (PY_CAPTURE in eval_templates.js). */
+const CAPTURE_SHAPES = new Set(["py_history", "py_plain", "py_class", "other"]);
+
 /**
  * The approved shim, if it is still the file that was approved.
  *
@@ -215,6 +226,13 @@ export function verifyEntry(saved) {
   const digest = createHash("sha256").update(source).digest("hex");
   if (saved.sha256 && digest !== saved.sha256) return null;
 
+  // True for a wrapper the CLI generated with tool capture wired in (every
+  // shape but `node_bridge`, `cli_subprocess` and `langgraph` defines this at
+  // module scope -- see PY_CAPTURE in eval_templates.js). A wrapper approved
+  // before that change has no such line and keeps matching its digest, so this
+  // is read from the file rather than the approval record.
+  const capturesTools = /^_TOOLS = /m.test(source);
+
   // Every file the approval produced, not only the wrapper.
   //
   // A `node_bridge` target is two files: the Python wrapper and the `.mjs` it
@@ -246,6 +264,7 @@ export function verifyEntry(saved) {
     sysPath,
     sha256: digest,
     files: extra,
+    capturesTools,
     shape: saved.shape || "",
     language: saved.language,
     framework: saved.framework,