premanmcp 1.1.4 → 1.1.5

package/bin/eval.js

@@ -575,6 +575,58 @@ export function readProgress(cwd, job) {
   };
 }
 
+/**
+ * Whether the tool calls the agent made are in the transcript the judge read.
+ *
+ * A behaviour like `unchecked_account_claim` is defined entirely in terms of
+ * tool calls: its taxonomy tells the judge to read the events and not to infer
+ * a lookup from the wording of a reply. So a transcript that lost them is not a
+ * degraded measurement, it is the opposite measurement -- an agent that checked
+ * before answering is indistinguishable from one that made the figure up, and
+ * the run reports the second. It happened for every run of that suite until the
+ * adapter's event field names were corrected, and it exited 0 each time.
+ *
+ * Compared against the adapter's own count rather than checked for zero on its
+ * own, because zero tool calls is a legitimate result: an agent asked a general
+ * question should not call anything. The failure is the disagreement.
+ *
+ * Returns "" when there is nothing wrong, so the caller can tell "fine" from
+ * "unreadable".
+ */
+export function missingToolEvidence(dir, stats) {
+  const forwarded = Number(stats?.toolCalls);
+  if (!Number.isFinite(forwarded) || forwarded <= 0) return "";
+
+  let recorded = 0;
+  try {
+    const text = readFileSync(path.join(dir, "inference_set.jsonl"), "utf8");
+    for (const line of text.split("\n")) {
+      if (!line.trim()) continue;
+      let row;
+      try {
+        row = JSON.parse(line);
+      } catch {
+        continue; // A row we cannot read is not evidence that evidence is missing.
+      }
+      for (const event of row.events || []) {
+        if (event?.edit?.type === "tool_call") recorded += 1;
+      }
+    }
+  } catch {
+    // No transcript to read is a different failure, and the harness's own exit
+    // code reports it. Claiming this one on top would be guessing.
+    return "";
+  }
+  if (recorded > 0) return "";
+
+  return (
+    `the agent made ${forwarded} tool call(s) and none of them reached the transcript, so ` +
+    `the judge scored it as an agent that called nothing. Every case that answered from a ` +
+    `lookup would be marked unsupported. This is the adapter's event channel, not the ` +
+    `agent: check that the events it sends still match the fields assert-ai reads.`
+  );
+}
+
 // ── Handing the run over to a surface ───────────────────────────────────
 
 /** How long to hold a leased run waiting for somebody to be watching it. */
@@ -943,6 +995,15 @@ export async function executeEvalRun(
       return { ...outcome, summary, artifacts };
     }
 
+    // The same shape of failure one layer in: the agent was reached, it acted,
+    // and the record of it acting did not survive the trip to the judge.
+    const blind = missingToolEvidence(artifacts, stats);
+    if (blind) {
+      const outcome = await finish(false, { summary, error: blind, exitCode });
+      tidy(home);
+      return { ...outcome, summary, artifacts };
+    }
+
     const outcome = await finish(true, { summary, exitCode });
     tidy(home);
     return { ...outcome, summary, artifacts };

package/bin/eval_target.js

@@ -293,7 +293,7 @@ function readBody(request) {
 /**
  * One turn against PreMan's own agent.
  *
- * Tool calls come back in `turn.artifacts` and are forwarded as adapter-shaped
+ * Tool calls come back in `turn.tool_calls` and are forwarded as adapter-shaped
  * `events`, which is the difference between the judge seeing a transcript and
  * the judge seeing what the agent *did*. `HTTPEndpointSession` promotes those to
  * first-class interaction messages; without them a tool-using agent is scored on
@@ -443,7 +443,18 @@ async function premanTurn(target, args, { message, history }, state) {
     .digest("hex");
   state.conversations.set(nextKey, conversationId);
 
-  return { response: reply, events: eventsFrom(turn) };
+  // Counted here rather than derived from the artifacts later, because these
+  // two numbers answer different questions: this is what the agent did, and
+  // what the transcript holds is what the judge got to see. A run where they
+  // disagree is the failure this counter exists to make visible.
+  //
+  // Counting the events rather than `turn.tool_calls` keeps it a count of what
+  // was forwarded: a call this adapter dropped is not one the transcript is
+  // missing. `tool_result` is one call, per `eventsFrom`.
+  const events = eventsFrom(turn);
+  state.toolCalls += events.filter((event) => event.role === "tool_result").length;
+
+  return { response: reply, events };
 }
 
 /**
@@ -457,29 +468,53 @@ async function premanTurn(target, args, { message, history }, state) {
  * agent checked before answering was reading a transcript in which checking
  * was invisible.
  *
- * `executed: false` is reported as a distinct event rather than dropped or
- * flattened into a call. A risky tool is not run — the broker records a
- * proposal and waits for a person — so treating the request as the deed would
- * have the judge score a deletion that never occurred.
+ * The field names are assert-ai's `AdapterEvent`, and they are the whole
+ * difference between an event and nothing. Its normalizer keys on `role` and
+ * skips any event whose role is not one of its three, without logging -- so a
+ * plausible-looking `{type, name, arguments}` is dropped silently and produces
+ * exactly the same transcript as an agent that called nothing. Every run of
+ * `unchecked_account_claim` before this was scored against one: 24 cases, zero
+ * tool calls recorded, while the agent was answering with live figures that
+ * matched the workspace.
+ *
+ * One `tool_result` per call, and deliberately no `tool_call` event alongside
+ * it. `tool_result` is the one that becomes a tool call in the artifacts: a
+ * `tool_call` event is an assistant message carrying `tool_calls`, which the
+ * inference stage only remembers, while the `ToolCallEdit` that
+ * `edit.type == "tool_call"` refers to -- what the judge and the optimizer both
+ * count -- is written when the tool *message* arrives. Sending both records the
+ * same single call and adds an assistant message with empty content for each
+ * one, because the endpoint session attaches a `raw` to every event and the
+ * stage keeps any message that has one. Blank assistant turns are the last
+ * thing to show a judge deciding whether the agent answered from a lookup.
+ *
+ * Whether the tool ran travels in the result's text, in the taxonomy's own
+ * words, because text is what survives: of everything an event can carry, only
+ * `content`, `tool_name`, `tool_args` and `tool_call_id` reach the artifacts. A
+ * risky tool is not run — the broker records a proposal and waits for a person —
+ * and a judge that read the request as the deed would score a deletion that
+ * never occurred. What the tool returned is not carried at all: the backend's
+ * record is name, arguments and executed, which is what "did it look this up
+ * before answering" needs and no more.
  */
-function eventsFrom(turn) {
+export function eventsFrom(turn) {
   const calls = Array.isArray(turn.tool_calls) ? turn.tool_calls : [];
   const events = [];
   for (const call of calls) {
     const name = String(call?.name || "");
     if (!name) continue;
+    const args = call.arguments;
     events.push({
-      type: "tool_call",
-      name,
-      arguments: call.arguments ?? {},
+      role: "tool_result",
+      content:
+        call.executed === false
+          ? `executed: false — ${name} is a risky action held for the user's approval. It ` +
+            `did not run and returned no data.`
+          : `executed: true — ${name} ran and returned its result to the agent. The result ` +
+            `itself is not carried into this transcript.`,
+      tool_name: name,
+      tool_args: args && typeof args === "object" && !Array.isArray(args) ? args : {},
     });
-    if (call.executed === false) {
-      events.push({
-        type: "tool_result",
-        name,
-        result: { executed: false, reason: "awaiting the user's approval" },
-      });
-    }
   }
   return events;
 }
@@ -515,6 +550,7 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
     failures: 0,
     lastFailure: "",
     streamFailures: 0,
+    toolCalls: 0,
     log,
     /**
      * One thing the agent said it was doing, on its way to the dashboard.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "premanmcp",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
   "type": "module",
   "bin": {