premanmcp 1.1.0 → 1.1.1

package/bin/eval.js

@@ -34,7 +34,7 @@ import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
-import { discoverTarget, startAdapter, stepFeed } from "./eval_target.js";
+import { adapterStats, discoverTarget, startAdapter, stepFeed } from "./eval_target.js";
 import { resolveEvalSurface } from "./link.js";
 import {
   backendUrl,
@@ -799,6 +799,29 @@ export async function executeEvalRun(
   }
 
   if (exitCode === 0) {
+    // A harness that exited 0 having never once reached the agent is not a
+    // result, and it is the most misleading shape a run can take: every case
+    // was scored, the numbers look like measurements, and what the judge
+    // actually read was the adapter's apology on each turn. The one that
+    // prompted this presented an `api.preman.live` key to an agent on
+    // localhost, 401ed eighteen times, and finished ok.
+    //
+    // Turns are counted rather than failures, because a run where the agent
+    // answered some of the time is a real measurement of a flaky agent and
+    // deciding otherwise here would throw away the finding.
+    const stats = adapterStats();
+    if (stats && stats.turns === 0 && stats.failures > 0) {
+      const outcome = await finish(false, {
+        summary,
+        error:
+          `not one of the ${stats.failures} turns reached the agent, so nothing here was ` +
+          `measured against it. The last reason was: ${stats.lastFailure || "unknown"}`,
+        exitCode,
+      });
+      rmSync(scratch, { recursive: true, force: true });
+      return { ...outcome, summary, artifacts };
+    }
+
     const outcome = await finish(true, { summary, exitCode });
     rmSync(scratch, { recursive: true, force: true });
     return { ...outcome, summary, artifacts };

package/bin/eval_target.js

@@ -35,7 +35,46 @@
 import { createServer } from "node:http";
 import { createHash } from "node:crypto";
 
-import { backendUrl, resolveApiKey } from "./shared.js";
+import { backendUrl, resolveApiKey, storedKeyOrigin } from "./shared.js";
+
+/**
+ * The credential to present to the agent under test.
+ *
+ * When the agent *is* a PreMan deployment, this is a PreMan key -- but not
+ * necessarily the one this CLI reports results with. Those are two different
+ * systems that happen to be the same product: the control plane is whichever
+ * deployment issued the saved key, and the agent is whatever answered on this
+ * machine. A run that presented an `api.preman.live` key to a backend on
+ * localhost 401ed on all eighteen turns and still reported a score, because the
+ * adapter turns a failure into a transcript the judge can read.
+ *
+ * Refusing beats warning. Every turn would fail, the whole run would be scored
+ * against the adapter's apology, and it would cost a full round of model calls
+ * to arrive at a number about nothing.
+ */
+function agentToken(args, baseUrl, log) {
+  const named = args.value("--agent-key", process.env.PREMAN_AGENT_KEY || "");
+  if (named) return named;
+
+  const key = resolveApiKey(args);
+  const origin = storedKeyOrigin(args);
+  if (!origin || sameDeployment(origin, baseUrl)) return key;
+
+  throw new TargetError(
+    `the agent here is ${baseUrl}, but the only PreMan key on this machine was issued by ` +
+      `${origin}. It would be refused on every turn and the run would score the refusals. ` +
+      `Set PREMAN_AGENT_KEY to a key ${baseUrl} accepts, or stop that agent so this measures ` +
+      `${origin} instead.`
+  );
+}
+
+function sameDeployment(a, b) {
+  try {
+    return new URL(a).host === new URL(b).host;
+  } catch {
+    return false;
+  }
+}
 
 export class TargetError extends Error {}
 
@@ -130,7 +169,7 @@ export async function discoverTarget(args, { log = () => {} } = {}) {
         // The agent's own credential, held here and never sent to PreMan's
         // control plane. The adapter is the only thing that ever presents it,
         // which is the whole reason the adapter runs on this machine.
-        token: resolveApiKey(args),
+        token: agentToken(args, `http://127.0.0.1:${port}`, log),
         label: `PreMan agent (local, port ${port})`,
         how: `probed localhost and got a PreMan /health on port ${port}`,
         probed,
@@ -146,7 +185,7 @@ export async function discoverTarget(args, { log = () => {} } = {}) {
     return {
       kind: "preman-chat",
       baseUrl: configured,
-      token: resolveApiKey(args),
+      token: agentToken(args, configured, log),
       label: `PreMan agent (${configured})`,
       how: `nothing local answered, so this is the backend the CLI is configured against`,
       probed,
@@ -463,6 +502,7 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
     conversations: new Map(),
     turns: 0,
     failures: 0,
+    lastFailure: "",
     streamFailures: 0,
     log,
     /**
@@ -500,6 +540,7 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
         reply(200, result);
       } catch (error) {
         state.failures += 1;
+        state.lastFailure = error.message;
         // 200 with an error string in `response`, not a 5xx. A transport failure
         // aborts the case and loses it; an agent that said something unhelpful is
         // a transcript the judge can score. Which of those this is depends on
@@ -519,12 +560,32 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
   const url = `http://127.0.0.1:${port}/chat`;
   log(`Adapter listening on ${url} -> ${target.label}`);
 
+  activeStats = state;
   return {
     url,
     translated: true,
     stats: state,
     close: async () => {
+      activeStats = null;
       await new Promise((resolve) => server.close(resolve));
     },
   };
 }
+
+/**
+ * The counters for the adapter this process is running, if it is running one.
+ *
+ * A module-level handle rather than an argument threaded down, because the
+ * adapter is started by the command and the counters are needed by
+ * `executeEvalRun`, which the runner loop calls — and the loop sits between
+ * them knowing nothing about either. There is at most one adapter per process:
+ * it is opened once for the whole invocation and the runs share it.
+ *
+ * Null when the target is a URL the customer named, which the CLI does not
+ * mediate and therefore cannot count.
+ */
+let activeStats = null;
+
+export function adapterStats() {
+  return activeStats;
+}

package/bin/shared.js

@@ -235,6 +235,20 @@ function activeLogin(args) {
   return apiKeyIsExplicit(args) ? null : readStoredCredentials();
 }
 
+/**
+ * Which deployment the saved key was minted against, or "" if that is unknown.
+ *
+ * Unknown is the honest answer for a key given on the command line or in the
+ * environment: nothing records where it came from, and guessing would be worse
+ * than admitting it. Callers presenting a key to something other than PreMan's
+ * own control plane need this to tell "this will work" from "this is a key for
+ * somewhere else".
+ */
+export function storedKeyOrigin(args) {
+  const login = activeLogin(args);
+  return login?.backend_url || "";
+}
+
 /**
  * Which PreMan deployment this invocation talks to.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "premanmcp",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
   "type": "module",
   "bin": {