premanmcp 1.0.4 → 1.1.0

package/bin/eval.js

@@ -0,0 +1,1200 @@
+/**
+ * Running one agent eval on this machine, for a run the backend leased here.
+ *
+ * The reason this exists rather than the eval running on PreMan's own workers:
+ * a developer's agent is on `localhost`, and PreMan refusing to fetch private
+ * addresses on an authenticated caller's say-so is correct, not an obstacle to
+ * work around. So the eval comes to the agent. The device holds the same
+ * outbound stream the coding-agent runner already holds, and the backend leases
+ * eval runs down it under a distinct event name.
+ *
+ * What arrives is a whole run, not a path: the spec plus the bytes it refers to,
+ * with every path already rewritten relative to a bundle root
+ * (`flowtest/services/eval_runner_bundle.py`). Nothing in the frame names a
+ * directory on a server, so nothing here has to resolve one.
+ *
+ * Non-negotiables, all of them borrowed from the coding-agent runner because a
+ * second set of rules for the same protocol is a second thing to get wrong:
+ *   - Process exit never resolves a run. Only a completion callback holding the
+ *     current lease does.
+ *   - Every callback carries the fencing token. A 409 means this device lost the
+ *     run — cancelled, or taken over — and the only correct response is to stop.
+ *   - The runner token never enters the child's environment.
+ *
+ * One rule is this module's own: the run executes in a scratch directory, never
+ * in the customer's project. assert-ai writes results under `cwd`, and a harness
+ * that scattered `artifacts/` into somebody's repository would be a bad guest
+ * even before the first `git status`.
+ */
+
+import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { discoverTarget, startAdapter, stepFeed } from "./eval_target.js";
+import { resolveEvalSurface } from "./link.js";
+import {
+  backendUrl,
+  callBackendJson,
+  cliInvocation,
+  describeFailure,
+  openUrl,
+  packageVersion,
+  resolveApiKey,
+} from "./shared.js";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+/** The shim the run is executed through. See `harnessDrift` for why it is a copy. */
+export const HARNESS_PATH = path.join(__dirname, "eval_harness.py");
+
+/**
+ * The four stages assert-ai runs, in order.
+ *
+ * Named here because the progress callback's `stage` is a closed enum on the
+ * server: free text would let this file invent a fifth pipeline step that the
+ * dashboard would then have to render.
+ */
+export const EVAL_STAGES = ["systematize", "test_set", "inference", "judge"];
+
+// Beats the server's 300s lease with room for a missed tick, and slow enough
+// that a six-case run does not spend its time talking about itself. Overridable
+// because it is also the interval at which a lost lease is *noticed*, and a test
+// that had to wait a real 20s for that would be a test people skip.
+const PROGRESS_MS = Math.max(
+  200,
+  Number(process.env.PREMAN_EVAL_PROGRESS_MS) || 20_000
+);
+const RUN_TIMEOUT_MS = Number(process.env.PREMAN_EVAL_TIMEOUT_MS) || 60 * 60_000;
+const RUN_TIMEOUT_CAP_MS = 4 * 60 * 60_000;
+const ERROR_CAP = 4_000;
+const MAX_STDERR_BYTES = 256 * 1024;
+
+export class EvalError extends Error {}
+
+// ── The bundle on disk ──────────────────────────────────────────────────
+
+/**
+ * Write the leased run into a scratch directory and return what to run.
+ *
+ * Every path in the frame is checked before it is joined, not because the
+ * backend is untrusted but because "the server would never send that" is the
+ * assumption every path-traversal bug is built on. A frame that names
+ * `../../.ssh/authorized_keys` is refused here rather than written.
+ */
+export function materialize(job, root) {
+  mkdirSync(root, { recursive: true, mode: 0o700 });
+  const bundle = path.join(root, "bundle");
+  mkdirSync(bundle, { recursive: true, mode: 0o700 });
+
+  for (const entry of job.files || []) {
+    const relative = String(entry?.path || "");
+    if (!relative) throw new EvalError("the run carried a file with no path");
+    const target = path.resolve(bundle, relative);
+    if (target !== bundle && !target.startsWith(bundle + path.sep)) {
+      throw new EvalError(`refusing to write outside the run directory: ${relative}`);
+    }
+    mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
+    writeFileSync(target, String(entry.text ?? ""), { mode: 0o600 });
+  }
+
+  // The spec is written as JSON with a .yaml name, which is legal and not a
+  // shortcut: YAML is a superset of JSON, assert-ai's loader accepts it, and
+  // JSON.stringify cannot mis-quote a value the way a hand-rolled YAML emitter
+  // can. Shipping a YAML serializer to avoid a filename that looks odd would be
+  // the worse trade.
+  const configPath = path.join(bundle, "preman-eval.yaml");
+  writeFileSync(configPath, `${JSON.stringify(job.spec ?? {}, null, 2)}\n`, { mode: 0o600 });
+  return { cwd: root, bundle, configPath };
+}
+
+// ── Finding a Python that can run it ────────────────────────────────────
+
+/**
+ * What a real Python answering the probe prints, and nothing else can.
+ *
+ * The marker is here because exiting zero is not evidence. `/bin/echo -c
+ * "import assert_ai..."` exits zero and prints the program back, and a probe
+ * that took any non-empty stdout as a version would call that a working
+ * interpreter — then fail at spawn time with a message about the harness. The
+ * prefix has to be produced by executing the snippet, so echoing it back does
+ * not satisfy the check.
+ */
+const PROBE_MARKER = "preman-assert-ai=";
+const PROBE_SNIPPET = `import assert_ai,importlib.metadata as m;print("${PROBE_MARKER}"+m.version("assert-ai"))`;
+
+/**
+ * Ask an interpreter whether it can run an eval, and which version it has.
+ *
+ * Importing is the only honest test. `assert-ai` is installed from git rather
+ * than PyPI, so "is it on PATH" and "is it importable by the interpreter we are
+ * about to use" are different questions, and only the second one predicts
+ * whether the run works.
+ */
+function probeInterpreter(command, argv) {
+  const probe = spawnSync(command, [...argv, "-c", PROBE_SNIPPET], {
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "ignore"],
+    timeout: 120_000,
+  });
+  if (probe.status !== 0) return "";
+  const line = String(probe.stdout || "")
+    .split("\n")
+    .map((value) => value.trim())
+    .find((value) => value.startsWith(PROBE_MARKER));
+  return line ? line.slice(PROBE_MARKER.length).trim() : "";
+}
+
+/**
+ * Which interpreter runs this eval, in a deliberate order.
+ *
+ * `PREMAN_EVAL_PYTHON` first, because someone who set it has already answered
+ * the question and second-guessing them is how a tool becomes unusable in the
+ * one environment it was configured for.
+ *
+ * Then `uv run` inside the project, because a customer whose repository pins
+ * assert-ai wants *their* pin — the eval is measuring their agent, and the
+ * harness version is part of what makes two runs comparable.
+ *
+ * Then a bare `python3`, which is the case where somebody installed the harness
+ * globally.
+ *
+ * There is no fourth branch that installs anything. Downloading and executing a
+ * Python package on a developer's machine, unattended, because a job frame
+ * arrived, is not a thing a runner should decide to do on its own.
+ */
+export function resolveInterpreter({ projectPath = process.cwd(), log = () => {} } = {}) {
+  const override = String(process.env.PREMAN_EVAL_PYTHON || "").trim();
+  if (override) {
+    const version = probeInterpreter(override, []);
+    if (!version) {
+      throw new EvalError(
+        `PREMAN_EVAL_PYTHON is set to ${override}, but that interpreter cannot import assert_ai.`
+      );
+    }
+    return { command: override, argv: [], version, how: "PREMAN_EVAL_PYTHON" };
+  }
+
+  if (existsSync(path.join(projectPath, "pyproject.toml"))) {
+    const version = probeInterpreter("uv", ["run", "--project", projectPath, "python"]);
+    if (version) {
+      return {
+        command: "uv",
+        argv: ["run", "--project", projectPath, "python"],
+        version,
+        how: "uv run in this project",
+      };
+    }
+  }
+
+  const version = probeInterpreter("python3", []);
+  if (version) return { command: "python3", argv: [], version, how: "python3 on PATH" };
+
+  throw new EvalError(
+    "no Python on this machine can import assert_ai, so this eval cannot run here. " +
+      "Add assert-ai to the project this runner is paired with, or point " +
+      "PREMAN_EVAL_PYTHON at an interpreter that has it."
+  );
+}
+
+/**
+ * The child's environment: inherited, minus this runner's own credentials.
+ *
+ * The provider keys are *not* stripped. That is the point of running here: the
+ * eval is paid for by the customer's own key, whether they exported it in this
+ * shell or saved it in the dashboard for the server to hand back — either way
+ * the spend is theirs and the transcripts stay on their machine. The runner
+ * token is a different matter: it authorizes reporting results for any of this
+ * device's runs, and a harness has no use for it.
+ */
+/**
+ * The provider key this spec needs and this machine does not have, if any.
+ *
+ * Mirrors `loop.invoke.PROVIDER_KEYS` server-side. Only the first key of each
+ * pair is required: `ANTHROPIC_WORKSPACE_ID` and the Azure bases are settings
+ * for people who need them, not credentials every run must carry, and demanding
+ * them here would refuse runs that would have worked.
+ */
+const PROVIDER_ENV = {
+  "anthropic/": "ANTHROPIC_API_KEY",
+  "openai/": "OPENAI_API_KEY",
+  "deepseek/": "DEEPSEEK_API_KEY",
+  "gemini/": "GEMINI_API_KEY",
+  "cohere/": "COHERE_API_KEY",
+  "mistral/": "MISTRAL_API_KEY",
+  "groq/": "GROQ_API_KEY",
+};
+
+const PROVIDER_NAMES = {
+  "anthropic/": "Anthropic",
+  "openai/": "OpenAI",
+  "deepseek/": "DeepSeek",
+  "gemini/": "Gemini",
+  "cohere/": "Cohere",
+  "mistral/": "Mistral",
+  "groq/": "Groq",
+};
+
+function modelNames(node, out = []) {
+  if (Array.isArray(node)) {
+    for (const item of node) modelNames(item, out);
+  } else if (node && typeof node === "object") {
+    for (const [key, value] of Object.entries(node)) {
+      if (key === "name" && typeof value === "string") out.push(value);
+      else modelNames(value, out);
+    }
+  }
+  return out;
+}
+
+export function missingModelKey(job, env = process.env) {
+  const supplied = job.env || {};
+  const seen = new Set();
+  for (const name of modelNames(job.spec || {})) {
+    for (const [prefix, variable] of Object.entries(PROVIDER_ENV)) {
+      if (!name.startsWith(prefix) || seen.has(variable)) continue;
+      seen.add(variable);
+      if (env[variable] || supplied[variable]) continue;
+      return (
+        `this run needs a model key for ${PROVIDER_NAMES[prefix]} and there isn't one. Save ` +
+        `yours once under Settings → Model keys and every machine on this account picks it ` +
+        `up, or export ${variable} in this terminal for a one-off.`
+      );
+    }
+  }
+  return "";
+}
+
+export function childEnv(extra = {}, defaults = {}) {
+  const env = { ...process.env, ...extra };
+  for (const key of ["PREMAN_API_KEY", "PREMAN_RUNNER_TOKEN", "PREMAN_PAIR_CODE"]) {
+    delete env[key];
+  }
+  // Underneath this shell rather than over it. Someone who exported a key in
+  // the terminal they ran this from is doing it on purpose — a different
+  // account, or a key the dashboard has never seen — and a value saved months
+  // ago should not silently win against the one they just typed. An empty
+  // string does not count as having typed anything.
+  for (const [key, value] of Object.entries(defaults)) {
+    if (!env[key] && value) env[key] = value;
+  }
+  return env;
+}
+
+// ── Reading progress off disk ───────────────────────────────────────────
+
+function countLines(file) {
+  try {
+    const text = readFileSync(file, "utf8");
+    return text.split("\n").filter((line) => line.trim()).length;
+  } catch {
+    return 0;
+  }
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Where this run's output lands.
+ *
+ * `artifacts_base()` upstream is `Path.cwd()`, so this is a consequence of the
+ * scratch directory rather than a configuration choice — which is also why the
+ * scratch directory has to be stable for the life of the run.
+ */
+export function runDir(cwd, job) {
+  return path.join(cwd, "artifacts", "results", String(job.suite), String(job.run));
+}
+
+/**
+ * The four stages as one activity list, current stage active, earlier ones done.
+ *
+ * `LocalRunnerProgressActivity` is capped at six entries and there are exactly
+ * four stages, so the whole pipeline fits and the dashboard never has to guess
+ * what a stage it has not seen before means.
+ *
+ * `elapsed_ms` is wall time for the run, not for the stage. Per-stage timing
+ * would need stage-start timestamps the artifacts do not carry, and a number
+ * assembled from when this process happened to first *notice* a stage would be
+ * wrong in a way nobody could tell from looking at it.
+ */
+export function activityFor(progress, startedAt) {
+  const at = EVAL_STAGES.indexOf(progress.stage);
+  const elapsed = Math.max(0, Date.now() - startedAt);
+  return EVAL_STAGES.slice(0, at < 0 ? 1 : at + 1).map((stage, i) => ({
+    id: `stage.${stage}`,
+    label:
+      stage === progress.stage && progress.cases_total
+        ? `${STAGE_LABEL[stage]} (${progress.cases_done || 0}/${progress.cases_total})`
+        : STAGE_LABEL[stage],
+    state: i === (at < 0 ? 0 : at) ? "active" : "complete",
+    elapsed_ms: elapsed,
+  }));
+}
+
+const STAGE_LABEL = {
+  systematize: "Reading the rubric",
+  test_set: "Writing test cases",
+  inference: "Running the agent",
+  judge: "Scoring transcripts",
+};
+
+/**
+ * The files PreMan stores for a run, in the order they become interesting.
+ *
+ * The same closed set the server enforces
+ * (`flowtest/services/eval_runner_artifacts.py`). Kept here as a list rather
+ * than "upload whatever is in the directory" for two reasons: the run directory
+ * also holds `.viewer/` and a versioned `artifacts/` tree that nothing reads
+ * back, and a device that uploaded everything would be sending megabytes the
+ * dashboard has no use for over a domestic uplink.
+ */
+export const RUN_ARTIFACTS = [
+  "manifest.json",
+  "config.yaml",
+  "inference_set.jsonl",
+  "scores.jsonl",
+  "metrics.json",
+  "artifacts.json",
+];
+
+/**
+ * Upload the artifacts that changed since last time.
+ *
+ * `(size, mtimeMs)` per file, which is the same fingerprint `RunSync` uses
+ * server-side and for the same reason: of these files two grow and they never
+ * grow at the same time, so re-sending all of them every tick would be almost
+ * entirely re-sending bytes that did not move.
+ *
+ * A failed upload is not a failed run. The subprocess is spending real money and
+ * may be most of the way through an hour; losing one copy is worse than nothing
+ * durable and far better than aborting, so the fingerprint is only recorded on
+ * success and the next tick retries.
+ */
+export async function syncArtifacts(job, cwd, sent, { call, log = () => {}, lease } = {}) {
+  const dir = runDir(cwd, job);
+  let uploaded = 0;
+  for (const name of RUN_ARTIFACTS) {
+    const file = path.join(dir, name);
+    let mark;
+    try {
+      const stat = statSync(file);
+      if (!stat.isFile() || stat.size === 0) continue;
+      mark = `${stat.size}:${stat.mtimeMs}`;
+    } catch {
+      continue;
+    }
+    if (sent.get(name) === mark) continue;
+
+    let body;
+    try {
+      body = readFileSync(file);
+    } catch (error) {
+      log(`could not read ${name}: ${error.message}`);
+      continue;
+    }
+    const form = new FormData();
+    form.set("lease_token", lease);
+    form.set("name", name);
+    form.set("file", new Blob([body]), name);
+    const result = await call(`/workbench/coding-agent/local-runner/evals/${job.id}/artifacts`, form);
+    if (result.status_code === 409) return { uploaded, lost: true };
+    if (!result.ok) {
+      log(`upload of ${name} answered ${result.status_code}; will retry`);
+      continue;
+    }
+    // Recorded from the stat taken before the read, so a file that grew
+    // mid-upload is re-sent next tick rather than recorded as complete.
+    sent.set(name, mark);
+    uploaded += 1;
+  }
+  return { uploaded, lost: false };
+}
+
+/**
+ * What the run is doing, read from its own artifacts rather than its stdout.
+ *
+ * assert-ai maintains `manifest.json` with a status per stage and appends one
+ * row to `inference_set.jsonl` per finished case. Parsing that is stable in a
+ * way that parsing console output is not: the log lines are Rich-formatted for
+ * a human and are free to change, while the manifest is the file the CLI's own
+ * `results status` command reads.
+ *
+ * Returns null when there is nothing to say yet, so a caller can tell "no
+ * progress" from "progress of zero".
+ */
+export function readProgress(cwd, job) {
+  const dir = runDir(cwd, job);
+  const manifest = readJson(path.join(dir, "manifest.json"));
+  if (!manifest) return null;
+
+  const stages = manifest.stages && typeof manifest.stages === "object" ? manifest.stages : {};
+  // Last stage the manifest mentions, in pipeline order. A stage that has
+  // finished still names itself, so the newest mentioned stage is the furthest
+  // the run has got — which is what somebody watching wants to know.
+  let stage = "";
+  for (const name of EVAL_STAGES) {
+    if (stages[name]) stage = name;
+  }
+  if (!stage) return null;
+
+  const done = countLines(path.join(dir, "inference_set.jsonl"));
+  const scored = countLines(path.join(dir, "scores.jsonl"));
+  return {
+    stage,
+    cases_done: stage === "judge" ? scored : done,
+    cases_total: done > 0 ? done : undefined,
+  };
+}
+
+// ── Handing the run over to a surface ───────────────────────────────────
+
+/** How long to hold a leased run waiting for somebody to be watching it. */
+export const AUDIENCE_WAIT_MS = envMs("PREMAN_EVAL_AUDIENCE_MS", 90_000, 1_000);
+
+/** How often to ask, while waiting. */
+const AUDIENCE_POLL_MS = envMs("PREMAN_EVAL_AUDIENCE_POLL_MS", 3_000, 250);
+
+function envMs(name, fallback, floor) {
+  const raw = Number.parseInt(process.env[name] || "", 10);
+  return Number.isFinite(raw) && raw >= floor ? raw : fallback;
+}
+
+/**
+ * Refuse to be the second eval running out of this directory.
+ *
+ * A device is identified by hostname and project path, so two of these started
+ * in the same checkout are one device as far as PreMan is concerned. The second
+ * registration overwrites the adapter URL the first advertised, and runs leased
+ * afterwards are sent to whichever port registered last -- so when either
+ * process exits, the other's cases arrive at a closed socket and the run records
+ * `cases_did_not_reach_the_agent` against the agent, which did nothing wrong.
+ * Several runs in this account's history died exactly that way.
+ *
+ * A lock file rather than a server-side check because the collision is between
+ * two processes on one machine, and the machine can answer it without a
+ * round-trip. Stale locks are ignored by asking the OS whether the pid is still
+ * there, so a crashed run does not wedge the directory.
+ */
+function claimEvalLock(projectPath) {
+  const lock = path.join(os.homedir(), ".preman", `eval-${createHash("sha256").update(path.resolve(projectPath)).digest("hex").slice(0, 16)}.pid`);
+  try {
+    const held = Number(readFileSync(lock, "utf8").trim());
+    if (Number.isInteger(held) && held > 0 && held !== process.pid) {
+      try {
+        process.kill(held, 0);
+        throw new EvalError(
+          `an eval is already running in this directory (pid ${held}). Two at once share ` +
+            `one device identity and take each other's cases offline — wait for it, or stop ` +
+            `it with \`kill ${held}\`.`
+        );
+      } catch (error) {
+        // ESRCH means the holder is gone and the lock is litter. Anything else
+        // is this function's own refusal, on its way up.
+        if (error instanceof EvalError) throw error;
+      }
+    }
+  } catch (error) {
+    if (error instanceof EvalError) throw error;
+    // No lock file, or an unreadable one. Either way it is ours to take.
+  }
+
+  mkdirSync(path.dirname(lock), { recursive: true });
+  writeFileSync(lock, `${process.pid}\n`, { mode: 0o600 });
+  const release = () => rmSync(lock, { force: true });
+  process.once("exit", release);
+  return release;
+}
+
+// Set once the wait has been spent in full with nobody arriving. Process-wide
+// on purpose: it is a fact about whoever ran this command, and it outlives the
+// run that discovered it.
+let audienceDeclined = false;
+
+export function resetAudience() {
+  audienceDeclined = false;
+}
+
+/**
+ * Open the dashboard on this run, then wait a bounded time for it to be reading.
+ *
+ * Why wait at all: a device-executed eval's whole advantage over reading
+ * artifacts afterwards is watching it happen, and the first stage does real work
+ * within seconds of starting. Beginning into a browser that has not finished
+ * loading means the interesting part is already history by the time anyone
+ * looks.
+ *
+ * Why bounded, and why it proceeds anyway: the watcher signal is a recency
+ * reading from a polling client, not a subscription. It is wrong in both
+ * directions — a slow tab reads as absent, and a background tab reads as
+ * present — so treating a "no" as a reason to refuse work would let a closed
+ * browser fail an eval the customer is paying for. The wait buys the common
+ * case and gives up on the rest, out loud.
+ *
+ * `headless` skips all of it, and is the default. Only an interactive
+ * `preman eval run` asks for the handoff: `preman runner` is a background daemon
+ * whose whole job is to execute what arrives, and pausing it to look for a
+ * browser would stall work nobody is sitting in front of.
+ */
+export async function awaitAudience(job, { call, log, lease, headless = true, surface = null } = {}) {
+  if (headless) return { watching: false, waited: 0, skipped: "headless" };
+
+  // Asked once, nobody came. "Is a person watching" is a fact about the person,
+  // not about the run, so re-asking it for every run in a queue spends the full
+  // window again on an answer already given -- seven behaviours meant ten
+  // minutes of waiting and the same five lines printed seven times, which reads
+  // as the thing running in circles.
+  if (audienceDeclined) {
+    if (surface?.webUrl) log(`Watch it at ${surface.webUrl}`);
+    return { watching: false, waited: 0, skipped: "declined-earlier" };
+  }
+
+  if (surface) {
+    // Best-effort by construction. `resolveEvalSurface` already narrowed to
+    // whichever of desktop, browser or plain text this machine supports, and
+    // even the desktop branch cannot confirm the window came forward — an older
+    // build ignores the route and lands wherever it was. So the URL is always
+    // printed, and "opened" is never claimed.
+    if (surface.webUrl) log(`Watch it at ${surface.webUrl}`);
+    try {
+      surface.open?.();
+      if (surface.surface === "desktop-raise") {
+        // Said plainly because the window will come forward showing whatever it
+        // was showing, and somebody who expected it to land on the run would
+        // otherwise read "not this" as "it did not work".
+        log("Brought the PreMan app forward. This build cannot jump to a view, so open Agent eval yourself.");
+      }
+    } catch (error) {
+      log(`could not open a window here (${error.message}); the link above still works`);
+    }
+  }
+
+  const until = Date.now() + AUDIENCE_WAIT_MS;
+  let asked = 0;
+  while (Date.now() < until) {
+    const result = await report(call, `/workbench/coding-agent/local-runner/evals/${job.id}/watchers`, { lease_token: lease }, log);
+    if (result.status_code === 409) return { lost: true };
+    if (result.watching) {
+      log("Dashboard is watching; starting the eval.");
+      return { watching: true, waited: asked * AUDIENCE_POLL_MS };
+    }
+    if (asked === 0) log("Waiting for the dashboard to open before starting…");
+    asked += 1;
+    await sleep(AUDIENCE_POLL_MS);
+  }
+
+  audienceDeclined = true;
+  log(
+    `Nothing has opened this run in ${Math.round(AUDIENCE_WAIT_MS / 1000)}s. Starting anyway — ` +
+      `results are stored as they are produced, so opening the link later loses nothing except ` +
+      `seeing it live. Anything else queued here will start straight away.`
+  );
+  return { watching: false, waited: AUDIENCE_WAIT_MS };
+}
+
+// ── Execution ───────────────────────────────────────────────────────────
+
+/** POST a runner callback. Returns `{ok, status_code}`; never throws. */
+async function report(call, route, body, log) {
+  try {
+    const result = await call(route, body);
+    if (!result.ok) log(`callback ${route} -> ${result.status_code} ${result.detail || ""}`);
+    return result;
+  } catch (error) {
+    log(`callback ${route} failed: ${error.message}`);
+    return { ok: false, status_code: 0 };
+  }
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Run one leased eval to a terminal callback.
+ *
+ * `call` is injected rather than imported so this is testable without a
+ * backend, and so the runner keeps the single place that knows how to attach a
+ * runner token to a request.
+ *
+ * The 409 handling is the part worth reading. A lost lease is not an error to
+ * retry: it means the run was cancelled or another attempt owns it, and the
+ * subprocess is spending the customer's tokens on a result nobody will accept.
+ * So the process is killed and no completion is sent — sending one would be this
+ * device reporting on a run it does not own.
+ */
+export async function executeEvalRun(
+  job,
+  { call, log = () => {}, projectPath = process.cwd(), headless = true, surface = null } = {}
+) {
+  const lease = String(job.lease_token || "");
+  if (!lease) {
+    log(`eval ${job.id} arrived without a lease token; ignoring`);
+    return { ok: false, reason: "no_lease" };
+  }
+  const route = (leaf) => `/workbench/coding-agent/local-runner/evals/${job.id}/${leaf}`;
+
+  const finish = async (ok, { summary = {}, error = "", exitCode = null } = {}) => {
+    const result = await report(
+      call,
+      route("complete"),
+      { lease_token: lease, ok, summary, error: error ? error.slice(0, ERROR_CAP) : undefined, exit_code: exitCode },
+      log
+    );
+    return { ok, reason: result.ok ? "reported" : "report_failed" };
+  };
+
+  let scratch = "";
+  let laid;
+  let python;
+  try {
+    scratch = mkdtempSync(path.join(os.tmpdir(), "preman-eval-"));
+    laid = materialize(job, scratch);
+    python = resolveInterpreter({ projectPath, log });
+    log(`eval ${job.id}: assert-ai ${python.version} via ${python.how}`);
+  } catch (error) {
+    // Every one of these is a fact about this machine or this frame, and no
+    // amount of retrying changes it. Failing with the reason is more use than
+    // letting the lease lapse and having the server call it a lost device.
+    log(`eval ${job.id} cannot run here: ${error.message}`);
+    if (scratch) rmSync(scratch, { recursive: true, force: true });
+    return finish(false, { error: error.message });
+  }
+
+  if (!existsSync(HARNESS_PATH)) {
+    rmSync(scratch, { recursive: true, force: true });
+    return finish(false, { error: `the eval harness is missing from this install (${HARNESS_PATH})` });
+  }
+
+  const missingKey = missingModelKey(job);
+  if (missingKey) {
+    // Checked before the stages start rather than left to the provider client,
+    // which reports it from inside `inference` as an authentication error
+    // against a model name -- true, and no use to somebody who simply has not
+    // saved a key anywhere.
+    rmSync(scratch, { recursive: true, force: true });
+    return finish(false, { error: missingKey });
+  }
+
+  const handoff = await awaitAudience(job, { call, log, lease, headless, surface });
+  if (handoff.lost) {
+    rmSync(scratch, { recursive: true, force: true });
+    return { ok: false, reason: "lease_lost" };
+  }
+
+  // Before the harness starts, not after it ends: a run that dies mid-way would
+  // otherwise leave its last steps to appear under whichever run came next.
+  stepFeed.reset();
+
+  const timeout = Math.min(RUN_TIMEOUT_MS, RUN_TIMEOUT_CAP_MS);
+  const child = spawn(python.command, [...python.argv, HARNESS_PATH, "run", "--config", laid.configPath], {
+    cwd: laid.cwd,
+    env: childEnv({ PREMAN_EVAL_RUN: `${job.suite}/${job.run}` }, job.env || {}),
+    stdio: ["ignore", "ignore", "pipe"],
+  });
+
+  let stderr = "";
+  child.stderr.on("data", (chunk) => {
+    if (stderr.length < MAX_STDERR_BYTES) stderr += chunk.toString("utf8");
+  });
+
+  let lost = false;
+  let lastStage = "";
+  let ticking = false;
+  const sent = new Map();
+  const started = Date.now();
+
+  const lose = (why) => {
+    lost = true;
+    log(`eval ${job.id} is no longer leased to this device (${why}); stopping`);
+    child.kill("SIGTERM");
+  };
+
+  const heartbeat = setInterval(() => {
+    // One tick at a time. Uploading a large inference_set.jsonl can outlast the
+    // interval, and overlapping ticks would send the same file twice and race
+    // each other's fingerprints.
+    if (ticking || lost) return;
+    ticking = true;
+    void (async () => {
+      try {
+        const progress = readProgress(laid.cwd, job) || { stage: "systematize" };
+        const result = await report(
+          call,
+          route("progress"),
+          {
+            lease_token: lease,
+            stage: progress.stage,
+            cases_done: progress.cases_done,
+            cases_total: progress.cases_total,
+            // Stages first, then whatever the agent has narrated since the last
+            // tick. The stages come from artifacts on disk and are true of the
+            // run; the steps come from the agent's own mouth and are true of
+            // this moment. Both are liveness and neither is scored.
+            activity: [...activityFor(progress, started), ...stepFeed.recent()],
+          },
+          log
+        );
+        if (result.status_code === 409) return lose("cancelled or taken over");
+        if (progress.stage !== lastStage) {
+          lastStage = progress.stage;
+          log(`eval ${job.id}: ${progress.stage}`);
+        }
+
+        // After the progress call, so a device that has lost the lease finds out
+        // before it spends an uplink on artifacts nobody will accept.
+        const synced = await syncArtifacts(job, laid.cwd, sent, { call, log, lease });
+        if (synced.lost) return lose("lease rejected an upload");
+      } finally {
+        ticking = false;
+      }
+    })();
+  }, PROGRESS_MS);
+
+  const killer = setTimeout(() => {
+    log(`eval ${job.id} exceeded ${Math.round(timeout / 60_000)} minutes; stopping it`);
+    child.kill("SIGKILL");
+  }, timeout);
+
+  const exitCode = await new Promise((resolve) => {
+    child.on("error", (error) => {
+      stderr += `\n${error.message}`;
+      resolve(-1);
+    });
+    child.on("close", (code) => resolve(code === null ? -1 : code));
+  });
+  clearInterval(heartbeat);
+  clearTimeout(killer);
+
+  // Deliberately before the scratch directory is removed: the summary is read
+  // off the disk the run just wrote to.
+  const summary = readJson(path.join(runDir(laid.cwd, job), "metrics.json")) || {};
+  const artifacts = runDir(laid.cwd, job);
+
+  if (lost) {
+    // No completion callback and no final sync. The run belongs to something
+    // else now, and this device reporting a result for it — or writing over its
+    // artifacts — is the exact thing the fencing token exists to prevent.
+    rmSync(scratch, { recursive: true, force: true });
+    return { ok: false, reason: "lease_lost" };
+  }
+
+  // The last sync, and the one that matters most: metrics.json and the final
+  // rows of scores.jsonl are all written after the last heartbeat fired, so
+  // without this the durable copy of a run is always missing its ending. Before
+  // the completion callback, so a run the dashboard sees as finished is one
+  // whose results are already readable.
+  const final = await syncArtifacts(job, laid.cwd, sent, { call, log, lease });
+  if (final.lost) {
+    rmSync(scratch, { recursive: true, force: true });
+    return { ok: false, reason: "lease_lost" };
+  }
+
+  if (exitCode === 0) {
+    const outcome = await finish(true, { summary, exitCode });
+    rmSync(scratch, { recursive: true, force: true });
+    return { ...outcome, summary, artifacts };
+  }
+
+  const tail = stderr.trim().split("\n").slice(-8).join(" / ");
+  const outcome = await finish(false, {
+    summary,
+    error: `the eval harness exited ${exitCode}: ${tail}`,
+    exitCode,
+  });
+  rmSync(scratch, { recursive: true, force: true });
+  return { ...outcome, summary, artifacts };
+}
+
+// ── Command ─────────────────────────────────────────────────────────────
+
+export const EVAL_HELP = `
+Eval options:
+  eval run --token <pmev_...>   Redeem a token from the dashboard: find the agent
+                                on this machine and run exactly the evals the
+                                token names
+  eval run                      Pair this project and run the eval PreMan has
+                                queued for it, then exit
+  eval doctor                   Report whether this machine can run an eval
+  --target <url>                Skip discovery; this URL is the agent. Must take
+                                POST {"message","history"} and answer {"response"}
+  --agent <name>                claude-code | cursor | codex (for pairing only)
+  --path <dir>                  Project to pair as. Defaults to cwd
+  --wait <seconds>              How long to wait for a queued eval (default 600)
+  --headless                    Do not open anything and do not wait for a
+                                dashboard before starting. Implied when stdout
+                                is not a terminal, so CI needs no flag
+`;
+
+/**
+ * Which editor this device pairs as, when `--agent` might not be naming one.
+ *
+ * `preman test --agent` uses the same word for a different job -- it says the
+ * subject is the agent rather than an endpoint, and takes no value. Reading the
+ * next token as the editor name there picks up whatever flag follows, and the
+ * column it lands in only accepts real editor names: `test --agent --only x`
+ * tried to register an editor called `--only` and failed the insert.
+ */
+function namedAgent(args) {
+  const named = args.value("--agent", "");
+  return named && !named.startsWith("-") ? named : "";
+}
+
+function pairingAgent(args) {
+  return namedAgent(args) || "cursor";
+}
+
+/**
+ * Run what this account has switched on. No dashboard, no token to paste.
+ *
+ * The token flow exists to carry a choice made in a browser to a terminal that
+ * is not authenticated as that user. This terminal is: it holds a `pm_live_`
+ * key, so the backend already knows whose account this is, and which behaviours
+ * are on is a fact about that account rather than about the window that set
+ * them. Everything the paste was carrying can therefore be asked for directly,
+ * and a token is still minted -- but as plumbing, in the two lines below,
+ * instead of something a person copies between two screens.
+ *
+ * Settings are deliberately not sent. Omitting them is what makes the backend
+ * apply the ones stored against the account, so this command measures with the
+ * numbers set on the settings page. Sending anything here -- even the defaults
+ * -- would override them and quietly make this command disagree with the page.
+ */
+async function runForThisAccount(args, deps) {
+  const say = (line) => process.stdout.write(`${line}\n`);
+
+  const key = resolveApiKey(args);
+  if (!key) {
+    throw new EvalError(
+      `no PreMan API key on this machine, so there is no account to read the ` +
+        `switched-on behaviours from. Run \`${cliInvocation()} login\`.`
+    );
+  }
+
+  const catalog = await callBackendJson(args, "GET", "/eval-behaviors", { token: key });
+  if (!catalog.ok) {
+    throw new EvalError(
+      `could not read what this account measures (${catalog.status_code}): ` +
+        `${describeFailure(catalog)}`
+    );
+  }
+
+  const enabled = (catalog.behaviors || []).filter((entry) => entry.enabled);
+  const only = args.value("--only", "").trim();
+  const chosen = only
+    ? enabled.filter((entry) => entry.id === only || entry.name === only)
+    : enabled;
+
+  if (only && chosen.length === 0) {
+    // Naming the ones that are on beats "not found": the usual cause is a
+    // behaviour that exists but is switched off, and that is a different fix
+    // from a typo.
+    const names = enabled.map((entry) => entry.id).join(", ") || "nothing";
+    throw new EvalError(`${only} is not switched on. Currently on: ${names}`);
+  }
+  if (chosen.length === 0) {
+    throw new EvalError(
+      "nothing is switched on for this account, so there is nothing to measure. " +
+        "Turn a behaviour on under Agent eval → Settings, or describe one in your " +
+        "own words there."
+    );
+  }
+
+  // One run each, and said plainly before anything is spent: assert-ai's
+  // `behavior:` is a single name, so this is N runs rather than one run
+  // covering N -- which is also N times the wait and the provider bill.
+  say(`${chosen.length} behaviour${chosen.length === 1 ? "" : "s"} switched on:`);
+  for (const entry of chosen) say(`  ${entry.title || entry.id}`);
+
+  const opened = await callBackendJson(args, "POST", "/eval-sessions", {
+    token: key,
+    json: { behavior_ids: chosen.map((entry) => entry.id) },
+  });
+  if (!opened.ok || !opened.token) {
+    throw new EvalError(
+      `could not start (${opened.status_code}): ${describeFailure(opened)}`
+    );
+  }
+
+  return runWithToken(opened.token, args, deps);
+}
+
+/**
+ * Redeem a token: find the agent, make it answerable, run what the token names.
+ *
+ * The order here is the only one that works, and each step depends on the last
+ * in a way that is easy to get wrong.
+ *
+ * The adapter has to be listening **before** the token is redeemed, because
+ * redeeming is what tells PreMan the URL to render into the spec — and PreMan
+ * renders, gates and queues the runs in that same request. A token redeemed
+ * against a port nothing is listening on produces N runs that all fail to
+ * connect, and the failure surfaces four stages later as an inference error.
+ *
+ * The runs are executed by the ordinary runner loop, once each. Not one call
+ * with a stop condition: the loop claims one run at a time by design — a device
+ * running two evals is two harnesses competing for one rate limit and one
+ * laptop — so N runs is N turns around it.
+ */
+async function runWithToken(
+  token,
+  args,
+  { runnerLoop, saveRunnerState, readRunnerState, deviceId }
+) {
+  const say = (line) => process.stdout.write(`${line}\n`);
+  const projectPath = path.resolve(args.value("--path", process.cwd()));
+  claimEvalLock(projectPath);
+
+  say("Looking for the agent to evaluate…");
+  const target = await discoverTarget(args, { log: say });
+  say(`  ${target.label}`);
+  say(`  ${target.how}`);
+
+  const adapter = await startAdapter(target, args, {
+    log: say,
+    // Straight into the feed the heartbeat drains. Not logged to the terminal:
+    // the operator running this already sees the agent's own output, and the
+    // audience for these is the dashboard.
+    onStep: (step) => stepFeed.push(step),
+  });
+  if (adapter.translated) {
+    // Set for this process so `childEnv` carries it to the harness. assert-ai
+    // refuses loopback endpoints by default and that default is right: it stops
+    // a spec fetching a private address on an authenticated caller's say-so.
+    // It is not this case. The address is a server this process opened moments
+    // ago for this run, and refusing it would refuse the only architecture in
+    // which a developer's own agent can be evaluated at all.
+    process.env.ASSERT_ALLOW_PRIVATE_ENDPOINTS = "1";
+  }
+
+  try {
+    const claimed = await callBackendJson(args, "POST", "/eval-sessions/claim", {
+      json: {
+        token,
+        device_id: deviceId(projectPath),
+        label: `${os.hostname()} (${path.basename(projectPath)})`,
+        agent: pairingAgent(args),
+        project_path: projectPath,
+        project_name: path.basename(projectPath),
+        platform: `${process.platform}-${process.arch}`,
+        app_version: packageVersion() || undefined,
+        capabilities: { agent_eval_v1: true },
+        discovery: {
+          url: adapter.url,
+          label: target.label,
+          how: target.how,
+          // The addresses that did not answer, kept because "why did it test
+          // that" is the only question a wrong target ever produces, and an
+          // answer reconstructed from memory afterwards is not one.
+          probed: (target.probed || []).map((p) => ({ url: p.url, ok: !!p.ok, why: p.why || "" })),
+        },
+      },
+    });
+    if (!claimed.ok) {
+      throw new EvalError(
+        `that token could not be redeemed (${claimed.status_code}): ${claimed.detail || "no reason given"}`
+      );
+    }
+
+    const runs = claimed.runs || [];
+    const problems = claimed.problems || [];
+    saveRunnerState({
+      runner_id: String(claimed.runner_id || ""),
+      runner_token: String(claimed.runner_token || ""),
+      agent: pairingAgent(args),
+      project_path: projectPath,
+      project_name: path.basename(projectPath),
+      backend_url: backendUrl(args),
+      device_id: deviceId(projectPath),
+      registered_at: new Date().toISOString(),
+    });
+    const state = readRunnerState();
+
+    for (const problem of problems) {
+      say(`  ! ${problem.behavior_id} could not be queued: ${problem.error}`);
+    }
+    if (!runs.length) {
+      throw new EvalError("the token was redeemed but nothing could be queued; see above");
+    }
+    if (!state) throw new EvalError("the runner identity could not be saved on this machine");
+
+    say("");
+    say(`${runs.length} eval${runs.length === 1 ? "" : "s"} queued for this machine:`);
+    for (const run of runs) say(`  ${run.suite}/${run.run}`);
+    // `open` has to be attached here, the same way the daemon does it. Without
+    // it `awaitAudience` calls `surface.open?.()` into nothing, so this path
+    // printed a URL and waited out the full timeout for a window it never asked
+    // anything to open.
+    const resolved = resolveEvalSurface(args, { runId: runs[0].id });
+    const surface = resolved ? { ...resolved, open: () => openUrl(resolved.href) } : null;
+    if (surface?.webUrl) say(`Watch it at ${surface.webUrl}`);
+    say("");
+
+    const headless = args.has("--headless") || args.has("--no-wait") || !process.stdout.isTTY;
+    let done = 0;
+    for (const run of runs) {
+      const result = await runnerLoop(args, state, {
+        once: true,
+        only: "eval",
+        headless,
+        log: (line) => say(`  ${line}`),
+      });
+      done += result.jobsRun || 0;
+      if (!result.jobsRun) {
+        say(`  stopped after ${done} of ${runs.length} (${result.reason})`);
+        break;
+      }
+    }
+
+    say("");
+    say(`Finished ${done} of ${runs.length}.`);
+    if (surface) say(`Results: ${surface.webUrl}`);
+    return { ok: done > 0, jobsRun: done, expected: runs.length };
+  } finally {
+    // Always, including on the throw paths above. The adapter holds a listening
+    // socket and a set of live conversations with the agent; leaving it up after
+    // a failed redemption would leave an unauthenticated door to somebody's
+    // agent open for as long as the shell stays alive.
+    await adapter.close();
+  }
+}
+
+/**
+ * Why `eval run` waits rather than submits.
+ *
+ * The run is chosen in the dashboard — which behaviours are enabled, which
+ * models, how much fan-out — and it is gated there, before a token is spent.
+ * A CLI that submitted its own would be a second place those decisions could be
+ * made, and the two would drift. So this end holds the stream and executes what
+ * arrives, which is also what makes the copyable command on the dashboard the
+ * whole of the setup story.
+ */
+/**
+ * `preman test --agent`: measure this account's behaviours against the agent here.
+ *
+ * The short spelling of the whole thing. Everything the dashboard handoff was
+ * carrying -- which behaviours, with which settings -- is already known from the
+ * API key in this terminal, so none of it needs to travel through a paste.
+ */
+export async function agentTestCommand(
+  commandArgs = [],
+  { makeArgs, runnerLoop, saveRunnerState, readRunnerState, deviceId } = {}
+) {
+  return runForThisAccount(makeArgs(commandArgs), {
+    runnerLoop,
+    saveRunnerState,
+    readRunnerState,
+    deviceId,
+  });
+}
+
+export async function evalCommand(
+  commandArgs = [],
+  {
+    makeArgs,
+    registerRunner,
+    readRunnerState,
+    saveRunnerState,
+    deviceId,
+    runnerLoop,
+    pairingIsLive,
+    resolveAgent = null,
+  } = {}
+) {
+  const sub = commandArgs.find((value) => !value.startsWith("-")) || "run";
+  const args = makeArgs(commandArgs);
+
+  if (sub === "doctor") {
+    const projectPath = path.resolve(args.value("--path", process.cwd()));
+    let python;
+    try {
+      python = resolveInterpreter({ projectPath });
+    } catch (error) {
+      process.stdout.write(`Cannot run evals here.\n  ${error.message}\n`);
+      return { ok: false, reason: "no_harness" };
+    }
+    process.stdout.write(
+      `Ready to run evals.\n` +
+        `  assert-ai: ${python.version} (${python.how})\n` +
+        `  harness: ${HARNESS_PATH}\n` +
+        `  project: ${projectPath}\n`
+    );
+    return { ok: true, python };
+  }
+
+  if (sub !== "run") throw new EvalError(`unknown eval subcommand: ${sub}${EVAL_HELP}`);
+
+  // The token path is a different command wearing the same name, and the
+  // difference is who chose the work. With a token the dashboard has already
+  // decided, so this end discovers the agent, redeems, and runs a known number.
+  // Without one this is a machine volunteering itself and waiting to be given
+  // something, which is the pairing flow below.
+  const token = String(args.value("--token", "") || "").trim();
+  if (token) {
+    return runWithToken(token, args, {
+      runnerLoop,
+      saveRunnerState,
+      readRunnerState,
+      deviceId,
+    });
+  }
+
+  const projectPath = path.resolve(args.value("--path", process.cwd()));
+  let state = readRunnerState();
+  if (!state) {
+    const agent = namedAgent(args) || (resolveAgent ? await resolveAgent() : "");
+    if (!agent) {
+      throw new EvalError(
+        "This machine is not paired yet, and which editor to pair as could not be worked " +
+          "out here. Say so directly: --agent cursor|claude-code|codex."
+      );
+    }
+    state = await registerRunner(args, { agent, projectPath });
+  } else if (!(await pairingIsLive(args, state))) {
+    state = await registerRunner(args, { agent: state.agent, projectPath: state.project_path });
+  }
+
+  const waitSeconds = Math.max(30, Math.min(Number(args.value("--wait", "600")) || 600, 3 * 3600));
+  const deadline = Date.now() + waitSeconds * 1000;
+
+  // CI is the case this exists for: no browser to open, no TTY to print a
+  // clickable link to, and nobody to wait for. Inferred as well as declared,
+  // because the flag is the thing people forget and a non-TTY stdout already
+  // means every other opener in this CLI has declined.
+  const headless = args.has("--headless") || args.has("--no-wait") || !process.stdout.isTTY;
+
+  process.stdout.write(
+    `Waiting for a queued eval (up to ${Math.round(waitSeconds / 60)} min).\n` +
+      `  project: ${state.project_path}\n` +
+      `  Start one from the dashboard's Agent Eval page.\n` +
+      (headless ? `  headless: results are stored, nothing will be opened here.\n` : "")
+  );
+
+  const result = await runnerLoop(args, state, {
+    once: true,
+    only: "eval",
+    headless,
+    // Waiting is bounded so an unattended terminal does not hold a paired
+    // device open forever. The bound is checked between stream reads, so it
+    // never interrupts a run that has already started.
+    stopWhen: () => Date.now() > deadline,
+    log: (line) => process.stdout.write(`${line}\n`),
+  });
+
+  if (!result.jobsRun) {
+    process.stdout.write(
+      `No eval arrived within ${Math.round(waitSeconds / 60)} min. This machine stays paired, ` +
+        `so starting one from the dashboard and running this again will pick it up.\n`
+    );
+    return { ok: false, reason: "nothing_queued" };
+  }
+  return { ok: true, reason: result.reason, jobsRun: result.jobsRun };
+}