premanmcp 1.1.7 → 1.1.9

package/bin/eval.js

@@ -44,7 +44,10 @@ import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
+import { download, progressLine } from "./desktop.js";
+import { repoEnv } from "./detect.js";
 import { adapterStats, discoverTarget, startAdapter, stepFeed } from "./eval_target.js";
+import { ensureRuntime } from "./eval_runtime.js";
 import { formatSurfaceLink, resolveEvalSurface } from "./link.js";
 import {
   backendUrl,
@@ -278,6 +281,26 @@ export function materialize(job, root) {
  * prefix has to be produced by executing the snippet, so echoing it back does
  * not satisfy the check.
  */
+/**
+ * The harness, named so that a resolver can fetch it alongside a customer's
+ * own pins.
+ *
+ * A git reference rather than a package name: PyPI serves upstream ASSERT, and
+ * PreMan's fork carries functional changes, so a customer resolving the public
+ * name would run differently-shaped code and nothing records which harness
+ * produced a result. Pinned to a commit for the same reason `pyproject.toml`
+ * pins one -- two machines must agree about what measured them.
+ *
+ * Overridable so a machine with no route to GitHub, or a contributor testing a
+ * local checkout, is not stuck.
+ */
+export const HARNESS_SPEC =
+  String(process.env.PREMAN_EVAL_HARNESS_SPEC || "").trim() ||
+  "git+https://github.com/AleWang16/ASSERT@14f4e2136ab753897ce4cb90613e1663730aeb70";
+
+/** How long a first resolve may take before it counts as a failure. */
+const COLD_RESOLVE_MS = 600_000;
+
 const PROBE_MARKER = "preman-assert-ai=";
 const PROBE_SNIPPET = `import assert_ai,importlib.metadata as m;print("${PROBE_MARKER}"+m.version("assert-ai"))`;
 
@@ -289,11 +312,11 @@ const PROBE_SNIPPET = `import assert_ai,importlib.metadata as m;print("${PROBE_M
  * about to use" are different questions, and only the second one predicts
  * whether the run works.
  */
-function probeInterpreter(command, argv) {
+function probeInterpreter(command, argv, { timeoutMs = 120_000 } = {}) {
   const probe = spawnSync(command, [...argv, "-c", PROBE_SNIPPET], {
     encoding: "utf8",
     stdio: ["ignore", "pipe", "ignore"],
-    timeout: 120_000,
+    timeout: timeoutMs,
   });
   if (probe.status !== 0) return "";
   const line = String(probe.stdout || "")
@@ -317,11 +340,23 @@ function probeInterpreter(command, argv) {
  * Then a bare `python3`, which is the case where somebody installed the harness
  * globally.
  *
- * There is no fourth branch that installs anything. Downloading and executing a
- * Python package on a developer's machine, unattended, because a job frame
- * arrived, is not a thing a runner should decide to do on its own.
+ * `runtime` is PreMan's own interpreter, already fetched, and it is ranked
+ * second rather than first for a reason worth stating: it is the right answer
+ * for the harness and the wrong one for the customer's agent. A `callable`
+ * target has the harness import their agent in-process, and a bundle that
+ * cannot import their LangGraph measures nothing. So the *caller* decides
+ * whether to offer it — `executeEvalRun` passes `""` for a callable target —
+ * and this function only ranks what it is given. See
+ * docs/EVAL_SELF_CONTAINED_RUNTIME.md.
+ *
+ * The harness is never added to a customer's declared dependencies. The runtime
+ * lives under `~/.preman` and touches nothing of theirs. The harness-alongside
+ * branch is the one exception worth stating plainly: on a project that has
+ * never been synced it creates `.venv/` and `uv.lock`, exactly as the
+ * customer's own `uv run` would, and their lock records their packages only --
+ * measured, not assumed. Nothing pins PreMan into their project.
  */
-export function resolveInterpreter({ projectPath = process.cwd(), log = () => {} } = {}) {
+export function resolveInterpreter({ projectPath = process.cwd(), log = () => {}, runtime = "" } = {}) {
   const override = String(process.env.PREMAN_EVAL_PYTHON || "").trim();
   if (override) {
     const version = probeInterpreter(override, []);
@@ -333,6 +368,14 @@ export function resolveInterpreter({ projectPath = process.cwd(), log = () => {}
     return { command: override, argv: [], version, how: "PREMAN_EVAL_PYTHON" };
   }
 
+  // Above the project check on purpose: that check keys on whatever directory
+  // the command was run from, which is how a run in `~/codex` picked up a conda
+  // interpreter that imported assert_ai and then produced nothing.
+  if (runtime) {
+    const version = probeInterpreter(runtime, []);
+    if (version) return { command: runtime, argv: [], version, how: "preman runtime" };
+  }
+
   if (existsSync(path.join(projectPath, "pyproject.toml"))) {
     const version = probeInterpreter("uv", ["run", "--project", projectPath, "python"]);
     if (version) {
@@ -343,6 +386,40 @@ export function resolveInterpreter({ projectPath = process.cwd(), log = () => {}
         how: "uv run in this project",
       };
     }
+
+    // The same project, with the harness resolved alongside it into an
+    // environment uv owns rather than one the customer does. Ranked below the
+    // branch above because a repository that pins the harness itself should
+    // keep its own pin; this is for the repository that has never heard of
+    // PreMan, which is every customer's.
+    //
+    // This is the only branch that can serve a `callable` target in a stranger's
+    // repository: their agent is imported in-process, so their dependencies and
+    // the harness have to be reachable from one interpreter, and PreMan's own
+    // bundle has none of theirs.
+    //
+    // What this does leave behind, measured on a repository that had never been
+    // synced: `.venv/` and `uv.lock` in their project -- the same two files
+    // their own `uv run` would have created. What it does *not* leave behind is
+    // any trace of the harness in `uv.lock`: the lock carried their 18 packages
+    // and no mention of assert-ai, so `--with` does not pin PreMan into a
+    // customer's project. That is the property worth protecting if this branch
+    // is ever rewritten.
+    //
+    // Given a long timeout on purpose. The first resolve on a cold machine
+    // fetches the harness's whole dependency tree, and treating a slow network
+    // as "this interpreter cannot import assert_ai" would send somebody looking
+    // for a missing package.
+    const added = ["run", "--project", projectPath, "--with", HARNESS_SPEC, "python"];
+    const resolved = probeInterpreter("uv", added, { timeoutMs: COLD_RESOLVE_MS });
+    if (resolved) {
+      return {
+        command: "uv",
+        argv: added,
+        version: resolved,
+        how: "uv run in this project, with PreMan's harness alongside it",
+      };
+    }
   }
 
   const version = probeInterpreter("python3", []);
@@ -413,10 +490,20 @@ export function missingModelKey(job, env = process.env) {
       if (!name.startsWith(prefix) || seen.has(variable)) continue;
       seen.add(variable);
       if (env[variable] || supplied[variable]) continue;
+      // Says whose key this is, because the old wording ("needs a model key
+      // for X and there isn't one") reads as a configuration slip the reader
+      // made. It is not: PreMan's own stages -- deriving the failure
+      // categories, writing the cases, judging the transcripts -- run on this
+      // provider's models whatever the agent under test is written against. A
+      // customer whose agent uses a different provider would otherwise go
+      // looking for a key they were never asked for and conclude the tool is
+      // broken.
       return (
-        `this run needs a model key for ${PROVIDER_NAMES[prefix]} and there isn't one. Save ` +
-        `yours once under Settings → Model keys and every machine on this account picks it ` +
-        `up, or export ${variable} in this terminal for a one-off.`
+        `PreMan's own evaluation stages run on ${PROVIDER_NAMES[prefix]} models, so a key ` +
+        `for ${PROVIDER_NAMES[prefix]} is needed even when your agent uses a different ` +
+        `provider. This is separate from whatever key your agent itself needs, which stays ` +
+        `on this machine. Save one under Settings → Model keys and every machine on this ` +
+        `account picks it up, or export ${variable} in this terminal for a one-off.`
       );
     }
   }
@@ -532,6 +619,11 @@ export const RUN_ARTIFACTS = [
   "scores.jsonl",
   "metrics.json",
   "artifacts.json",
+  // Which bank case each row answers, when this run was handed a subset. The
+  // server writes it for a run it executes itself; listed here so the two paths
+  // keep publishing the same names, which is what lets one dashboard read a run
+  // without caring where it ran.
+  "case_map.json",
 ];
 
 /**
@@ -705,6 +797,138 @@ export function statsSince(baseline, stats) {
   };
 }
 
+/**
+ * Did this run write anything down at all?
+ *
+ * The most dangerous shape a run can take, and the one that cost this project
+ * an afternoon: the harness exits 0, writes no metrics and no transcript, and
+ * every layer above reports success. On disk the suite directory was empty; in
+ * the database the row said `succeeded` with a `{}` summary after 5m38s. A
+ * broken interpreter and a working one are indistinguishable from the outside,
+ * which is the worst property a measurement tool can have.
+ *
+ * Deliberately not keyed on the adapter's counters, and that is the whole point
+ * of this function existing separately from the two guards above it. Those read
+ * `targetStats`, which nothing increments on a `callable` target -- the harness
+ * imports the agent in-process and no request ever arrives at an adapter -- so
+ * on the default target kind they are not unreliable, they are inert. A
+ * counter-based check would be correct over HTTP and dead where it is needed
+ * most. An empty summary and an empty artifact directory are facts about the
+ * run on every path, including a future one that splits the harness and the
+ * agent into two processes.
+ *
+ * Any one file with bytes in it is enough. A partial run is a finding worth
+ * keeping; only nothing at all is a failure to measure.
+ */
+/**
+ * The module name the generated wrapper is always written under.
+ *
+ * Fixed rather than derived: the server writes `DEVICE_CALLABLE_REF` itself and
+ * interpolates nothing a caller sent, which is what makes importing it by name
+ * on a customer's machine safe. If that constant ever moves, this has to move
+ * with it or the check below silently stops checking.
+ */
+const CALLABLE_MODULE = "preman_agent_entry";
+
+/**
+ * Does the approved wrapper still import, in the interpreter this run will use?
+ *
+ * Returns "" when it does, a short reason when it does not. An approved entry
+ * point is trusted on a digest, which proves the file has not changed and says
+ * nothing about whether it still runs. The two come apart exactly where it
+ * matters -- a rebuilt virtual environment, a dependency that went away, a
+ * different interpreter resolved than the one it was approved against -- and
+ * the first time that happened the run reached `inference`, spent model calls,
+ * and reported a missing attribute on a function that plainly existed.
+ *
+ * Checked here rather than at discovery because the interpreter is already
+ * resolved by this point. Doing it earlier would mean resolving one twice, and
+ * on the harness-alongside branch a resolve is seconds warm and minutes cold.
+ *
+ * A load and not a turn. Making the agent answer is what the approval gate
+ * does, and it costs a real model call; paying that on every run would tax
+ * every suite for a check that only earns its keep when something drifted. It
+ * is not free either -- `preman_selfcheck` loads the customer's module, which
+ * for a framework repository means importing crewai or langchain -- but only
+ * this probe pays it, where the gate's version bills a model.
+ *
+ * Why a named function rather than importing the wrapper and letting its own
+ * side effects do the work: the template loads the agent lazily on purpose,
+ * because a customer's module can bind a port or open a connection at import
+ * and a probe should not be the thing that triggers it. The two designs also
+ * fail differently, which is the deciding reason. A template that stops
+ * defining `preman_selfcheck` makes this skip *and say so*; a template that
+ * stopped loading eagerly would make it import cleanly and pass -- a false
+ * negative with no signal, which is the shape of the bug this exists to catch.
+ *
+ * A wrapper generated before the contract existed has no `preman_selfcheck`,
+ * so it is skipped rather than failed. Those are only re-checked once they are
+ * regenerated; the hole is closed going forward, not retroactively.
+ */
+export function entryImportFailure(python, sysPath, { timeoutMs = 120_000, log = null } = {}) {
+  if (!python?.command || !sysPath) return "";
+
+  const snippet = [
+    `import sys`,
+    `sys.path.insert(0,${JSON.stringify(sysPath)})`,
+    `import ${CALLABLE_MODULE} as _w`,
+    `_c=getattr(_w,"preman_selfcheck",None)`,
+    `print("preman-selfcheck=absent") if _c is None else (_c(),print("preman-selfcheck=ok"))`,
+  ].join(";");
+
+  const probe = spawnSync(python.command, [...(python.argv || []), "-c", snippet], {
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "pipe"],
+    timeout: timeoutMs,
+  });
+
+  // Killed, or never started. `spawnSync` reports both as `status: null` with no
+  // stderr, and the fallback below would turn that into "exit null" and block
+  // the run -- a check that could not finish, reported as proof of breakage.
+  // The timeout is the one that can really happen: `preman_selfcheck` imports
+  // the customer's agent, and importing a framework is slow.
+  if (probe.error || probe.status === null) {
+    const why = probe.error?.message || `it did not finish within ${Math.round(timeoutMs / 1000)}s`;
+    log?.(
+      `whether the approved entry point still loads could not be checked (${why}); ` +
+        `running anyway rather than refusing on a check that did not complete`
+    );
+    return "";
+  }
+
+  if (probe.status === 0) {
+    if (String(probe.stdout || "").includes("preman-selfcheck=absent")) {
+      log?.(
+        `the approved entry point predates PreMan's self-check, so whether it still loads ` +
+          `was not verified; regenerate it with \`${cliInvocation()} eval target\` to have it checked`
+      );
+    }
+    return "";
+  }
+
+  const lines = String(probe.stderr || "").trim().split("\n").filter(Boolean);
+  return lines[lines.length - 1]?.slice(0, 300) || `exit ${probe.status}`;
+}
+
+export function producedNothing(dir, summary) {
+  if (summary && typeof summary === "object" && Object.keys(summary).length) return "";
+
+  const wrote = ["metrics.json", "scores.jsonl", "inference_set.jsonl"].some((name) => {
+    try {
+      return statSync(path.join(dir, name)).size > 0;
+    } catch {
+      return false;
+    }
+  });
+  if (wrote) return "";
+
+  return (
+    "the harness exited cleanly having written no metrics and no transcript, so there is " +
+    "nothing here that was measured. The usual cause is an interpreter that can import " +
+    "assert_ai but cannot complete a run; `eval doctor` reports which one this machine picks."
+  );
+}
+
 export function missingToolEvidence(dir, stats) {
   const forwarded = Number(stats?.toolCalls);
   if (!Number.isFinite(forwarded) || forwarded <= 0) return "";
@@ -945,7 +1169,15 @@ export async function executeEvalRun(
     home = suiteHome(job);
     pruneRuns(home, job);
     laid = materialize(job, home);
-    python = resolveInterpreter({ projectPath, log });
+    // Offered only when the harness is the only thing that has to import
+    // anything. A `callable` target imports the customer's agent in-process, so
+    // it must resolve against an environment that has *their* dependencies —
+    // PreMan's bundle has assert-ai and nothing of theirs.
+    const runtime =
+      String(job.target_kind || "") === "callable"
+        ? ""
+        : await ensureRuntime({ log, download, progress: progressLine });
+    python = resolveInterpreter({ projectPath, log, runtime });
     log(`eval ${job.id}: assert-ai ${python.version} via ${python.how}`);
   } catch (error) {
     // Every one of these is a fact about this machine or this frame, and no
@@ -960,7 +1192,26 @@ export async function executeEvalRun(
     return finish(false, { error: `the eval harness is missing from this install (${HARNESS_PATH})` });
   }
 
-  const missingKey = missingModelKey(job);
+  // Before the harness, because everything after this point costs model calls.
+  if (String(job.target_kind || "") === "callable") {
+    const broken = entryImportFailure(python, process.env.PREMAN_EVAL_SYS_PATH || "", {
+      log: (line) => log(`eval ${job.id}: ${line}`),
+    });
+    if (broken) {
+      tidy(home);
+      return finish(false, {
+        error:
+          `the approved entry point no longer loads in this environment: ${broken}. It was ` +
+          `trusted on its digest, which only proves the file is unchanged -- re-run ` +
+          `\`${cliInvocation()} eval target\` here to regenerate and re-approve it.`,
+      });
+    }
+  }
+
+  // The environment the harness will actually get, so the check cannot pass
+  // where the run fails or refuse where the key was in the repository all along.
+  const runEnv = childEnv({}, { ...(job.env || {}), ...repoEnv(projectPath) });
+  const missingKey = missingModelKey(job, runEnv);
   if (missingKey) {
     // Checked before the stages start rather than left to the provider client,
     // which reports it from inside `inference` as an authentication error
@@ -987,7 +1238,7 @@ export async function executeEvalRun(
   const timeout = Math.min(RUN_TIMEOUT_MS, RUN_TIMEOUT_CAP_MS);
   const child = spawn(python.command, [...python.argv, HARNESS_PATH, "run", "--config", laid.configPath], {
     cwd: laid.cwd,
-    env: childEnv({ PREMAN_EVAL_RUN: `${job.suite}/${job.run}` }, job.env || {}),
+    env: childEnv({ PREMAN_EVAL_RUN: `${job.suite}/${job.run}` }, { ...(job.env || {}), ...repoEnv(projectPath) }),
     stdio: ["ignore", "ignore", "pipe"],
   });
 
@@ -1135,15 +1386,45 @@ export async function executeEvalRun(
       return { ...outcome, summary, artifacts };
     }
 
+    // Last, because the two guards above name a cause and this one only reports
+    // that there is nothing to report. Still ahead of `finish(true, ...)`,
+    // because a run with no results is not a passing run.
+    const empty = producedNothing(artifacts, summary);
+    if (empty) {
+      const outcome = await finish(false, { summary, error: empty, exitCode });
+      tidy(home);
+      return { ...outcome, summary, artifacts };
+    }
+
     const outcome = await finish(true, { summary, exitCode });
     tidy(home);
     return { ...outcome, summary, artifacts };
   }
 
+  // The harness's own account of the failure, kept on disk before it is reduced
+  // to something that fits in a row.
+  //
+  // The tail is eight lines because that is what a terminal and a database
+  // column can carry, and the first time an inference failure had to be
+  // diagnosed the rest had already been thrown away: the tail named a symptom
+  // -- a wrapper reporting `has no attribute` on a function that plainly
+  // existed -- while the exception that said *why* was the one before it, and
+  // gone. Written beside the artifacts, which `tidy` leaves in place, so the
+  // account outlives the process that produced it.
+  const failureLog = path.join(laid.cwd, "last-failure.log");
+  let kept = "";
+  try {
+    writeFileSync(failureLog, stderr, { mode: 0o600 });
+    kept = ` (full output: ${failureLog})`;
+  } catch {
+    // A run that cannot write its own post-mortem still has to report the
+    // failure it was handed.
+  }
+
   const tail = stderr.trim().split("\n").slice(-8).join(" / ");
   const outcome = await finish(false, {
     summary,
-    error: `the eval harness exited ${exitCode}: ${tail}`,
+    error: `the eval harness exited ${exitCode}: ${tail}${kept}`,
     exitCode,
   });
   tidy(home);
@@ -1277,9 +1558,15 @@ Eval options:
                                 token names
   eval run                      Pair this project and run the eval PreMan has
                                 queued for it, then exit
+  eval target                   Find the code in this repository that runs your
+                                agent, write a wrapper the eval can call, and
+                                remember it once you approve it
   eval doctor                   Report whether this machine can run an eval
+  --entry <file>:<symbol>       Skip the search; this is the agent. Still writes
+                                a wrapper and still asks you to approve it
   --target <url>                Skip discovery; this URL is the agent. Must take
                                 POST {"message","history"} and answer {"response"}
+  --preman-agent <url>          Evaluate PreMan's own chat agent at this address
   --agent <name>                claude-code | cursor | codex (for pairing only)
   --path <dir>                  Project to pair as. Defaults to cwd
   --wait <seconds>              How long to wait for a queued eval (default 600)
@@ -1322,6 +1609,66 @@ function pairingAgent(args) {
  * numbers set on the settings page. Sending anything here -- even the defaults
  * -- would override them and quietly make this command disagree with the page.
  */
+/**
+ * A run that ended without this device ever being handed it.
+ *
+ * The server can refuse a run at claim time -- a target it will not dispatch, a
+ * bundle it cannot build, a budget it will not spend -- and it records the
+ * reason on the row without dispatching an event. The runner is left listening
+ * on a stream that will never carry the job, which reads as a hang: the terminal
+ * stops at "connected as runner" and the only account of what happened is a
+ * column in Postgres. That is the same shape as every other bug found today, so
+ * it is worth a poll.
+ *
+ * Only `failed` and `cancelled` end the wait. A successful run also reaches a
+ * terminal status, but it does so *because this device finished it*, from inside
+ * the job the loop is still tidying up after -- cutting the loop off there would
+ * report `jobsRun: 0` for a run that worked.
+ */
+const TERMINAL_RUN_STATES = new Set(["failed", "cancelled"]);
+
+export function watchRunTerminal(args, runId, { token, intervalMs = 3_000, fetchRun = null } = {}) {
+  const controller = new AbortController();
+  let over = null;
+  let timer = null;
+  let stopped = false;
+
+  const read =
+    fetchRun ||
+    ((id) => callBackendJson(args, "GET", `/eval-runs/${encodeURIComponent(id)}`, { token }));
+
+  const tick = async () => {
+    if (stopped) return;
+    try {
+      const seen = await read(runId);
+      const status = String(seen?.status || "");
+      if (seen?.ok !== false && TERMINAL_RUN_STATES.has(status)) {
+        over = { status, error: String(seen.error || "") };
+        stopped = true;
+        // Wakes the loop: `stopWhen` is not read while the stream is idle.
+        controller.abort();
+        return;
+      }
+    } catch {
+      // A poll that cannot reach the backend says nothing about the run, and a
+      // run is not over because the network blinked.
+    }
+    if (!stopped) timer = setTimeout(tick, intervalMs);
+  };
+
+  timer = setTimeout(tick, intervalMs);
+
+  return {
+    signal: controller.signal,
+    stopWhen: () => over !== null,
+    ended: () => over,
+    stop: () => {
+      stopped = true;
+      if (timer) clearTimeout(timer);
+    },
+  };
+}
+
 async function runForThisAccount(args, deps) {
   const say = (line) => process.stdout.write(`${line}\n`);
 
@@ -1423,6 +1770,13 @@ async function runWithToken(
   // This is for the ones it cannot: a signal, a closed tab, a crash in a
   // callback -- after which the socket would otherwise outlive the run.
   const forgetAdapter = registerEvalTeardown(() => adapter.close());
+  if (target.kind === "callable") {
+    // The import root for the customer's own wrapper. assert-ai resolves a
+    // callable only as a dotted module name and has no spec key that adds an
+    // import path, so the harness reads this and inserts it before handing off.
+    process.env.PREMAN_EVAL_SYS_PATH = target.sysPath;
+  }
+
   if (adapter.translated) {
     // Set for this process so `childEnv` carries it to the harness. assert-ai
     // refuses loopback endpoints by default and that default is right: it stops
@@ -1446,7 +1800,19 @@ async function runWithToken(
         app_version: packageVersion() || undefined,
         capabilities: { agent_eval_v1: true },
         discovery: {
-          url: adapter.url,
+          // Exactly one of these, which the backend enforces. A callable has
+          // no address: the harness imports it in-process, so there is nothing
+          // for the server to render a sandbox against.
+          ...(target.kind === "callable"
+            ? {
+                callable: target.ref,
+                sys_path: target.sysPath,
+                entry_file: target.entryFile,
+                entry_sha256: target.sha256,
+                language: target.language,
+                framework: target.framework,
+              }
+            : { url: adapter.url }),
           label: target.label,
           how: target.how,
           // The addresses that did not answer, kept because "why did it test
@@ -1501,15 +1867,28 @@ async function runWithToken(
     const headless = args.has("--headless") || args.has("--no-wait") || !process.stdout.isTTY;
     let done = 0;
     for (const run of runs) {
-      const result = await runnerLoop(args, state, {
-        once: true,
-        only: "eval",
-        headless,
-        log: (line) => say(`  ${line}`),
-      });
+      const watch = watchRunTerminal(args, run.id, { token: resolveApiKey(args) });
+      let result;
+      try {
+        result = await runnerLoop(args, state, {
+          once: true,
+          only: "eval",
+          headless,
+          log: (line) => say(`  ${line}`),
+          signal: watch.signal,
+          stopWhen: watch.stopWhen,
+        });
+      } finally {
+        watch.stop();
+      }
       done += result.jobsRun || 0;
       if (!result.jobsRun) {
-        say(`  stopped after ${done} of ${runs.length} (${result.reason})`);
+        // The row's own reason, when there is one. `result.reason` describes why
+        // the loop stopped, which on this path is "the watcher aborted it" --
+        // true and useless. What somebody needs is why the run ended.
+        const over = watch.ended();
+        const why = over ? over.error || `the run ended as ${over.status}` : result.reason;
+        say(`  stopped after ${done} of ${runs.length} (${why})`);
         break;
       }
     }
@@ -1579,11 +1958,40 @@ export async function evalCommand(
   const sub = commandArgs.find((value) => !value.startsWith("-")) || "run";
   const args = makeArgs(commandArgs);
 
+  if (sub === "target") {
+    // Its own command so the model call and the file write happen once, on
+    // purpose, with somebody watching -- never as a side effect of a run.
+    const projectPath = path.resolve(args.value("--path", process.cwd()));
+    const say = (line = "") => process.stdout.write(`${line}\n`);
+    const { searchForEntry, entryFromNamedSymbol } = await import("./eval_entry.js");
+    const named = String(args.value("--entry", "") || "").trim();
+    try {
+      const found = named
+        ? await entryFromNamedSymbol(named, { projectPath, args, log: say })
+        : await searchForEntry({ projectPath, args, log: say });
+      say(`  ${found.label}`);
+      say(`  ${found.how}`);
+      return { ok: true, target: found };
+    } catch (error) {
+      say(error.message);
+      return { ok: false, reason: "no_entry" };
+    }
+  }
+
   if (sub === "doctor") {
     const projectPath = path.resolve(args.value("--path", process.cwd()));
     let python;
     try {
-      python = resolveInterpreter({ projectPath });
+      // Doctor answers "could a run happen here", so it has to resolve the same
+      // way a run does -- including fetching the runtime if that is what a run
+      // would do. Reporting an interpreter a real run would not pick is the one
+      // thing a diagnostic must not do.
+      const runtime = await ensureRuntime({
+        log: (line) => process.stdout.write(`${line}\n`),
+        download,
+        progress: progressLine,
+      });
+      python = resolveInterpreter({ projectPath, runtime });
     } catch (error) {
       process.stdout.write(`Cannot run evals here.\n  ${error.message}\n`);
       return { ok: false, reason: "no_harness" };