premanmcp 1.1.4 → 1.1.6

package/bin/eval.js

@@ -45,7 +45,7 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 
 import { adapterStats, discoverTarget, startAdapter, stepFeed } from "./eval_target.js";
-import { resolveEvalSurface } from "./link.js";
+import { formatSurfaceLink, resolveEvalSurface } from "./link.js";
 import {
   backendUrl,
   callBackendJson,
@@ -85,6 +85,42 @@ const MAX_STDERR_BYTES = 256 * 1024;
 
 export class EvalError extends Error {}
 
+/**
+ * What a run has open that a signal would otherwise orphan.
+ *
+ * A run holds two things the process does not get back by dying: the Python
+ * harness, a child that goes on measuring somebody's agent with nobody reading
+ * the result, and the adapter's listening socket -- which the code that opens it
+ * calls "an unauthenticated door to somebody's agent open for as long as the
+ * shell stays alive". Both were released only by a `finally`, so a SIGTERM or a
+ * closed tab left them standing.
+ *
+ * A set rather than a single handle because a session can be told to stop
+ * between one run and the next, and a teardown that has already run is not an
+ * error -- it is simply no longer registered.
+ */
+const OPEN_RUNS = new Set();
+
+export function registerEvalTeardown(fn) {
+  OPEN_RUNS.add(fn);
+  return () => OPEN_RUNS.delete(fn);
+}
+
+/** Close everything a run has open. Safe to call when nothing is running. */
+export async function cancelEvalRuns() {
+  const pending = [...OPEN_RUNS];
+  OPEN_RUNS.clear();
+  for (const fn of pending) {
+    try {
+      await fn();
+    } catch {
+      // Best effort by construction: this runs on the way out, and a socket
+      // that is already closed is the outcome being asked for.
+    }
+  }
+  return pending.length;
+}
+
 // ── Where a suite lives between runs ────────────────────────────────────
 
 /** Everything this machine keeps about evals it has run. */
@@ -575,6 +611,58 @@ export function readProgress(cwd, job) {
   };
 }
 
+/**
+ * Whether the tool calls the agent made are in the transcript the judge read.
+ *
+ * A behaviour like `unchecked_account_claim` is defined entirely in terms of
+ * tool calls: its taxonomy tells the judge to read the events and not to infer
+ * a lookup from the wording of a reply. So a transcript that lost them is not a
+ * degraded measurement, it is the opposite measurement -- an agent that checked
+ * before answering is indistinguishable from one that made the figure up, and
+ * the run reports the second. It happened for every run of that suite until the
+ * adapter's event field names were corrected, and it exited 0 each time.
+ *
+ * Compared against the adapter's own count rather than checked for zero on its
+ * own, because zero tool calls is a legitimate result: an agent asked a general
+ * question should not call anything. The failure is the disagreement.
+ *
+ * Returns "" when there is nothing wrong, so the caller can tell "fine" from
+ * "unreadable".
+ */
+export function missingToolEvidence(dir, stats) {
+  const forwarded = Number(stats?.toolCalls);
+  if (!Number.isFinite(forwarded) || forwarded <= 0) return "";
+
+  let recorded = 0;
+  try {
+    const text = readFileSync(path.join(dir, "inference_set.jsonl"), "utf8");
+    for (const line of text.split("\n")) {
+      if (!line.trim()) continue;
+      let row;
+      try {
+        row = JSON.parse(line);
+      } catch {
+        continue; // A row we cannot read is not evidence that evidence is missing.
+      }
+      for (const event of row.events || []) {
+        if (event?.edit?.type === "tool_call") recorded += 1;
+      }
+    }
+  } catch {
+    // No transcript to read is a different failure, and the harness's own exit
+    // code reports it. Claiming this one on top would be guessing.
+    return "";
+  }
+  if (recorded > 0) return "";
+
+  return (
+    `the agent made ${forwarded} tool call(s) and none of them reached the transcript, so ` +
+    `the judge scored it as an agent that called nothing. Every case that answered from a ` +
+    `lookup would be marked unsupported. This is the adapter's event channel, not the ` +
+    `agent: check that the events it sends still match the fields assert-ai reads.`
+  );
+}
+
 // ── Handing the run over to a surface ───────────────────────────────────
 
 /** How long to hold a leased run waiting for somebody to be watching it. */
@@ -673,7 +761,7 @@ export async function awaitAudience(job, { call, log, lease, headless = true, su
   // minutes of waiting and the same five lines printed seven times, which reads
   // as the thing running in circles.
   if (audienceDeclined) {
-    if (surface?.webUrl) log(`Watch it at ${surface.webUrl}`);
+    if (surface?.webUrl) log(surface.webUrl);
     return { watching: false, waited: 0, skipped: "declined-earlier" };
   }
 
@@ -683,15 +771,14 @@ export async function awaitAudience(job, { call, log, lease, headless = true, su
     // even the desktop branch cannot confirm the window came forward — an older
     // build ignores the route and lands wherever it was. So the URL is always
     // printed, and "opened" is never claimed.
-    if (surface.webUrl) log(`Watch it at ${surface.webUrl}`);
+    for (const line of formatSurfaceLink(surface, { stream: process.stdout }).split("\n")) {
+      if (line) log(line);
+    }
     try {
+      // The caveat a raise carries is in the label above, so nothing is added
+      // here: this build comes forward on whatever it was showing, and a reader
+      // who expected the run would otherwise read the wrong view as a failure.
       surface.open?.();
-      if (surface.surface === "desktop-raise") {
-        // Said plainly because the window will come forward showing whatever it
-        // was showing, and somebody who expected it to land on the run would
-        // otherwise read "not this" as "it did not work".
-        log("Brought the PreMan app forward. This build cannot jump to a view, so open Agent eval yourself.");
-      }
     } catch (error) {
       log(`could not open a window here (${error.message}); the link above still works`);
     }
@@ -769,7 +856,10 @@ export async function executeEvalRun(
       { lease_token: lease, ok, summary, error: error ? error.slice(0, ERROR_CAP) : undefined, exit_code: exitCode },
       log
     );
-    return { ok, reason: result.ok ? "reported" : "report_failed" };
+    // `error` travels back with the outcome as well as to the backend. It used
+    // to go only to the backend, so the terminal that started the run watched
+    // it end at the word "failed" with the reason readable in a browser.
+    return { ok, reason: result.ok ? "reported" : "report_failed", error };
   };
 
   let home = "";
@@ -785,9 +875,8 @@ export async function executeEvalRun(
     // Every one of these is a fact about this machine or this frame, and no
     // amount of retrying changes it. Failing with the reason is more use than
     // letting the lease lapse and having the server call it a lost device.
-    log(`eval ${job.id} cannot run here: ${error.message}`);
     tidy(home);
-    return finish(false, { error: error.message });
+    return finish(false, { error: `cannot run here: ${error.message}` });
   }
 
   if (!existsSync(HARNESS_PATH)) {
@@ -822,6 +911,8 @@ export async function executeEvalRun(
     stdio: ["ignore", "ignore", "pipe"],
   });
 
+  const forgetChild = registerEvalTeardown(() => child.kill("SIGTERM"));
+
   let stderr = "";
   child.stderr.on("data", (chunk) => {
     if (stderr.length < MAX_STDERR_BYTES) stderr += chunk.toString("utf8");
@@ -894,6 +985,7 @@ export async function executeEvalRun(
   });
   clearInterval(heartbeat);
   clearTimeout(killer);
+  forgetChild();
 
   // Deliberately before the bundle is removed: the summary is read off the
   // disk the run just wrote to.
@@ -931,7 +1023,9 @@ export async function executeEvalRun(
     // answered some of the time is a real measurement of a flaky agent and
     // deciding otherwise here would throw away the finding.
     const stats = adapterStats();
-    if (stats && stats.turns === 0 && stats.failures > 0) {
+    // `reached` rather than `turns`: the adapter counts a turn when it starts
+    // one, so a run where every turn failed has as many turns as cases.
+    if (stats && (stats.reached ?? stats.turns) === 0 && stats.failures > 0) {
       const outcome = await finish(false, {
         summary,
         error:
@@ -943,6 +1037,15 @@ export async function executeEvalRun(
       return { ...outcome, summary, artifacts };
     }
 
+    // The same shape of failure one layer in: the agent was reached, it acted,
+    // and the record of it acting did not survive the trip to the judge.
+    const blind = missingToolEvidence(artifacts, stats);
+    if (blind) {
+      const outcome = await finish(false, { summary, error: blind, exitCode });
+      tidy(home);
+      return { ...outcome, summary, artifacts };
+    }
+
     const outcome = await finish(true, { summary, exitCode });
     tidy(home);
     return { ...outcome, summary, artifacts };
@@ -958,6 +1061,124 @@ export async function executeEvalRun(
   return { ...outcome, summary, artifacts };
 }
 
+/**
+ * How long a run took, in the units somebody reads without converting.
+ */
+function elapsed(seconds) {
+  const total = Math.round(Number(seconds) || 0);
+  if (!total) return "";
+  if (total < 60) return `${total}s`;
+  const minutes = Math.floor(total / 60);
+  return `${minutes}m${String(total % 60).padStart(2, "0")}s`;
+}
+
+/**
+ * One run's numbers as a line, the way the page states them.
+ *
+ * `scored` is the denominator and `not_run` is outside it, which is the part a
+ * terminal most easily gets wrong: a run where six of eight cases never reached
+ * the agent has two results, not eight, and "6 passed" would be a fabrication.
+ * A split decision is called out rather than folded into the flagged count,
+ * because a judge panel that disagreed with itself is a weaker finding than one
+ * that did not, and the two should not read identically.
+ */
+export function describeRunSummary(row) {
+  const took = elapsed(row.elapsed_s);
+  const suffix = took ? ` · ${took}` : "";
+
+  // Read whatever the run's status, because a case that never reached the agent
+  // is not a failed run to the backend: `_build_run_summary` only ever says
+  // `done` or `running`, and hangs `failure` off a finished run whenever
+  // `not_run` is non-zero. Reading it under a "failed" status alone dropped the
+  // detail in exactly the case it was wanted -- six cases that never reached
+  // the agent, and no word here about why.
+  const failure = row.failure || {};
+  const named = [failure.reason, failure.detail].filter(Boolean).join(": ");
+  const affected = failure.cases_affected ? ` (${failure.cases_affected} cases)` : "";
+  const why = named ? `${named}${affected}` : "";
+
+  if (row.status === "running") return { mark: "·", text: `still finishing${suffix}`, why };
+  if (row.status === "failed") return { mark: "✗", text: `failed${suffix}`, why };
+
+  const parts = [];
+  if (row.flagged) {
+    const split = row.split ? `, ${row.split} of them split` : "";
+    parts.push(`${row.flagged} of ${row.scored} flagged${split}`);
+  } else if (row.scored) {
+    parts.push(`${row.scored} scored, nothing flagged`);
+  } else {
+    parts.push("nothing was scored");
+  }
+  if (row.not_run) parts.push(`${row.not_run} never ran`);
+
+  // A tick is reserved for a run that measured everything it set out to and
+  // found nothing. Cases that never reached the agent are not passes, so a run
+  // carrying any of them is marked as the partial result it is rather than
+  // being waved through on the strength of the cases that did run.
+  const mark = row.flagged || !row.scored ? "✗" : row.not_run ? "·" : "✓";
+  return { mark, text: `${parts.join(" · ")}${suffix}`, why };
+}
+
+/**
+ * Read back what the runs measured, and say it here.
+ *
+ * The terminal used to end at "finished: failed" and a URL, which is the one
+ * moment somebody is definitely looking at the terminal and not at a dashboard.
+ * The numbers come from `/eval-traces` rather than from the artifacts this
+ * machine just wrote, deliberately: `case_state` on the server is the single
+ * definition of what counts as a pass, a split or a case that never ran, and a
+ * second copy of that rule in JavaScript would agree with the app only by
+ * coincidence -- which is the exact failure that rule was consolidated to end.
+ *
+ * Best-effort throughout. A missing key, an unreachable backend, a run whose
+ * artifacts have not landed yet: none of them is a reason to fail a run that
+ * has already finished, so every one of them prints less rather than erroring.
+ */
+export async function summarizeRuns(args, key, runs, { say = () => {} } = {}) {
+  if (!key || !runs.length) return [];
+
+  const wanted = new Map(runs.map((run) => [`${run.suite}/${run.run}`, run]));
+  const found = new Map();
+  for (const suite of new Set(runs.map((run) => run.suite))) {
+    let result;
+    try {
+      result = await callBackendJson(
+        args,
+        "GET",
+        `/eval-traces/suites/${encodeURIComponent(suite)}/runs`,
+        { token: key }
+      );
+    } catch {
+      continue;
+    }
+    if (!result.ok) continue;
+    for (const row of result.list || []) {
+      const id = `${row.suite}/${row.run}`;
+      if (wanted.has(id)) found.set(id, row);
+    }
+  }
+
+  if (!found.size) return [];
+
+  say("");
+  say("Results:");
+  const lines = [];
+  for (const [id, run] of wanted) {
+    const row = found.get(id);
+    // A queued behaviour whose artifacts never arrived is named and left at
+    // that. Saying nothing about it would read as a run that did not happen.
+    if (!row) {
+      say(`  · ${id} — nothing to read back yet`);
+      continue;
+    }
+    const { mark, text, why } = describeRunSummary(row);
+    say(`  ${mark} ${id} — ${text}`);
+    if (why) say(`      ${why}`);
+    lines.push({ id, row });
+  }
+  return lines;
+}
+
 // ── Command ─────────────────────────────────────────────────────────────
 
 export const EVAL_HELP = `
@@ -1109,6 +1330,10 @@ async function runWithToken(
     // audience for these is the dashboard.
     onStep: (step) => stepFeed.push(step),
   });
+  // The `finally` below still closes it on every path this function can see.
+  // This is for the ones it cannot: a signal, a closed tab, a crash in a
+  // callback -- after which the socket would otherwise outlive the run.
+  const forgetAdapter = registerEvalTeardown(() => adapter.close());
   if (adapter.translated) {
     // Set for this process so `childEnv` carries it to the harness. assert-ai
     // refuses loopback endpoints by default and that default is right: it stops
@@ -1181,7 +1406,7 @@ async function runWithToken(
     const surface = resolved
       ? { ...resolved, open: () => openUrl(resolved.href, { app: resolved.app }) }
       : null;
-    if (surface?.webUrl) say(`Watch it at ${surface.webUrl}`);
+    if (surface) say(formatSurfaceLink(surface, { stream: process.stdout }));
     say("");
 
     const headless = args.has("--headless") || args.has("--no-wait") || !process.stdout.isTTY;
@@ -1202,13 +1427,20 @@ async function runWithToken(
 
     say("");
     say(`Finished ${done} of ${runs.length}.`);
-    if (surface) say(`Results: ${surface.webUrl}`);
+    // The numbers before the link, because the link is where somebody goes when
+    // the numbers are not enough -- and on this path they usually are.
+    await summarizeRuns(args, resolveApiKey(args), runs, { say });
+    if (surface) {
+      say("");
+      say(formatSurfaceLink(surface, { stream: process.stdout }));
+    }
     return { ok: done > 0, jobsRun: done, expected: runs.length };
   } finally {
     // Always, including on the throw paths above. The adapter holds a listening
     // socket and a set of live conversations with the agent; leaving it up after
     // a failed redemption would leave an unauthenticated door to somebody's
     // agent open for as long as the shell stays alive.
+    forgetAdapter();
     await adapter.close();
   }
 }

package/bin/eval_harness.py

@@ -85,11 +85,15 @@ def _config_mentions_anthropic() -> bool:
 
 
 WORKSPACE_ID = os.environ.get("ANTHROPIC_WORKSPACE_ID")
-if _config_mentions_anthropic() and not WORKSPACE_ID:
-    sys.exit("ANTHROPIC_WORKSPACE_ID is not set (expected a wrkspc_... value)")
 
 import litellm
 
+# Workaround 1 applies to an identity-linked key, which carries no workspace
+# binding of its own. A workspace-scoped key already has one and the Messages
+# API accepts it bare, so an absent id is not by itself an error — it only means
+# there is no header to attach. Refusing the run here failed keys that work; a
+# key that genuinely needs the header now hears it from Anthropic, in its own
+# words, at the first call.
 if WORKSPACE_ID and _config_mentions_anthropic():
     litellm.headers = {"anthropic-workspace-id": WORKSPACE_ID}
 

package/bin/eval_target.js

@@ -33,6 +33,9 @@
  */
 
 import { createServer } from "node:http";
+
+import { askAgent } from "./agent_dialects.js";
+import { REPO_CONFIG, savedAgent } from "./target.js";
 import { createHash } from "node:crypto";
 
 import { backendUrl, resolveApiKey, storedKeyOrigin } from "./shared.js";
@@ -157,6 +160,23 @@ export async function discoverTarget(args, { log = () => {} } = {}) {
     };
   }
 
+  // What the repository says its agent is, saved by `preman target detect`. It
+  // ranks above discovery for the same reason `--target` does: somebody already
+  // answered this question, and probing past their answer would measure a
+  // different process than the one they pointed at. Below `--target` because a
+  // flag is about this invocation and the file is about the project.
+  const saved = savedAgent();
+  if (saved) {
+    return {
+      kind: "external",
+      agent: saved,
+      url: `${saved.url}${saved.path}`,
+      label: `your agent (${saved.url}${saved.path})`,
+      how: `${REPO_CONFIG} names it, and it speaks ${saved.dialect}`,
+      probed: [],
+    };
+  }
+
   const probed = [];
   for (const port of LOCAL_PORTS) {
     const result = await probeAgent(`http://127.0.0.1:${port}`);
@@ -293,7 +313,7 @@ function readBody(request) {
 /**
  * One turn against PreMan's own agent.
  *
- * Tool calls come back in `turn.artifacts` and are forwarded as adapter-shaped
+ * Tool calls come back in `turn.tool_calls` and are forwarded as adapter-shaped
  * `events`, which is the difference between the judge seeing a transcript and
  * the judge seeing what the agent *did*. `HTTPEndpointSession` promotes those to
  * first-class interaction messages; without them a tool-using agent is scored on
@@ -443,7 +463,18 @@ async function premanTurn(target, args, { message, history }, state) {
     .digest("hex");
   state.conversations.set(nextKey, conversationId);
 
-  return { response: reply, events: eventsFrom(turn) };
+  // Counted here rather than derived from the artifacts later, because these
+  // two numbers answer different questions: this is what the agent did, and
+  // what the transcript holds is what the judge got to see. A run where they
+  // disagree is the failure this counter exists to make visible.
+  //
+  // Counting the events rather than `turn.tool_calls` keeps it a count of what
+  // was forwarded: a call this adapter dropped is not one the transcript is
+  // missing. `tool_result` is one call, per `eventsFrom`.
+  const events = eventsFrom(turn);
+  state.toolCalls += events.filter((event) => event.role === "tool_result").length;
+
+  return { response: reply, events };
 }
 
 /**
@@ -457,33 +488,75 @@ async function premanTurn(target, args, { message, history }, state) {
  * agent checked before answering was reading a transcript in which checking
  * was invisible.
  *
- * `executed: false` is reported as a distinct event rather than dropped or
- * flattened into a call. A risky tool is not run — the broker records a
- * proposal and waits for a person — so treating the request as the deed would
- * have the judge score a deletion that never occurred.
+ * The field names are assert-ai's `AdapterEvent`, and they are the whole
+ * difference between an event and nothing. Its normalizer keys on `role` and
+ * skips any event whose role is not one of its three, without logging -- so a
+ * plausible-looking `{type, name, arguments}` is dropped silently and produces
+ * exactly the same transcript as an agent that called nothing. Every run of
+ * `unchecked_account_claim` before this was scored against one: 24 cases, zero
+ * tool calls recorded, while the agent was answering with live figures that
+ * matched the workspace.
+ *
+ * One `tool_result` per call, and deliberately no `tool_call` event alongside
+ * it. `tool_result` is the one that becomes a tool call in the artifacts: a
+ * `tool_call` event is an assistant message carrying `tool_calls`, which the
+ * inference stage only remembers, while the `ToolCallEdit` that
+ * `edit.type == "tool_call"` refers to -- what the judge and the optimizer both
+ * count -- is written when the tool *message* arrives. Sending both records the
+ * same single call and adds an assistant message with empty content for each
+ * one, because the endpoint session attaches a `raw` to every event and the
+ * stage keeps any message that has one. Blank assistant turns are the last
+ * thing to show a judge deciding whether the agent answered from a lookup.
+ *
+ * Whether the tool ran travels in the result's text, in the taxonomy's own
+ * words, because text is what survives: of everything an event can carry, only
+ * `content`, `tool_name`, `tool_args` and `tool_call_id` reach the artifacts. A
+ * risky tool is not run — the broker records a proposal and waits for a person —
+ * and a judge that read the request as the deed would score a deletion that
+ * never occurred. What the tool returned is not carried at all: the backend's
+ * record is name, arguments and executed, which is what "did it look this up
+ * before answering" needs and no more.
  */
-function eventsFrom(turn) {
+export function eventsFrom(turn) {
   const calls = Array.isArray(turn.tool_calls) ? turn.tool_calls : [];
   const events = [];
   for (const call of calls) {
     const name = String(call?.name || "");
     if (!name) continue;
+    const args = call.arguments;
     events.push({
-      type: "tool_call",
-      name,
-      arguments: call.arguments ?? {},
+      role: "tool_result",
+      content:
+        call.executed === false
+          ? `executed: false — ${name} is a risky action held for the user's approval. It ` +
+            `did not run and returned no data.`
+          : `executed: true — ${name} ran and returned its result to the agent. The result ` +
+            `itself is not carried into this transcript.`,
+      tool_name: name,
+      tool_args: args && typeof args === "object" && !Array.isArray(args) ? args : {},
     });
-    if (call.executed === false) {
-      events.push({
-        type: "tool_result",
-        name,
-        result: { executed: false, reason: "awaiting the user's approval" },
-      });
-    }
   }
   return events;
 }
 
+/**
+ * One turn against an agent PreMan did not write.
+ *
+ * The whole translation is `askAgent`: the dialect decides the request and
+ * reads the reply, and what comes back here is text plus whatever tool calls
+ * the answer carried. Counted the same way the PreMan path counts, because
+ * `executeEvalRun` refuses a run where no turn ever reached the agent, and that
+ * guard is worth more against a customer's agent than against our own -- it is
+ * the one that catches an adapter pointed at the wrong port answering 404 to
+ * every case while the run reports a clean sheet.
+ */
+async function externalTurn(target, { message, history }, state) {
+  const { text, events } = await askAgent(target.agent, String(message || ""), history || []);
+  state.toolCalls += events.length;
+  for (const event of events) state.step(`${event.tool_name}`);
+  return { response: text, events };
+}
+
 /**
  * Serve assert-ai's contract on loopback until the eval is done.
  *
@@ -512,9 +585,16 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
   const state = {
     conversations: new Map(),
     turns: 0,
+    // Attempted and arrived are different numbers, and only one of them answers
+    // "did this run ever reach the agent". `turns` is incremented before the
+    // call, so an adapter pointed at the wrong port answers every case with an
+    // apology and still reports a full set of turns -- which is the exact
+    // failure `executeEvalRun`'s guard exists to catch, and it could not.
+    reached: 0,
     failures: 0,
     lastFailure: "",
     streamFailures: 0,
+    toolCalls: 0,
     log,
     /**
      * One thing the agent said it was doing, on its way to the dashboard.
@@ -547,7 +627,11 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
       try {
         const body = await readBody(request);
         state.turns += 1;
-        const result = await premanTurn(target, args, body, state);
+        const result =
+          target.kind === "external"
+            ? await externalTurn(target, body, state)
+            : await premanTurn(target, args, body, state);
+        state.reached += 1;
         reply(200, result);
       } catch (error) {
         state.failures += 1;

package/bin/link.js

@@ -182,25 +182,52 @@ function resolveSurface(
     };
   }
 
-  // A raise is now the last resort rather than the second choice, and only when
-  // there is no page to send anyone to instead.
+  // A raise outranks the browser again, and only where the app is actually on
+  // this machine.
   //
-  // It was ahead of the web branch on the reasoning that somebody watching an
-  // eval is already sitting in front of the app. True, but it trades a link
-  // that lands on the run for a window that lands wherever it was left --
-  // which, for an app too old to route, is the one thing it can never be. What
-  // that bought in practice was a stale view and the question "where is my
-  // run", while the browser would have shown it. Raising still beats nothing,
-  // so it survives for the case where no dashboard is known at all.
-  if (!base) {
-    return preferApp && installed
-      ? { href: `preman://open?source=${source}`, webUrl, app, surface: "desktop-raise" }
-      : null;
+  // It was demoted on the reasoning that a raise cannot land you on the run
+  // while a browser link can, so the browser was the more useful click. That
+  // remains true and is the cost being paid here: the decision was reversed
+  // deliberately, because somebody who has installed the desktop app has said
+  // where they want PreMan to open, and a CLI that keeps sending them to a web
+  // tab instead is overruling them on their own machine. The caller prints the
+  // run's URL underneath every branch, and says plainly that an app too old to
+  // route will surface wherever it was left -- so what a raise costs is a
+  // couple of clicks inside the app, not the run.
+  if (preferApp && installed) {
+    return { href: `preman://open?source=${source}`, webUrl, app, surface: "desktop-raise" };
   }
+  if (!base) return null;
   if (base !== DEFAULT_FRONTEND) return { href: webUrl, webUrl, surface: "web-local" };
   return { href: webUrl, webUrl, surface: "web" };
 }
 
+/**
+ * A two-line block for a surface somebody already resolved: clickable headline,
+ * plain URL underneath.
+ *
+ * Takes the surface rather than resolving one, because the eval flow needs the
+ * same object it opens windows with -- resolving twice would let the line a
+ * reader clicks and the window the CLI opens disagree about where the run is.
+ *
+ * The label says which of the three it is, since "the app" and "the dashboard"
+ * are different places to go and look. A `desktop-raise` says what it cannot
+ * do, in the line itself: that build brings its window forward on whatever it
+ * was showing, and a reader who expected the run would otherwise read the
+ * wrong view as a failure.
+ */
+export function formatSurfaceLink(surface, { label = "", stream = process.stdout } = {}) {
+  if (!surface) return "";
+  const named =
+    label ||
+    (surface.surface === "desktop"
+      ? "\u25c6 Open in the PreMan app"
+      : surface.surface === "desktop-raise"
+        ? "\u25c6 Open the PreMan app \u2014 this build lands where you left it, so pick Agent eval"
+        : "\u25c6 Open in the PreMan dashboard");
+  return renderLinkBlock({ href: surface.href, webUrl: surface.webUrl, label: named, stream });
+}
+
 /** Wrap `label` so supporting terminals make it clickable. */
 export function terminalHyperlink(url, label) {
   return `\u001b]8;;${url}\u001b\\${label}\u001b]8;;\u001b\\`;