premanmcp 1.0.4 → 1.1.0

package/bin/link.js

@@ -15,7 +15,7 @@
  */
 
 import { desktopAppInstalled, installedDesktopVersion } from "./desktop.js";
-import { DEFAULT_FRONTEND, frontendUrl } from "./shared.js";
+import { DEFAULT_FRONTEND, frontendUrl, pairedFrontendUrl } from "./shared.js";
 
 export { installedDesktopVersion };
 
@@ -98,18 +98,93 @@ export function desktopSupportsRouting(destination = "/Applications") {
  * handing it a localhost route would open the wrong build.
  */
 export function resolveRunSurface(args, { batchId = "", destination = "/Applications" } = {}) {
-  const base = frontendUrl(args);
   const route = batchId ? `${RUN_ROUTE}?batch=${encodeURIComponent(batchId)}` : RUN_ROUTE;
-  const webUrl = `${base}${route}`;
+  return resolveSurface(args, route, { destination, source: "push" });
+}
 
-  if (base !== DEFAULT_FRONTEND) return { href: webUrl, webUrl, surface: "web-local" };
-  if (desktopAppInstalled(destination) && desktopSupportsRouting(destination)) {
+/** The dashboard view an eval link points at. */
+export const EVAL_ROUTE = "/workbench/agent-eval";
+
+/**
+ * Where to send someone to watch the eval this CLI is about to run.
+ *
+ * Same three-branch resolution as a push link, and worth being explicit about
+ * what it does and does not promise, because the eval flow leans on it harder:
+ * a push link is a nicety after the work is done, whereas an eval is only worth
+ * watching *while* it runs.
+ *
+ * The desktop branch needs macOS, an installed app, and a build new enough to
+ * read `route`; older builds bring the window forward and land on whatever view
+ * was last open. So `webUrl` is printed in every branch and the caller must
+ * treat "opened" as best-effort. See docs/EVAL_REVEAL_CONTRACT.md.
+ */
+export function resolveEvalSurface(args, { runId = "", destination = "/Applications" } = {}) {
+  const route = runId ? `${EVAL_ROUTE}?run=${encodeURIComponent(runId)}` : EVAL_ROUTE;
+  return resolveSurface(args, route, {
+    destination,
+    source: "eval",
+    requirePairing: true,
+    // An eval is watched while it happens, and the app is where somebody
+    // watching it is already sitting. A browser tab is the fallback, not the
+    // default, for the one link in this CLI that is time-sensitive.
+    preferApp: true,
+  });
+}
+
+/**
+ * `requirePairing` is the difference between a pointer and an appointment.
+ *
+ * A push link is read after the fact: it says "your results are in the
+ * dashboard", and naming the usual dashboard is a fair answer even when this
+ * particular invocation went somewhere unusual. Guessing costs a click.
+ *
+ * An eval link is an appointment to watch something happen in the next few
+ * seconds. Sending someone to a deployment the run does not exist on gives them
+ * a page that loads, renders, and shows nothing — which reads as a lost run
+ * rather than a wrong address, and the run is over by the time that is worked
+ * out. So the eval side refuses to guess and the caller drops the link.
+ */
+/**
+ * `preferApp` is for the case where the app is the point.
+ *
+ * Routing into the app needs a build that reads `route`, and only the hosted
+ * dashboard is worth routing to — the app ships its own bundled UI, so handing
+ * it a localhost path opens the wrong build. Both conditions fail often, and
+ * until now failing either meant a browser.
+ *
+ * That is the wrong answer for somebody sitting in front of the app watching an
+ * eval. Raising is a strictly weaker action than routing — the URL carries no
+ * route, the window simply comes forward on whatever it was showing — so it is
+ * safe when routing is not: no build can misread it, and no dashboard can be
+ * the wrong one, because none is named. It cannot land you on the run, and the
+ * caller says so rather than implying otherwise.
+ */
+function resolveSurface(
+  args,
+  route,
+  { destination, source, requirePairing = false, preferApp = false }
+) {
+  const base = requirePairing ? pairedFrontendUrl(args) : frontendUrl(args);
+  const webUrl = base ? `${base}${route}` : "";
+  const installed = desktopAppInstalled(destination);
+
+  if (base === DEFAULT_FRONTEND && installed && desktopSupportsRouting(destination)) {
     return {
-      href: `preman://open?source=push&route=${encodeURIComponent(route)}`,
+      href: `preman://open?source=${source}&route=${encodeURIComponent(route)}`,
       webUrl,
       surface: "desktop",
     };
   }
+
+  // Deliberately ahead of the "no dashboard is known" return below: not knowing
+  // which browser page holds the run says nothing about whether the app is
+  // installed, and raising it is still the most useful thing available.
+  if (preferApp && installed) {
+    return { href: `preman://open?source=${source}`, webUrl, surface: "desktop-raise" };
+  }
+
+  if (!base) return null;
+  if (base !== DEFAULT_FRONTEND) return { href: webUrl, webUrl, surface: "web-local" };
   return { href: webUrl, webUrl, surface: "web" };
 }
 

package/bin/runner.js

@@ -38,7 +38,10 @@ import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
-import { CREDENTIALS_DIR, backendUrl, callBackendJson, cliInvocation, makeArgs, nowMs, onPath, packageVersion, resolveApiKey } from "./shared.js";
+import { executeEvalRun } from "./eval.js";
+import { discoverTarget, startAdapter, stepFeed } from "./eval_target.js";
+import { resolveEvalSurface } from "./link.js";
+import { CREDENTIALS_DIR, backendUrl, callBackendJson, cliInvocation, makeArgs, nowMs, onPath, openUrl, packageVersion, resolveApiKey } from "./shared.js";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -70,6 +73,11 @@ const ERROR_CAP = 4_000;
 export const RUNNER_CAPABILITIES = {
   tool_call_telemetry_v1: true,
   tool_fact_manifest_v1: true,
+  // `find_eval_runner` requires this before it will lease an eval here, and the
+  // requirement is not cosmetic: the stream parser below ignores an event name
+  // it does not know, so a build without eval support would take the lease,
+  // execute nothing, and look like a device that went offline.
+  agent_eval_v1: true,
   transport: "cli",
   client: "premanmcp",
 };
@@ -398,6 +406,74 @@ async function callback(args, state, route, body, log) {
   }
 }
 
+/**
+ * Give a leased job straight back, because this session will not run it.
+ *
+ * Both protocols reach a terminal state through their own `complete`, and both
+ * accept "this device declines" as a legitimate outcome — which is why this is a
+ * completion with `ok: false` rather than an attempt to un-lease.
+ */
+async function declineJob(args, state, kind, job, log) {
+  const reason =
+    kind === "eval"
+      ? "this session is running a coding-agent job, not evals"
+      : "this session is waiting on an eval and will not run coding-agent jobs";
+  log(`declining ${kind} ${job.id}: ${reason}`);
+  if (kind === "eval") {
+    return callback(
+      args,
+      state,
+      `/workbench/coding-agent/local-runner/evals/${job.id}/complete`,
+      { lease_token: String(job.lease_token || ""), ok: false, error: reason },
+      log
+    );
+  }
+  return callback(
+    args,
+    state,
+    `/workbench/coding-agent/local-runner/jobs/${job.id}/complete`,
+    {
+      lease_token: String(job.lease_token || ""),
+      ok: false,
+      error: reason,
+      attempt_count: Math.max(1, Number(job.attempt_count) || 1),
+      tool_events: [],
+      tool_call_count: 0,
+      metadata: { transport: "cli", declined: true },
+    },
+    log
+  );
+}
+
+/**
+ * Hand a leased eval to `eval.js`, with this runner's way of making a callback.
+ *
+ * The callback shape is injected rather than imported there so the eval executor
+ * has no opinion about tokens or base URLs — this file stays the single place
+ * that knows a runner token goes on a runner request.
+ */
+export async function runLeasedEval(args, state, job, { log = () => {}, headless = true } = {}) {
+  // One callback seam for both kinds of traffic this run produces: JSON for the
+  // progress and completion reports, multipart for the artifact uploads. The
+  // executor should not have to know which transport a given route wants.
+  const call = (route, body) =>
+    callBackendJson(args, "POST", route, {
+      token: state.runner_token,
+      ...(body instanceof FormData ? { form: body } : { json: body }),
+    });
+  const surface = resolveEvalSurface(args, { runId: job.id });
+  const result = await executeEvalRun(job, {
+    call,
+    log,
+    headless,
+    projectPath: state.project_path || process.cwd(),
+    surface: { ...surface, open: () => openUrl(surface.href) },
+  });
+  if (result.reason === "lease_lost") log(`eval ${job.id} was taken over or cancelled`);
+  else log(`eval ${job.id} finished: ${result.ok ? "ok" : "failed"}`);
+  return result;
+}
+
 export async function executeJob(args, state, job, { log = () => {}, fullAccess = false } = {}) {
   const lease = String(job.lease_token || "");
   const jobRoute = `/workbench/coding-agent/local-runner/jobs/${job.id}`;
@@ -600,6 +676,21 @@ function defaultLog(message) {
  * leave a machine that reads as paired and never runs anything again. A missed
  * heartbeat only costs one TTL window: the next one puts this runner back online.
  */
+/**
+ * What this machine currently offers, refreshed on every heartbeat.
+ *
+ * Set by the daemon once it has an adapter listening, and deliberately *not*
+ * persisted in the runner state file. An adapter is a socket this process
+ * opened; a URL for one that outlived the process it belonged to is a URL that
+ * points at nothing, and writing it to disk is how it would come back after a
+ * restart looking authoritative.
+ */
+let advertised = null;
+
+export function advertise(capabilities) {
+  advertised = capabilities;
+}
+
 export async function sendHeartbeat(args, state, { busy = false, log = () => {} } = {}) {
   try {
     // Reported honestly because the matcher prefers idle runners: a busy device
@@ -608,7 +699,17 @@ export async function sendHeartbeat(args, state, { busy = false, log = () => {}
       args,
       "POST",
       "/workbench/coding-agent/local-runner/heartbeat",
-      { token: state.runner_token, json: { state: busy ? "busy" : "idle" } }
+      {
+        token: state.runner_token,
+        json: {
+          state: busy ? "busy" : "idle",
+          // On every beat rather than once at registration, because both facts
+          // it carries expire: an agent restarted on a new port makes the old
+          // URL wrong, not merely stale, and a dashboard offering a Run button
+          // for it would fail on the press.
+          ...(advertised ? { capabilities: advertised } : {}),
+        },
+      }
     );
     return { revoked: result.status_code === 401 };
   } catch (error) {
@@ -617,6 +718,52 @@ export async function sendHeartbeat(args, state, { busy = false, log = () => {}
   }
 }
 
+/**
+ * Find this machine's agent and hold an adapter open for as long as the daemon runs.
+ *
+ * Why the daemon does this at startup rather than when a run arrives: the
+ * dashboard's Run button has to know, *before* anybody presses it, that there is
+ * something here to measure and where it is. That is what turns "copy this
+ * command into a terminal" into a button, and it is only knowable if the address
+ * exists ahead of the request rather than being created in response to one.
+ *
+ * Failure here is not fatal and must not be. A machine paired for fix tasks is
+ * perfectly useful with no agent to evaluate on it, and refusing to start the
+ * daemon because discovery came up empty would break the feature this one is
+ * bolted onto. The daemon says so once and carries on without eval capability.
+ */
+async function serveEvalTarget(args, { log }) {
+  let target;
+  try {
+    target = await discoverTarget(args, { log: () => {} });
+  } catch (error) {
+    log(`no agent to evaluate on this machine (${error.message}); eval runs will not be offered`);
+    return null;
+  }
+
+  try {
+    const adapter = await startAdapter(target, args, {
+      log: () => {},
+      onStep: (step) => stepFeed.push(step),
+    });
+    if (adapter.translated) {
+      // Same reason as the interactive path: the address is a socket this
+      // process opened moments ago, and assert-ai's refusal to fetch private
+      // addresses is right in general and wrong for exactly this.
+      process.env.ASSERT_ALLOW_PRIVATE_ENDPOINTS = "1";
+    }
+    log(`offering ${target.label} for evals`);
+    advertise({
+      agent_eval_v1: true,
+      eval_target: { url: adapter.url, label: target.label, how: target.how },
+    });
+    return adapter;
+  } catch (error) {
+    log(`could not make this machine's agent answerable (${error.message}); eval runs will not be offered`);
+    return null;
+  }
+}
+
 /**
  * Hold the stream and run what it leases, until stopped.
  *
@@ -627,7 +774,17 @@ export async function sendHeartbeat(args, state, { busy = false, log = () => {}
 export async function runnerLoop(
   args,
   state,
-  { log = defaultLog, once = false, stopWhen = null, fullAccess = false } = {}
+  {
+    log = defaultLog,
+    once = false,
+    stopWhen = null,
+    fullAccess = false,
+    only = "",
+    // Headless unless a caller says otherwise. `preman runner` is a background
+    // daemon; only the interactive `preman eval run` has somebody in front of it
+    // to hand a run over to.
+    headless = true,
+  } = {}
 ) {
   let stopped = false;
   let busy = false;
@@ -677,7 +834,11 @@ export async function runnerLoop(
       let terminal = "";
       const push = createSseParser((event, data) => {
         if (event === "connected") log(`connected as runner ${data.runner_id || state.runner_id}`);
-        else if (event === "job") queue.push(data);
+        else if (event === "job") queue.push({ kind: "fix", job: data });
+        // A distinct name rather than a discriminator inside `job`, so a runner
+        // older than eval support ignores the frame instead of reading an eval
+        // as a coding task and looking for a fix task that is not there.
+        else if (event === "eval_job") queue.push({ kind: "eval", job: data });
         else if (event === "revoked" || event === "replaced") terminal = event;
         else if (event === "reconnect") terminal = "reconnect";
       });
@@ -687,11 +848,24 @@ export async function runnerLoop(
         for await (const chunk of response.body) {
           push(decoder.decode(chunk, { stream: true }));
           while (queue.length) {
-            const job = queue.shift();
-            log(`leased job ${job.id}`);
+            const { kind, job } = queue.shift();
+            // `preman eval run` is somebody waiting on one eval, not somebody
+            // volunteering their machine as a coding-agent runner. Executing a
+            // queued fix task because it happened to arrive down the same
+            // stream would edit files they did not ask to have edited.
+            //
+            // Declined rather than dropped: a leased job nobody answers sits
+            // until its lease expires, which delays the fix and tells whoever
+            // queued it nothing. Declining hands it straight back.
+            if (only && only !== kind) {
+              await declineJob(args, state, kind, job, log);
+              continue;
+            }
+            log(`leased ${kind === "eval" ? "eval" : "job"} ${job.id}`);
             busy = true;
             try {
-              await executeJob(args, state, job, { log, fullAccess });
+              if (kind === "eval") await runLeasedEval(args, state, job, { log, headless });
+              else await executeJob(args, state, job, { log, fullAccess });
             } finally {
               busy = false;
             }
@@ -876,6 +1050,14 @@ async function startForeground(args, commandArgs) {
   process.on("uncaughtException", die("uncaught exception"));
   process.on("unhandledRejection", die("unhandled rejection"));
 
+  const adapter = await serveEvalTarget(args, { log });
+
+  // Pushed immediately rather than waiting for the loop's first tick. The
+  // capability only reaches the server on a heartbeat, so without this the
+  // dashboard spends the first interval telling somebody their machine has
+  // found no agent — while the terminal beside it says it has.
+  if (adapter) await sendHeartbeat(args, state, { log });
+
   try {
     const result = await runnerLoop(args, state, {
       log,
@@ -884,6 +1066,10 @@ async function startForeground(args, commandArgs) {
     });
     log(`stopped after ${result.jobsRun} job(s): ${result.reason}`);
   } finally {
+    // Closed on every path. The adapter is an unauthenticated door to this
+    // machine's agent, held open only for as long as something is listening for
+    // work behind it.
+    if (adapter) await adapter.close().catch(() => {});
     if (!shuttingDown) {
       await goOffline(args, state, log);
       rmSync(RUNNER_PID_FILE, { force: true });

package/bin/shared.js

@@ -263,6 +263,39 @@ export function frontendUrl(args) {
     .replace(/\/+$/, "");
 }
 
+/**
+ * The dashboard that can actually show a thing living on *this* backend, or
+ * nothing when there is no way to know.
+ *
+ * `frontendUrl` above states the pairing rule and then does not enforce it: the
+ * stored `frontend_url` is used whatever backend this invocation was pointed
+ * at. For prose that is harmless -- "sign in at ..." wants the dashboard the
+ * customer normally uses, and always has one to name. For a *link to a specific
+ * run* it is a wrong answer dressed as a right one: point `--backend` at a
+ * local API and the run gets a production URL where it does not exist, so the
+ * link resolves, loads, and shows nothing. That is worse than no link, because
+ * an empty dashboard reads as a lost run rather than a bad address.
+ *
+ * So this one refuses to guess. An unrecognised backend with no `--frontend`
+ * and no matching login returns `""`, and callers omit the link. Saying nothing
+ * about where to look is honest; naming somewhere the run has never been is not.
+ */
+export function pairedFrontendUrl(args) {
+  const explicit = args.value("--frontend", process.env.PREMAN_FRONTEND || "");
+  if (explicit) return explicit.replace(/\/+$/, "");
+
+  const backend = backendUrl(args);
+  const login = activeLogin(args);
+  const paired = String(login?.backend_url || "").replace(/\/+$/, "");
+  // Only while this invocation is talking to the deployment the key was minted
+  // against -- which is exactly the condition `frontendUrl`'s docstring names.
+  if (login?.frontend_url && paired && paired === backend) {
+    return String(login.frontend_url).replace(/\/+$/, "");
+  }
+  if (backend === DEFAULT_BACKEND) return DEFAULT_FRONTEND;
+  return "";
+}
+
 /**
  * Open a URL in the user's default browser, best effort.
  *
@@ -428,7 +461,7 @@ export async function callBackendJson(
   args,
   method,
   routePath,
-  { json, token, query, headers: extraHeaders } = {}
+  { json, form, token, query, headers: extraHeaders } = {}
 ) {
   const url = new URL(routePath.replace(/^\/+/, ""), `${backendUrl(args)}/`);
   if (query) {
@@ -439,6 +472,8 @@ export async function callBackendJson(
 
   const headers = { Accept: "application/json" };
   const hasBody = json !== undefined && json !== null;
+  // No Content-Type for a form: fetch sets it from the FormData, and the
+  // multipart boundary it appends is not something a caller can supply.
   if (hasBody) headers["Content-Type"] = "application/json";
   if (token) headers.Authorization = `Bearer ${token}`;
   if (extraHeaders) {
@@ -450,7 +485,7 @@ export async function callBackendJson(
   const resp = await fetch(url, {
     method,
     headers,
-    body: hasBody ? JSON.stringify(json) : undefined,
+    body: form !== undefined && form !== null ? form : hasBody ? JSON.stringify(json) : undefined,
   });
   const text = await resp.text();
   let body = {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "premanmcp",
-  "version": "1.0.4",
+  "version": "1.1.0",
   "description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
   "type": "module",
   "bin": {
@@ -15,7 +15,7 @@
     "test": "npm run build && npm run test:proxy",
     "test:proxy": "node --test scripts/smoke-proxy.mjs",
     "test:connect": "node scripts/smoke-connect.mjs",
-    "test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs",
+    "test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs",
     "test:dmg": "node --test --test-timeout=300000 scripts/smoke-install-desktop-volume.mjs"
   },
   "devDependencies": {