premanmcp 1.1.4 → 1.1.6

package/bin/runner.js

@@ -470,7 +470,10 @@ export async function runLeasedEval(args, state, job, { log = () => {}, headless
     surface: { ...surface, open: () => openUrl(surface.href, { app: surface.app }) },
   });
   if (result.reason === "lease_lost") log(`eval ${job.id} was taken over or cancelled`);
-  else log(`eval ${job.id} finished: ${result.ok ? "ok" : "failed"}`);
+  else if (result.ok) log(`eval ${job.id} finished: ok`);
+  // The reason, on the line that says it failed. Without it this was the whole
+  // of what a terminal learned about a run that did not work.
+  else log(`eval ${job.id} finished: failed${result.error ? ` — ${result.error}` : ""}`);
   return result;
 }
 

package/bin/target.js

@@ -0,0 +1,505 @@
+/**
+ * `preman target` — teach PreMan which agent to test, from the repo it lives in.
+ *
+ * The eval stack can measure an agent it does not own: a session renders
+ * `target: {kind: "endpoint", url}` against whatever address the device reports.
+ * What was missing is everything before that — the customer's agent is on their
+ * machine, behind a route only they know, speaking a shape nobody wrote down,
+ * and the CLI's own discovery only ever looked for *PreMan's* agent.
+ *
+ * So this is the step that turns "point the eval at a URL" into "PreMan found
+ * your agent". It probes what is running (`agent_dialects.js` decides how to
+ * talk to it) and writes the answer into the repository's `.preman.json`, which
+ * is where the pre-push hook already reads its team-level decisions from. In the
+ * repo rather than under `~/.preman` on purpose: the next person to clone this,
+ * and the CI job that will run these evals on every push, need the same answer
+ * without repeating the discovery.
+ *
+ * Nothing here starts a run or spends a provider bill. The worst outcome is a
+ * saved address that turns out to be wrong, and `preman target try` is the
+ * cheap way to find that out before anything is measured.
+ */
+
+import { spawn } from "node:child_process";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import { askAgent, dialectNamed, probeAgent, PROBE_MESSAGE } from "./agent_dialects.js";
+import { detectCandidates } from "./detect.js";
+import { cliInvocation } from "./shared.js";
+
+export const TARGET_HELP = `
+Agent under test:
+  target detect           Find the agent running for this project and remember it
+  target try [message]    Send one message through what was remembered
+  target show             What this repository has saved
+  target forget           Remove it
+  --url <base>            Skip discovery; the agent is here
+  --path <path>           Skip path discovery; the agent answers on this path
+  --port <n>              Also try this port when discovering
+  --start                 Start the project first, using the command it names
+  --start-with <cmd>      Start it with this command instead
+`;
+
+/** Committed to the repository, so the answer travels with the code. */
+export const REPO_CONFIG = ".preman.json";
+
+export function configPath(root = process.cwd()) {
+  return path.join(root, REPO_CONFIG);
+}
+
+/**
+ * The nearest directory at or above `from` that holds a `.preman.json`.
+ *
+ * Sold as "your team and CI find it", and until this it was found only by a
+ * command run from the repository root -- so an eval started from `services/api`
+ * discovered nothing and fell back to probing for PreMan's own agent. Stops at
+ * the filesystem root, and stops early at a `.git` directory, because a config
+ * above the repository belongs to a different project.
+ */
+export function findConfigRoot(from = process.cwd()) {
+  let dir = path.resolve(from);
+  for (;;) {
+    if (existsSync(path.join(dir, REPO_CONFIG))) return dir;
+    if (existsSync(path.join(dir, ".git"))) return null;
+    const parent = path.dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+export function readConfig(root = process.cwd()) {
+  try {
+    const parsed = JSON.parse(readFileSync(configPath(root), "utf8"));
+    return parsed && typeof parsed === "object" ? parsed : {};
+  } catch {
+    // Absent or malformed reads the same way: nothing has been decided here.
+    return {};
+  }
+}
+
+/**
+ * Merge rather than overwrite, and keep the file readable.
+ *
+ * `.preman.json` already carries `blockOnBreaking`, a decision somebody made
+ * about their pushes. Rewriting the file as "the agent" would take that with it.
+ */
+export function saveAgent(agent, root = process.cwd()) {
+  const config = readConfig(root);
+  const next = { ...config, agent };
+  writeFileSync(configPath(root), `${JSON.stringify(next, null, 2)}\n`, "utf8");
+  return next;
+}
+
+export function savedAgent(root = process.cwd()) {
+  const agent = readConfig(findConfigRoot(root) || root).agent;
+  if (!agent || typeof agent !== "object") return null;
+  const url = String(agent.url || "").trim();
+  if (!url) return null;
+  return {
+    url,
+    path: String(agent.path || ""),
+    dialect: String(agent.dialect || "message"),
+    start: String(agent.start || ""),
+  };
+}
+
+/**
+ * Where this project's agent might be listening.
+ *
+ * `detectCandidates` reads the things a project already says about itself --
+ * env files, package scripts, compose, Procfile, framework defaults -- and it
+ * was written for finding an API. An agent is served by the same kinds of
+ * process, so the list is the same list; what differs is the question asked of
+ * each address, which is `probeAgent`'s job rather than this one's.
+ */
+export function candidateUrls(root = process.cwd(), { extraPorts = [] } = {}) {
+  const seen = new Set();
+  const urls = [];
+  const add = (url, why) => {
+    const clean = String(url || "").replace(/\/+$/, "");
+    if (!clean || seen.has(clean)) return;
+    seen.add(clean);
+    urls.push({ url: clean, why });
+  };
+
+  for (const candidate of detectCandidates(root, { extraPorts })) {
+    // `signals` is what a candidate carries -- ".env:PORT", "package.json:dev".
+    // Reading a `detail` it has never had printed the raw enum instead, so
+    // every address said "env_file" whatever had actually named it.
+    add(candidate.url, candidate.signals?.[0] || candidate.source || "");
+  }
+  // Then the ports things listen on anyway. A project that says nothing about
+  // itself is the common case, not the edge one -- `node agent.mjs` names no
+  // port, and a repo with no compose file, no Procfile and no framework in its
+  // dependencies leaves detection with nothing to read. Guessing is cheap here
+  // because a port with nothing behind it refuses the connection immediately.
+  for (const port of COMMON_PORTS) add(`http://127.0.0.1:${port}`, `port ${port}`);
+  return urls;
+}
+
+/**
+ * Where an agent ends up when nobody decided.
+ *
+ * Framework defaults first (Next and Vite on 3000 and 5173, FastAPI and Django
+ * on 8000), then the ports people reach for when writing a server by hand.
+ */
+export const COMMON_PORTS = [3000, 8000, 8080, 5000, 5173, 4000, 8001, 9000, 7860];
+
+/**
+ * How long a project gets to come up before this gives up on it.
+ *
+ * A minute because the first run of a dev server is the slow one -- installing,
+ * compiling, warming a model client -- and a developer who has just been told
+ * PreMan will start their agent will wait through it once. It polls every
+ * second rather than sleeping the whole minute, so a fast project costs a
+ * second and not a minute.
+ */
+export const WAIT_FOR_START_MS = 60_000;
+
+/** How long a project gets to stop politely before it is killed. */
+export const STOP_GRACE_MS = 5_000;
+
+/**
+ * How this project starts itself.
+ *
+ * Read from the files a project already uses to say so, in the order of how
+ * much of a decision each one is: a command somebody saved here beats a script
+ * in `package.json`, which beats a Procfile's web process, which beats a guess
+ * from whichever framework's files are lying around.
+ *
+ * Guesses are included and labelled as guesses, because the alternative for a
+ * developer whose repo says nothing is to type the command themselves -- which
+ * is fine, and is what `--start "<cmd>"` is for, but should not be the first
+ * thing asked of somebody who ran `preman target detect` expecting it to work.
+ */
+export function startCommands(root = process.cwd()) {
+  const found = [];
+  const add = (command, why) => {
+    const clean = String(command || "").trim();
+    if (clean && !found.some((entry) => entry.command === clean)) found.push({ command: clean, why });
+  };
+
+  const saved = savedAgent(root);
+  if (saved?.start) add(saved.start, `${REPO_CONFIG} remembers it`);
+
+  const pkg = readJson(path.join(root, "package.json"));
+  for (const name of ["dev", "start", "serve", "dev:server", "server"]) {
+    if (pkg?.scripts?.[name]) add(`npm run ${name}`, `package.json has a ${name} script`);
+  }
+
+  const procfile = readText(path.join(root, "Procfile"));
+  const web = /^web:\s*(.+)$/m.exec(procfile);
+  if (web) add(web[1], "the Procfile's web process");
+
+  for (const name of ["docker-compose.yml", "docker-compose.yaml", "compose.yml", "compose.yaml"]) {
+    if (existsSync(path.join(root, name))) {
+      add("docker compose up", `${name} is here`);
+      break;
+    }
+  }
+
+  // Python projects rarely write the command down anywhere a machine can read,
+  // so these are read off the files that exist rather than off configuration.
+  if (existsSync(path.join(root, "manage.py"))) add("python manage.py runserver", "a guess, from manage.py");
+  if (existsSync(path.join(root, "main.py"))) add("python main.py", "a guess, from main.py");
+  if (existsSync(path.join(root, "app.py"))) add("python app.py", "a guess, from app.py");
+
+  return found;
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function readText(file) {
+  try {
+    return readFileSync(file, "utf8");
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Run the project, wait for its agent to answer, and hand back a way to stop it.
+ *
+ * Started in its own process group and stopped by group, because the command is
+ * nearly always a wrapper -- `npm run dev` is npm, which is a shell, which is
+ * the server -- and killing the pid we spawned leaves the server holding the
+ * port. A developer whose next run says "something is already on 3000" would
+ * have PreMan to thank for it.
+ *
+ * Output is captured rather than inherited: a dev server's log is not this
+ * command's output, and interleaving it with discovery makes both unreadable.
+ * It is kept, and printed only when the wait fails, which is the one moment it
+ * is the most useful thing on the screen.
+ */
+export async function startProject(command, { root = process.cwd(), waitMs = 60_000, log = () => {} } = {}) {
+  const child = spawn(command, {
+    cwd: root,
+    shell: true,
+    detached: true,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  let output = "";
+  const keep = (chunk) => {
+    if (output.length < 8000) output += chunk.toString("utf8");
+  };
+  child.stdout.on("data", keep);
+  child.stderr.on("data", keep);
+
+  let exited = null;
+  child.on("exit", (code, signal) => {
+    exited = signal || code;
+  });
+
+  const signal = (name) => {
+    try {
+      // Negative pid: the group, not just the wrapper.
+      process.kill(-child.pid, name);
+    } catch {
+      try {
+        child.kill(name);
+      } catch {
+        // Already gone, which is the state being asked for.
+      }
+    }
+  };
+
+  const stop = async () => {
+    if (exited !== null) return;
+    signal("SIGTERM");
+    // Then SIGKILL, and the pipes closed behind it. A dev server that ignores
+    // SIGTERM keeps this process alive forever otherwise: its stdout is still
+    // open here, so Node has a reason to stay, and `preman target detect`
+    // simply never returns.
+    const deadline = Date.now() + STOP_GRACE_MS;
+    while (exited === null && Date.now() < deadline) {
+      await new Promise((resume) => setTimeout(resume, 100));
+    }
+    if (exited === null) {
+      log(`${command.split(" ")[0]} did not stop when asked; killing it`);
+      signal("SIGKILL");
+    }
+    child.stdout?.destroy();
+    child.stderr?.destroy();
+  };
+
+  return { child, stop, output: () => output, exited: () => exited };
+}
+
+/** A base address and the path on it, from whichever form somebody typed. */
+export function splitUrl(value) {
+  const raw = String(value || "").trim();
+  if (!raw) return { base: "", path: "" };
+  // `localhost:3000` is a URL to a person and a `localhost:` scheme to the
+  // parser, which turns the port into the path. Anything that is not http(s)
+  // is treated as the host somebody meant.
+  const candidate = /^https?:\/\//i.test(raw) ? raw : `http://${raw}`;
+  try {
+    const url = new URL(candidate);
+    const path = url.pathname === "/" ? "" : url.pathname.replace(/\/+$/, "");
+    return { base: `${url.protocol}//${url.host}`, path };
+  } catch {
+    return { base: raw.replace(/\/+$/, ""), path: "" };
+  }
+}
+
+/** Try each candidate until one answers like an agent. */
+export async function findAgent(root, { url = "", paths = null, extraPorts = [], log = () => {} } = {}) {
+  const candidates = url
+    ? [{ url: String(url).replace(/\/+$/, ""), why: "you named it" }]
+    : candidateUrls(root, { extraPorts });
+
+  if (!candidates.length) return { found: null, tried: [] };
+
+  const tried = [];
+  for (const candidate of candidates) {
+    log(`looking at ${candidate.url}${candidate.why ? ` (${candidate.why})` : ""}`);
+    const result = await probeAgent(candidate.url, paths ? { paths } : {});
+    if (result.dialect) return { found: result, tried };
+    tried.push({ url: candidate.url, why: summarise(result.tried) });
+  }
+  return { found: null, tried };
+}
+
+/** The one sentence a failed sweep can honestly offer about an address. */
+function summarise(attempts = []) {
+  if (!attempts.length) return "nothing answered";
+  const statuses = attempts.filter((a) => /^\d{3}/.test(a.why || ""));
+  if (statuses.length === attempts.length) return "answered, but not like an agent";
+  const first = attempts.find((a) => a.why && !/^\d{3}/.test(a.why));
+  return first?.why || "nothing answered";
+}
+
+export async function targetCommand(commandArgs = [], { makeArgs, root = process.cwd() } = {}) {
+  const args = makeArgs(commandArgs);
+  const sub = commandArgs.find((value) => !value.startsWith("-")) || "detect";
+  const say = (line = "") => process.stdout.write(`${line}\n`);
+  const cli = cliInvocation();
+
+  if (sub === "show") {
+    const agent = savedAgent(root);
+    if (!agent) {
+      say(`Nothing saved in ${REPO_CONFIG}. Run \`${cli} target detect\`.`);
+      return { ok: false, reason: "nothing_saved" };
+    }
+    say(`${agent.url}${agent.path}`);
+    say(`  speaks ${agent.dialect}`);
+    if (agent.start) say(`  started with ${agent.start}`);
+    return { ok: true, agent };
+  }
+
+  if (sub === "forget") {
+    const config = readConfig(root);
+    if (!config.agent) {
+      say(`Nothing saved in ${REPO_CONFIG}.`);
+      return { ok: true, forgot: false };
+    }
+    delete config.agent;
+    writeFileSync(configPath(root), `${JSON.stringify(config, null, 2)}\n`, "utf8");
+    say(`Forgotten. ${REPO_CONFIG} no longer names an agent.`);
+    return { ok: true, forgot: true };
+  }
+
+  if (sub === "try") {
+    const agent = savedAgent(root);
+    if (!agent) {
+      say(`Nothing saved to try. Run \`${cli} target detect\` first.`);
+      return { ok: false, reason: "nothing_saved" };
+    }
+    const message = commandArgs.filter((value) => value !== "try" && !value.startsWith("-")).join(" ") || PROBE_MESSAGE;
+    say(`> ${message}`);
+    try {
+      const { text, events } = await askAgent(agent, message);
+      say("");
+      say(text);
+      if (events.length) {
+        say("");
+        say(`tools it used: ${events.map((event) => event.tool_name).join(", ")}`);
+      } else {
+        // Said plainly rather than left blank: a run against an agent whose
+        // tool calls are invisible measures what it said and not what it did,
+        // and that is a different product promise.
+        say("");
+        say("no tool calls in the answer, so an eval will score what it says, not what it does");
+      }
+      return { ok: true, text, events };
+    } catch (error) {
+      say(`could not get an answer: ${error.message}`);
+      return { ok: false, reason: "no_answer" };
+    }
+  }
+
+  if (sub !== "detect") {
+    say(`Unknown target subcommand: ${sub}${TARGET_HELP}`);
+    return { ok: false, reason: "unknown_subcommand" };
+  }
+
+  // `--url http://localhost:3000/api/chat` is what people type, and it is the
+  // same thing as `--url http://localhost:3000 --path /api/chat`. Splitting it
+  // here rather than appending to it is the difference between probing the
+  // address they named and probing `/api/chat/chat`.
+  const named = splitUrl(args.value("--url", ""));
+  const explicit = named.base;
+  const explicitPath = args.value("--path", "") || named.path;
+  const port = args.value("--port", "");
+  say("Looking for the agent this project runs...");
+
+  let { found, tried } = await findAgent(root, {
+    url: explicit,
+    paths: explicitPath ? [explicitPath] : null,
+    extraPorts: port ? [port] : [],
+    log: (line) => say(`  ${line}`),
+  });
+
+  // Nothing running, and permission to do something about it. Starting is
+  // opt-in and never inferred: this runs a command out of somebody's repository,
+  // which is the one thing in this CLI that executes code nobody typed here.
+  const startWith = args.value("--start-with", "");
+  const mayStart = startWith || args.has("--start");
+  let started = null;
+  let startCommand = "";
+
+  if (!found && mayStart) {
+    const commands = startCommands(root);
+    startCommand = startWith || commands[0]?.command || "";
+    if (!startCommand) {
+      say("");
+      say("Nothing here says how this project starts.");
+      say(`  ${cli} target detect --start-with "npm run dev"`);
+      return { ok: false, reason: "no_start_command" };
+    }
+    const why = startWith ? "you named it" : commands[0].why;
+    say("");
+    say(`Starting the project: ${startCommand}  (${why})`);
+    started = await startProject(startCommand, { root });
+
+    const deadline = Date.now() + WAIT_FOR_START_MS;
+    while (Date.now() < deadline && started.exited() === null) {
+      await new Promise((resume) => setTimeout(resume, 1000));
+      const retry = await findAgent(root, {
+        url: explicit,
+        paths: explicitPath ? [explicitPath] : null,
+        extraPorts: port ? [port] : [],
+      });
+      if (retry.found) {
+        found = retry.found;
+        break;
+      }
+    }
+
+    await started.stop();
+    if (!found) {
+      say("");
+      say(
+        started.exited() === null
+          ? `It started, but nothing answered like an agent within ${Math.round(WAIT_FOR_START_MS / 1000)}s.`
+          : `That command exited (${started.exited()}) before anything answered.`
+      );
+      const log = started.output().trim();
+      // The log is the useful half of this failure and is otherwise thrown
+      // away: a missing env var says so in line three of a dev server's output.
+      if (log) {
+        say("");
+        say(log.split("\n").slice(-12).join("\n"));
+      }
+      return { ok: false, reason: "start_failed" };
+    }
+  }
+
+  if (!found) {
+    say("");
+    say("No agent answered here.");
+    for (const attempt of tried) say(`  ${attempt.url} — ${attempt.why}`);
+    say("");
+    say("Start it yourself and run this again, or let PreMan start it:");
+    say(`  ${cli} target detect --start`);
+    say(`  ${cli} target detect --url http://localhost:3000 --path /api/chat`);
+    return { ok: false, reason: "not_found", tried };
+  }
+
+  const agent = { url: found.url, path: found.path, dialect: found.dialect };
+  if (startCommand) agent.start = startCommand;
+  saveAgent(agent, root);
+
+  say("");
+  say(`Found it: ${agent.url}${agent.path}`);
+  say(`  it speaks ${dialectNamed(agent.dialect)?.summary || agent.dialect}`);
+  say(`  it said: ${found.sample}`);
+  if (found.events?.length) {
+    say(`  and it reported using ${found.events.map((event) => event.tool_name).join(", ")}`);
+  }
+  say("");
+  if (agent.start) say(`  it starts with ${agent.start}`);
+  say("");
+  say(`Saved to ${REPO_CONFIG}, so your team and CI find it without repeating this.`);
+  say(`  ${cli} target try "I want a refund"   to send it something`);
+  say(`  ${cli} eval                            to measure it`);
+  return { ok: true, agent };
+}

package/dist/server.js

@@ -8,6 +8,12 @@ import { createInterface } from "node:readline";
 import { readFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+// The stdio bridge is the process an MCP host launches and nobody started on
+// purpose, so it is the one most likely to be found later as an unexplained
+// `node` in a process list. Named apart from the CLI and the session, because
+// "which of these three is the one my editor spawned" is the question being
+// asked when somebody goes looking.
+process.title = "preman-mcp";
 let sessionId = "";
 let protocolVersion = "2025-06-18";
 async function credentials() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "premanmcp",
-  "version": "1.1.4",
+  "version": "1.1.6",
   "description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
   "type": "module",
   "bin": {
@@ -15,7 +15,7 @@
     "test": "npm run build && npm run test:proxy",
     "test:proxy": "node --test scripts/smoke-proxy.mjs",
     "test:connect": "node scripts/smoke-connect.mjs",
-    "test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs",
+    "test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs",
     "test:dmg": "node --test --test-timeout=300000 scripts/smoke-install-desktop-volume.mjs"
   },
   "devDependencies": {