premanmcp 1.0.4 → 1.1.0

package/bin/eval_target.js

@@ -0,0 +1,530 @@
+/**
+ * Finding the agent to evaluate, and making it answerable.
+ *
+ * Two jobs that look like one, and separating them is the whole design.
+ *
+ * **Finding it.** Nobody should have to tell PreMan where their agent is. The
+ * answer is a port on this machine, and the machine already knows: the project
+ * is right here, the process is running, and the URL is in a config file or a
+ * listening socket. Asking the customer to paste it is asking them to look up
+ * something we could have read.
+ *
+ * **Making it answerable.** assert-ai drives a target by POSTing
+ * `{message, history}` and expecting `{response}` back. Almost no real agent
+ * speaks that, and the ones that do are the ones somebody already wrote an
+ * adapter for. So this module stands up the adapter: a local server that speaks
+ * assert-ai's contract on one side and the agent's own API on the other,
+ * carrying whatever credentials the agent needs.
+ *
+ * The adapter is the reason the credentials story works. It runs here, on the
+ * developer's machine, holding the developer's key. PreMan's servers render the
+ * spec and store the results, and at no point does a key for the agent under
+ * test leave this process.
+ *
+ * ## Why the adapter is not optional even for an agent that "already has an HTTP API"
+ *
+ * A chat API is not a turn API. PreMan's own is the worked example: a
+ * conversation is a server-side object you create once and then append to, so
+ * `history` is not a field you send — it is state the server already holds. An
+ * adapter that ignored that and replayed the whole history every turn would be
+ * measuring a different agent than the one customers talk to. So the mapping
+ * from "assert-ai's stateless turn" to "this agent's idea of a conversation"
+ * is per-agent work, and this is where it lives.
+ */
+
+import { createServer } from "node:http";
+import { createHash } from "node:crypto";
+
+import { backendUrl, resolveApiKey } from "./shared.js";
+
+export class TargetError extends Error {}
+
+/** Ports a locally-running PreMan backend is plausibly on, in order of likelihood. */
+const LOCAL_PORTS = [8000, 8001, 8010, 8011, 8002, 8080];
+
+const PROBE_TIMEOUT_MS = 1_500;
+
+/** The route that makes a service *the agent* rather than merely a PreMan service. */
+const AGENT_ROUTE = "/workbench/conversations/{conversation_id}/messages";
+
+async function get(url, { timeout = PROBE_TIMEOUT_MS } = {}) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeout);
+  try {
+    return await fetch(url, { signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Is the agent we know how to drive answering here?
+ *
+ * Two questions, and asking only the first is what makes discovery pick the
+ * wrong thing. `/health` says a PreMan-shaped service is on this port; it does
+ * not say the service has a chat agent. A developer's machine can easily have an
+ * old build, a cut-down demo API, or a differently-scoped deployment answering
+ * `/health` with the right name — and pointing an eval at one produces a run
+ * where every case fails with a 404 that looks like the agent misbehaving.
+ *
+ * So the second question is asked of the service's own OpenAPI document: does it
+ * publish the route the adapter is about to POST to. That is a capability check
+ * rather than a version check, which means it keeps working when the route moves
+ * behind a different service and stops working the moment it is renamed — both
+ * of which are the answers you want.
+ */
+async function probeAgent(url) {
+  try {
+    const health = await get(`${url}/health`);
+    if (!health.ok) return { url, ok: false, why: `health answered ${health.status}` };
+    const body = await health.json().catch(() => ({}));
+    const service = String(body?.service || "");
+    if (!/preman/i.test(service)) {
+      return { url, ok: false, why: `something else is on this port (${service || "no service name"})` };
+    }
+
+    const spec = await get(`${url}/openapi.json`, { timeout: 6_000 });
+    if (!spec.ok) return { url, ok: false, why: `a PreMan service, but it publishes no API document` };
+    const document = await spec.json().catch(() => ({}));
+    if (!document?.paths?.[AGENT_ROUTE]) {
+      return { url, ok: false, why: `a PreMan service, but it has no chat agent on it` };
+    }
+    return { url, ok: true, service };
+  } catch (error) {
+    return { url, ok: false, why: error.name === "AbortError" ? "no answer" : error.message };
+  }
+}
+
+/**
+ * Work out which agent this eval is about.
+ *
+ * The order is deliberate and it is "most explicit first". A `--target` the
+ * customer typed beats anything discovered, because they have already answered
+ * the question and a tool that overrode them would be unusable in the one
+ * environment they configured it for. Then a backend on this machine, because
+ * somebody running an eval from a project directory with a dev server up almost
+ * certainly means that one. Then the configured backend, which is the honest
+ * fallback and is also the answer for anyone who is not a PreMan developer.
+ */
+export async function discoverTarget(args, { log = () => {} } = {}) {
+  const explicit = String(args.value("--target", "") || "").trim();
+  if (explicit) {
+    return {
+      kind: "raw",
+      url: explicit,
+      label: explicit,
+      how: "you named it with --target, so nothing was probed",
+      probed: [],
+    };
+  }
+
+  const probed = [];
+  for (const port of LOCAL_PORTS) {
+    const result = await probeAgent(`http://127.0.0.1:${port}`);
+    probed.push(result);
+    if (result.ok) {
+      log(`Found a PreMan backend on port ${port}.`);
+      return {
+        kind: "preman-chat",
+        baseUrl: `http://127.0.0.1:${port}`,
+        // The agent's own credential, held here and never sent to PreMan's
+        // control plane. The adapter is the only thing that ever presents it,
+        // which is the whole reason the adapter runs on this machine.
+        token: resolveApiKey(args),
+        label: `PreMan agent (local, port ${port})`,
+        how: `probed localhost and got a PreMan /health on port ${port}`,
+        probed,
+      };
+    }
+  }
+
+  const configured = backendUrl(args);
+  const remote = await probeAgent(configured);
+  probed.push(remote);
+  if (remote.ok) {
+    log(`No local backend; using the one this CLI is configured against.`);
+    return {
+      kind: "preman-chat",
+      baseUrl: configured,
+      token: resolveApiKey(args),
+      label: `PreMan agent (${configured})`,
+      how: `nothing local answered, so this is the backend the CLI is configured against`,
+      probed,
+    };
+  }
+
+  throw new TargetError(
+    `Could not find an agent to evaluate. Tried ${probed.length} address(es): ` +
+      probed.map((p) => `${p.url} (${p.why})`).join(", ") +
+      `. Point at one directly with --target <url>, where that URL accepts ` +
+      `POST {"message","history"} and answers {"response"}.`
+  );
+}
+
+// ── What the agent said it was doing ────────────────────────────────────
+
+/**
+ * How many recent steps travel with a progress tick.
+ *
+ * The wire allows twenty and four of those are the pipeline stages, so this is
+ * what is left. It is a window on purpose: somebody watching wants the last few
+ * seconds, and a longer tail would push the stage summary off a small panel to
+ * show steps that have already been superseded.
+ */
+const STEP_WINDOW = 16;
+
+/**
+ * The steps the adapter has seen, waiting to be reported.
+ *
+ * A module-level singleton, which is worth defending. Two things need it and
+ * they sit on opposite sides of the CLI: the adapter fills it from inside a
+ * turn, and the heartbeat drains it from inside the job loop. Threading a
+ * buffer between them would mean a parameter on `runnerLoop` and on everything
+ * it calls, all to carry something that is per-process anyway — this CLI runs
+ * one adapter and one eval at a time, by construction.
+ *
+ * Bounded, and dropped rather than queued when it overflows. This is a view of
+ * what is happening now; a backlog of steps from a minute ago is not a thing
+ * anybody is waiting to see, and keeping it would grow with the run.
+ */
+function createStepFeed() {
+  let entries = [];
+  return {
+    push(entry) {
+      entries.push(entry);
+      if (entries.length > STEP_WINDOW) entries = entries.slice(-STEP_WINDOW);
+    },
+    recent() {
+      return entries.slice();
+    },
+    /** Between runs, so one eval's feed cannot appear under the next one's. */
+    reset() {
+      entries = [];
+    },
+  };
+}
+
+export const stepFeed = createStepFeed();
+
+// ── The adapter ─────────────────────────────────────────────────────────
+
+/**
+ * Which PreMan conversation continues this exchange.
+ *
+ * assert-ai sends the whole history every turn and holds no server-side handle;
+ * PreMan holds the conversation and expects only the new message. So the history
+ * has to be turned back into an identity, and the identity is a hash of it.
+ *
+ * Hashing the *prior* turns rather than all of them is what makes the lookup
+ * work: on turn three assert-ai sends [u1, a1, u2, a2, u3], and what we stored
+ * after turn two was keyed on [u1, a1, u2, a2]. Dropping the trailing user
+ * message is what turns "here is everything so far" into "here is the
+ * conversation you already have".
+ */
+function conversationKey(history) {
+  const prior = history.filter((m) => m && m.role && typeof m.content === "string");
+  while (prior.length && prior[prior.length - 1].role === "user") prior.pop();
+  if (!prior.length) return "";
+  return createHash("sha256")
+    .update(JSON.stringify(prior.map((m) => [m.role, m.content])))
+    .digest("hex");
+}
+
+function readBody(request) {
+  return new Promise((resolve, reject) => {
+    let raw = "";
+    request.on("data", (chunk) => {
+      raw += chunk;
+      // A turn is a sentence. Anything past this is a client that has lost
+      // track of what it is sending, and buffering it would be this process
+      // absorbing somebody else's bug as memory pressure.
+      if (raw.length > 2 * 1024 * 1024) reject(new Error("request body too large"));
+    });
+    request.on("end", () => {
+      try {
+        resolve(raw ? JSON.parse(raw) : {});
+      } catch (error) {
+        reject(error);
+      }
+    });
+    request.on("error", reject);
+  });
+}
+
+/**
+ * One turn against PreMan's own agent.
+ *
+ * Tool calls come back in `turn.artifacts` and are forwarded as adapter-shaped
+ * `events`, which is the difference between the judge seeing a transcript and
+ * the judge seeing what the agent *did*. `HTTPEndpointSession` promotes those to
+ * first-class interaction messages; without them a tool-using agent is scored on
+ * its prose alone, which is exactly how a confidently-worded wrong action passes.
+ */
+async function post(target, route, body) {
+  const response = await fetch(new URL(route, `${target.baseUrl}/`), {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      accept: "application/json",
+      authorization: `Bearer ${target.token}`,
+    },
+    body: JSON.stringify(body),
+  });
+  const text = await response.text();
+  let parsed = {};
+  try {
+    parsed = text ? JSON.parse(text) : {};
+  } catch {
+    parsed = { raw: text };
+  }
+  if (!response.ok) {
+    const detail = typeof parsed.detail === "string" ? parsed.detail : text.slice(0, 200);
+    throw new Error(`${route} answered ${response.status}: ${detail}`);
+  }
+  return parsed;
+}
+
+/**
+ * Read one SSE turn, announcing each step as it arrives.
+ *
+ * The streaming endpoint is used for the *live* half of the eval, not the
+ * scored half. Both arrive here: `status` frames are the agent narrating its own
+ * tool activity while the turn is still running, and the `done` frame carries
+ * the identical persisted turn the JSON endpoint would have returned. So
+ * streaming costs nothing in fidelity and buys the only view of a run that
+ * updates faster than a case row.
+ *
+ * What a `status` label is, and what it is not: it is prose the agent wrote for
+ * a human — "Running smoke tests…" — emitted immediately before it invokes a
+ * tool. It is a truthful signal that a tool call is happening now, and it is
+ * *not* the tool's name or its arguments. Nothing scored is derived from it;
+ * the judge reads `turn.artifacts`, same as before. Treating these as evidence
+ * would be scoring an agent on its own commentary.
+ */
+async function streamTurn(target, conversationId, message, state) {
+  const response = await fetch(
+    new URL(
+      `workbench/conversations/${encodeURIComponent(conversationId)}/messages/stream`,
+      `${target.baseUrl}/`
+    ),
+    {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        accept: "text/event-stream",
+        authorization: `Bearer ${target.token}`,
+      },
+      body: JSON.stringify({ content: message }),
+    }
+  );
+
+  if (!response.ok || !response.body) {
+    throw new Error(`the stream answered ${response.status}`);
+  }
+
+  let buffered = "";
+  let turn = null;
+  let failure = "";
+
+  for await (const chunk of response.body) {
+    buffered += Buffer.from(chunk).toString("utf8");
+    // Frames are `data: {json}\n\n`. Splitting on the blank line rather than on
+    // every newline is what keeps a pretty-printed payload in one piece.
+    const frames = buffered.split("\n\n");
+    buffered = frames.pop() ?? "";
+
+    for (const frame of frames) {
+      const line = frame.split("\n").find((l) => l.startsWith("data:"));
+      if (!line) continue;
+      let event;
+      try {
+        event = JSON.parse(line.slice(5).trim());
+      } catch {
+        continue; // A frame we cannot read is not a reason to fail the turn.
+      }
+
+      if (event.type === "status" && event.label) {
+        state.step(String(event.label));
+      } else if (event.type === "done") {
+        turn = event.turn || {};
+      } else if (event.type === "error") {
+        failure = String(event.error || event.detail || "the turn failed");
+      }
+    }
+  }
+
+  if (turn) return turn;
+  throw new Error(failure || "the stream ended without finishing the turn");
+}
+
+async function premanTurn(target, args, { message, history }, state) {
+  const key = conversationKey(history || []);
+  let conversationId = key ? state.conversations.get(key) : "";
+
+  if (!conversationId) {
+    const created = await post(target, "workbench/conversations", {
+      title: "PreMan agent eval",
+    });
+    conversationId = String(created.id || created.conversation_id || "");
+    if (!conversationId) throw new Error("the agent created a conversation with no id");
+  }
+
+  let turn;
+  try {
+    turn = await streamTurn(target, conversationId, message, state);
+  } catch (error) {
+    // The live view is a nicety; the measurement is not. An agent whose build
+    // predates the streaming route, or a proxy that buffers SSE into
+    // uselessness, should produce a run with no step feed rather than no run.
+    state.streamFailures += 1;
+    if (state.streamFailures === 1) {
+      state.log(`no live steps from this agent (${error.message}); measuring it anyway`);
+    }
+    const sent = await post(
+      target,
+      `workbench/conversations/${encodeURIComponent(conversationId)}/messages`,
+      { content: message }
+    );
+    turn = sent.turn || {};
+  }
+
+  const reply = String(turn.content || "");
+
+  // Keyed on the exchange that just happened, so the next turn's history hashes
+  // to this same conversation.
+  const nextKey = createHash("sha256")
+    .update(
+      JSON.stringify([
+        ...(history || [])
+          .filter((m) => m && m.role && typeof m.content === "string")
+          .map((m) => [m.role, m.content]),
+        ["assistant", reply],
+      ])
+    )
+    .digest("hex");
+  state.conversations.set(nextKey, conversationId);
+
+  return { response: reply, events: eventsFrom(turn) };
+}
+
+/**
+ * PreMan's `turn.artifacts` as the events assert-ai understands.
+ *
+ * Best-effort and deliberately lossy: only artifacts that clearly describe a
+ * tool invocation become events. Forwarding everything would put rendered
+ * markdown and UI intents in front of the judge as though they were actions the
+ * agent took, and a judge scoring "did it do something it should not have" would
+ * be reading the wrong evidence.
+ */
+function eventsFrom(turn) {
+  const artifacts = Array.isArray(turn.artifacts) ? turn.artifacts : [];
+  const events = [];
+  for (const artifact of artifacts) {
+    const name = artifact?.tool || artifact?.name || artifact?.tool_name;
+    if (!name) continue;
+    events.push({ type: "tool_call", name: String(name), arguments: artifact.arguments ?? artifact.input ?? {} });
+    if (artifact.result !== undefined || artifact.output !== undefined) {
+      events.push({
+        type: "tool_result",
+        name: String(name),
+        result: artifact.result ?? artifact.output,
+      });
+    }
+  }
+  return events;
+}
+
+/**
+ * Serve assert-ai's contract on loopback until the eval is done.
+ *
+ * Bound to 127.0.0.1 explicitly rather than to every interface. The only client
+ * is a subprocess on this machine, and a target endpoint reachable from the
+ * network is a small unauthenticated proxy to the customer's own agent, held
+ * open for the length of a run.
+ *
+ * Port zero, so the OS picks. A fixed port would collide with the agent, with a
+ * previous run that has not finished tearing down, or with whatever else the
+ * developer has open — and every one of those failures reads as "the eval
+ * couldn't reach your agent".
+ */
+export async function startAdapter(target, args, { log = () => {}, onStep = null } = {}) {
+  if (target.kind === "raw") {
+    // Nothing to translate: the customer named something that already speaks
+    // this. Standing a proxy in front of it would add a hop that can fail and
+    // buy nothing.
+    //
+    // No step feed either, and that is honest rather than a gap: a URL the
+    // customer named speaks assert-ai's turn contract and nothing else, so
+    // there is no channel on which it could narrate itself.
+    return { url: target.url, close: async () => {}, translated: false };
+  }
+
+  const state = {
+    conversations: new Map(),
+    turns: 0,
+    failures: 0,
+    streamFailures: 0,
+    log,
+    /**
+     * One thing the agent said it was doing, on its way to the dashboard.
+     *
+     * Ids are positional rather than derived from the label, because the same
+     * label recurs legitimately — an agent that runs smoke tests three times
+     * says so three times — and collapsing those would under-report the work.
+     */
+    step: (label) => {
+      if (!onStep) return;
+      state.steps += 1;
+      onStep({ id: `step.${state.turns}.${state.steps}`, label, state: "complete" });
+    },
+    steps: 0,
+  };
+
+  const server = createServer((request, response) => {
+    const reply = (status, body) => {
+      const text = JSON.stringify(body);
+      response.writeHead(status, {
+        "content-type": "application/json",
+        "content-length": Buffer.byteLength(text),
+      });
+      response.end(text);
+    };
+
+    if (request.method !== "POST") return reply(405, { error: "post a turn here" });
+
+    void (async () => {
+      try {
+        const body = await readBody(request);
+        state.turns += 1;
+        const result = await premanTurn(target, args, body, state);
+        reply(200, result);
+      } catch (error) {
+        state.failures += 1;
+        // 200 with an error string in `response`, not a 5xx. A transport failure
+        // aborts the case and loses it; an agent that said something unhelpful is
+        // a transcript the judge can score. Which of those this is depends on
+        // whose fault it was, and from in here it usually is not assert-ai's.
+        log(`adapter turn failed: ${error.message}`);
+        reply(200, { response: `[the agent could not answer: ${error.message}]` });
+      }
+    })();
+  });
+
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", resolve);
+  });
+
+  const { port } = server.address();
+  const url = `http://127.0.0.1:${port}/chat`;
+  log(`Adapter listening on ${url} -> ${target.label}`);
+
+  return {
+    url,
+    translated: true,
+    stats: state,
+    close: async () => {
+      await new Promise((resolve) => server.close(resolve));
+    },
+  };
+}

package/bin/hook.js

@@ -139,6 +139,57 @@ function isGeneratedInvocation(invocation) {
   return invocation === "preman" || /^npm exec -y premanmcp@\S+ --$/.test(invocation);
 }
 
+/**
+ * The release a generated invocation pins, or "" when it does not pin one.
+ *
+ * The bare `preman` form resolves at push time, and `@latest` is whatever we
+ * ship next, so neither can be out of date. Only an explicit version can be.
+ */
+export function pinnedRelease(invocation) {
+  const match = /^npm exec -y premanmcp@(\S+) --$/.exec(String(invocation || "").trim());
+  const version = match ? match[1] : "";
+  return version === "latest" ? "" : version;
+}
+
+/**
+ * Is `left` an earlier release than `right`?
+ *
+ * Anything this cannot read as three numbers compares as not-older, so a
+ * prerelease tag or a mirror's own numbering is left alone rather than being
+ * rewritten on a guess.
+ */
+export function isOlderRelease(left, right) {
+  const parse = (value) => String(value).split(".").map((part) => Number.parseInt(part, 10));
+  const a = parse(left);
+  const b = parse(right);
+  if (a.length !== 3 || b.length !== 3) return false;
+  if ([...a, ...b].some((part) => !Number.isFinite(part))) return false;
+  for (let index = 0; index < 3; index += 1) {
+    if (a[index] !== b[index]) return a[index] < b[index];
+  }
+  return false;
+}
+
+/**
+ * Does this hook run a PreMan older than the one asking?
+ *
+ * A pinned hook keeps working long after it stops being current -- it answers
+ * `help`, it runs `verify`, it just does all of it as the version that wrote
+ * it. Found in the wild: a hook still on 0.13.0 while 1.0.4 was installed,
+ * printing an error message whose formatting had been fixed several releases
+ * earlier, with nothing anywhere to say why.
+ *
+ * Only ever true when we would be moving forward. An older CLI run for some
+ * other reason must not drag a newer hook back with it.
+ */
+export function hookIsBehind(invocation) {
+  if (declaredInvocation()) return false;
+  const pinned = pinnedRelease(invocation);
+  const running = packageVersion();
+  if (!pinned || !running) return false;
+  return isOlderRelease(pinned, running);
+}
+
 function hookBody(invocation) {
   // `exec` is deliberately absent: we want the wrapper to survive the CLI exiting
   // non-zero and still exit 0 itself.
@@ -319,9 +370,11 @@ function rememberHookCheck(cwd, now, fields) {
  * PreMan is running here -- any command, the MCP server starting -- is taken as
  * the moment to check.
  *
- * Deliberately narrow. It only ever rewrites a hook we wrote that no longer
- * answers: an absent hook is not installed behind the user, a foreign hook is
- * not touched, and a hook that works is left on whatever version it pins.
+ * Deliberately narrow. It only ever rewrites a hook we wrote: an absent hook is
+ * not installed behind the user, and a foreign hook is not touched. A working
+ * hook is rewritten only when it pins a release older than the one running --
+ * see `hookIsBehind` -- because "it answers" and "it is the CLI we ship" stopped
+ * being the same question once a pin could outlive several releases.
  */
 export function repairDeadHook({ cwd = process.cwd(), now = Date.now(), force = false } = {}) {
   const remember = (fields) => rememberHookCheck(cwd, now, fields);
@@ -337,11 +390,14 @@ export function repairDeadHook({ cwd = process.cwd(), now = Date.now(), force =
     return remember({ action: "not-a-repository" });
   }
   if (status.state !== "installed") return remember({ action: status.state });
-  if (status.works) return remember({ action: "healthy", invocation: status.invocation });
+  const behind = hookIsBehind(status.invocation);
+  if (status.works && !behind) {
+    return remember({ action: "healthy", invocation: status.invocation });
+  }
 
   const result = installHook(makeArgs([]));
   return remember({
-    action: result.action === "updated" ? "repaired" : result.action,
+    action: result.action === "updated" ? (behind ? "upgraded" : "repaired") : result.action,
     invocation: result.invocation || "",
     replaced: status.invocation,
   });