premanmcp 1.1.7 → 1.1.9

package/bin/eval_target.js

@@ -33,12 +33,27 @@
  */
 
 import { createServer } from "node:http";
+import { existsSync, readFileSync } from "node:fs";
+import path from "node:path";
 
 import { askAgent } from "./agent_dialects.js";
-import { REPO_CONFIG, savedAgent } from "./target.js";
+import { REPO_CONFIG, savedAgent, savedEntry } from "./target.js";
+
+/**
+ * The placeholder the dashboard puts in the copyable command.
+ *
+ * Pasted unedited it has to mean "I did not answer this", not "look for a file
+ * called FILE" -- the command is built to be run verbatim, and refusing it
+ * would make the suggested line the one line that cannot work. Matched exactly
+ * rather than by shape, because a mistyped real path is a mistake worth
+ * reporting and must not be quietly turned into a search.
+ *
+ * Kept in step with `routes/eval_sessions.py::_command`, which writes it.
+ */
+const ENTRY_PLACEHOLDER = "FILE:FUNCTION";
 import { createHash } from "node:crypto";
 
-import { backendUrl, resolveApiKey, storedKeyOrigin } from "./shared.js";
+import { resolveApiKey, storedKeyOrigin } from "./shared.js";
 
 /**
  * The credential to present to the agent under test.
@@ -81,74 +96,23 @@ function sameDeployment(a, b) {
 
 export class TargetError extends Error {}
 
-/** Ports a locally-running PreMan backend is plausibly on, in order of likelihood. */
-const LOCAL_PORTS = [8000, 8001, 8010, 8011, 8002, 8080];
-
-const PROBE_TIMEOUT_MS = 1_500;
-
-/** The route that makes a service *the agent* rather than merely a PreMan service. */
-const AGENT_ROUTE = "/workbench/conversations/{conversation_id}/messages";
-
-async function get(url, { timeout = PROBE_TIMEOUT_MS } = {}) {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeout);
-  try {
-    return await fetch(url, { signal: controller.signal });
-  } finally {
-    clearTimeout(timer);
-  }
-}
-
-/**
- * Is the agent we know how to drive answering here?
- *
- * Two questions, and asking only the first is what makes discovery pick the
- * wrong thing. `/health` says a PreMan-shaped service is on this port; it does
- * not say the service has a chat agent. A developer's machine can easily have an
- * old build, a cut-down demo API, or a differently-scoped deployment answering
- * `/health` with the right name — and pointing an eval at one produces a run
- * where every case fails with a 404 that looks like the agent misbehaving.
- *
- * So the second question is asked of the service's own OpenAPI document: does it
- * publish the route the adapter is about to POST to. That is a capability check
- * rather than a version check, which means it keeps working when the route moves
- * behind a different service and stops working the moment it is renamed — both
- * of which are the answers you want.
- */
-async function probeAgent(url) {
-  try {
-    const health = await get(`${url}/health`);
-    if (!health.ok) return { url, ok: false, why: `health answered ${health.status}` };
-    const body = await health.json().catch(() => ({}));
-    const service = String(body?.service || "");
-    if (!/preman/i.test(service)) {
-      return { url, ok: false, why: `something else is on this port (${service || "no service name"})` };
-    }
-
-    const spec = await get(`${url}/openapi.json`, { timeout: 6_000 });
-    if (!spec.ok) return { url, ok: false, why: `a PreMan service, but it publishes no API document` };
-    const document = await spec.json().catch(() => ({}));
-    if (!document?.paths?.[AGENT_ROUTE]) {
-      return { url, ok: false, why: `a PreMan service, but it has no chat agent on it` };
-    }
-    return { url, ok: true, service };
-  } catch (error) {
-    return { url, ok: false, why: error.name === "AbortError" ? "no answer" : error.message };
-  }
-}
-
 /**
  * Work out which agent this eval is about.
  *
- * The order is deliberate and it is "most explicit first". A `--target` the
- * customer typed beats anything discovered, because they have already answered
- * the question and a tool that overrode them would be unusable in the one
- * environment they configured it for. Then a backend on this machine, because
- * somebody running an eval from a project directory with a dev server up almost
- * certainly means that one. Then the configured backend, which is the honest
- * fallback and is also the answer for anyone who is not a PreMan developer.
+ * The order is "most explicit first", and every step of it names something
+ * rather than guessing at it. There used to be a step that swept localhost for
+ * a PreMan `/health`, and it is gone: it could only ever match PreMan's own
+ * backend -- a customer's agent on port 3000 was rejected as "something else is
+ * on this port" -- so in any repository but this one it was not discovery, it
+ * was a way to silently evaluate the wrong process and report the score as if
+ * it were the customer's. Evaluating PreMan's own agent is still possible, but
+ * it has to be asked for now.
+ *
+ * What replaces it is the entry point somebody approved: the file in this
+ * repository that runs its agent, called in-process. That is the thing an eval
+ * was always about, and it cannot be confused with a neighbouring service.
  */
-export async function discoverTarget(args, { log = () => {} } = {}) {
+export async function discoverTarget(args, { log = () => {}, allowSearch = true } = {}) {
   const explicit = String(args.value("--target", "") || "").trim();
   if (explicit) {
     return {
@@ -160,64 +124,138 @@ export async function discoverTarget(args, { log = () => {} } = {}) {
     };
   }
 
-  // What the repository says its agent is, saved by `preman target detect`. It
-  // ranks above discovery for the same reason `--target` does: somebody already
-  // answered this question, and probing past their answer would measure a
-  // different process than the one they pointed at. Below `--target` because a
-  // flag is about this invocation and the file is about the project.
-  const saved = savedAgent();
-  if (saved) {
+  // PreMan's own chat agent, which used to be found by sweeping and is now
+  // named. Not probed, for the same reason `--target` is not: an address the
+  // caller typed is an answer, and checking it would only turn a clear failure
+  // at the first turn into a different message here.
+  const premanAgent = String(args.value("--preman-agent", "") || "").trim();
+  if (premanAgent) {
     return {
-      kind: "external",
-      agent: saved,
-      url: `${saved.url}${saved.path}`,
-      label: `your agent (${saved.url}${saved.path})`,
-      how: `${REPO_CONFIG} names it, and it speaks ${saved.dialect}`,
+      kind: "preman-chat",
+      baseUrl: premanAgent,
+      // Held here and never sent to PreMan's control plane. The adapter is the
+      // only thing that presents it, which is why the adapter runs locally.
+      token: agentToken(args, premanAgent, log),
+      label: `PreMan agent (${premanAgent})`,
+      how: "you named it with --preman-agent",
       probed: [],
     };
   }
 
-  const probed = [];
-  for (const port of LOCAL_PORTS) {
-    const result = await probeAgent(`http://127.0.0.1:${port}`);
-    probed.push(result);
-    if (result.ok) {
-      log(`Found a PreMan backend on port ${port}.`);
-      return {
-        kind: "preman-chat",
-        baseUrl: `http://127.0.0.1:${port}`,
-        // The agent's own credential, held here and never sent to PreMan's
-        // control plane. The adapter is the only thing that ever presents it,
-        // which is the whole reason the adapter runs on this machine.
-        token: agentToken(args, `http://127.0.0.1:${port}`, log),
-        label: `PreMan agent (local, port ${port})`,
-        how: `probed localhost and got a PreMan /health on port ${port}`,
-        probed,
-      };
-    }
+  const projectPath = String(args.value("--path", "") || process.cwd());
+
+  // An entry point named for this invocation. Still generates and still asks:
+  // what `--entry` skips is the search, not the review.
+  const named = String(args.value("--entry", process.env.PREMAN_EVAL_ENTRY || "") || "").trim();
+  if (named && named !== ENTRY_PLACEHOLDER) {
+    const { entryFromNamedSymbol } = await import("./eval_entry.js");
+    return entryFromNamedSymbol(named, { projectPath, args, log });
+  }
+  if (named === ENTRY_PLACEHOLDER) {
+    log(
+      `--entry still has ${ENTRY_PLACEHOLDER} in it, so PreMan will look for the entry point itself.`
+    );
+  }
+
+  const saved = savedEntry(projectPath);
+  if (saved) {
+    const verified = verifyEntry(saved);
+    if (verified) return verified;
+    log(
+      `The approved entry point ${saved.file} has changed since it was approved, ` +
+        `so it is being re-checked rather than trusted.`
+    );
   }
 
-  const configured = backendUrl(args);
-  const remote = await probeAgent(configured);
-  probed.push(remote);
-  if (remote.ok) {
-    log(`No local backend; using the one this CLI is configured against.`);
+  // What the repository says its agent is over HTTP, saved by
+  // `preman target detect`. Below the code entry point because a repository
+  // carrying both has deliberately run the search, and calling the agent is a
+  // more faithful measurement than calling a server that happens to be up.
+  const savedHttp = savedAgent(projectPath);
+  if (savedHttp) {
     return {
-      kind: "preman-chat",
-      baseUrl: configured,
-      token: agentToken(args, configured, log),
-      label: `PreMan agent (${configured})`,
-      how: `nothing local answered, so this is the backend the CLI is configured against`,
-      probed,
+      kind: "external",
+      agent: savedHttp,
+      url: `${savedHttp.url}${savedHttp.path}`,
+      label: `your agent (${savedHttp.url}${savedHttp.path})`,
+      how: `${REPO_CONFIG} names it, and it speaks ${savedHttp.dialect}`,
+      probed: [],
     };
   }
 
-  throw new TargetError(
-    `Could not find an agent to evaluate. Tried ${probed.length} address(es): ` +
-      probed.map((p) => `${p.url} (${p.why})`).join(", ") +
-      `. Point at one directly with --target <url>, where that URL accepts ` +
-      `POST {"message","history"} and answers {"response"}.`
-  );
+  // Writing a file into somebody's repository and running model-written code
+  // against their agent is not a thing to do because a job frame arrived. A
+  // daemon, a CI job and `--headless` all land here and stop.
+  if (!allowSearch || !process.stdout.isTTY || args.has("--headless") || args.has("--no-wait")) {
+    throw new TargetError(
+      `No approved entry point for this project, and nothing can be set up without ` +
+        `somebody to review it. Run \`preman eval target\` here once, approve the entry ` +
+        `point it finds, and this will work unattended afterwards. Or name one directly ` +
+        `with --entry <file>:<symbol>, or point at a running agent with --target <url>.`
+    );
+  }
+
+  const { searchForEntry } = await import("./eval_entry.js");
+  return searchForEntry({ projectPath, args, log });
+}
+
+/**
+ * The approved shim, if it is still the file that was approved.
+ *
+ * The digest is the whole point of recording one. An approval is of a specific
+ * text, and a shim edited afterwards -- by a person, a merge, or a tool -- has
+ * not been reviewed. Returning null sends the caller back through the gate
+ * rather than failing, because a hand-edited shim is usually an improvement
+ * somebody made on purpose and re-approving it takes one keystroke.
+ */
+export function verifyEntry(saved) {
+  const file = path.isAbsolute(saved.file) ? saved.file : path.join(saved.root, saved.file);
+  if (!existsSync(file)) return null;
+  const source = readFileSync(file, "utf8");
+  const digest = createHash("sha256").update(source).digest("hex");
+  if (saved.sha256 && digest !== saved.sha256) return null;
+
+  // Every file the approval produced, not only the wrapper.
+  //
+  // A `node_bridge` target is two files: the Python wrapper and the `.mjs` it
+  // spawns. Checking one of them means the other can be edited after approval
+  // and still run, while the digest that was supposed to notice says the target
+  // is unchanged. An older record has no `files` and falls back to the single
+  // hash above, so an approval made before this check keeps working.
+  const extra = Array.isArray(saved.files) ? saved.files : [];
+  for (const item of extra) {
+    const rel = String(item?.path || "");
+    const want = String(item?.sha256 || "");
+    if (!rel || !want) return null;
+    const full = path.isAbsolute(rel) ? rel : path.join(saved.root, rel);
+    if (!existsSync(full)) return null;
+    const got = createHash("sha256").update(readFileSync(full, "utf8")).digest("hex");
+    if (got !== want) return null;
+  }
+
+  const sysPath = saved.sysPath
+    ? path.isAbsolute(saved.sysPath)
+      ? saved.sysPath
+      : path.join(saved.root, saved.sysPath)
+    : path.dirname(file);
+
+  return {
+    kind: "callable",
+    ref: saved.ref,
+    file,
+    sysPath,
+    sha256: digest,
+    files: extra,
+    shape: saved.shape || "",
+    language: saved.language,
+    framework: saved.framework,
+    entryFile: saved.entryFile,
+    label: `your agent (${saved.entryFile || saved.ref})`,
+    how:
+      `${REPO_CONFIG} names ${saved.file}, you approved it` +
+      `${saved.approvedAt ? ` on ${saved.approvedAt}` : ""}, and it is unchanged since`,
+    probed: [],
+  };
 }
 
 // ── What the agent said it was doing ────────────────────────────────────
@@ -582,6 +620,15 @@ export async function startAdapter(target, args, { log = () => {}, onStep = null
     return { url: target.url, close: async () => {}, translated: false };
   }
 
+  if (target.kind === "callable") {
+    // There is no socket. The harness imports the entry point and calls it in
+    // the same process, so there is nothing to listen on and nothing to
+    // translate -- the shim already returns what an adapter would have sent.
+    // Per-turn narration still happens, but it comes from the shim's own log
+    // rather than from requests arriving here; see `targetStats`.
+    return { url: "", close: async () => {}, translated: false };
+  }
+
   const state = {
     conversations: new Map(),
     turns: 0,