premanmcp 1.1.11 → 1.1.13

package/bin/probe.js

@@ -0,0 +1,323 @@
+/**
+ * `preman probe` — fire a drawn population at the developer's own build.
+ *
+ * The local half of User simulation, and it exists because of topology rather
+ * than permission. The hosted swarm runs in a PreMan worker, so
+ * `http://127.0.0.1:3000` evaluated there names a port on *that worker* — the
+ * hosted path aimed at a laptop would send requests to the wrong computer, and
+ * `remote_target.py` refuses loopback outright for the same reason. No grant
+ * fixes that. The traffic has to start on the machine the software is on, which
+ * is this one.
+ *
+ * Three steps, and the middle one is the only part that happens here:
+ *
+ * 1. The backend draws the population and hands back a plan — a flat list of
+ *    calls, each carrying the visit, cohort and action it belongs to.
+ * 2. This command makes those calls against the local build and times them.
+ * 3. The backend turns what came back into the same findings artifact the
+ *    hosted worker writes, so a local run reads like a hosted one and lands in
+ *    the same Findings tab.
+ *
+ * What this command deliberately does not decide:
+ *
+ * **What to send.** Read-only filtering happens server-side, in
+ * `local_runs.plan_calls`, so a write never reaches the plan. The check below
+ * is a second one rather than the only one — belt and braces on the promise the
+ * whole subsystem rests on, and cheap enough to be worth having on the machine
+ * where the requests actually leave.
+ *
+ * **What a call was.** Only a status, a latency and an error travel back.
+ * Method, path, cohort and action are re-derived on the server from the seeded
+ * draw, so nothing here can relabel a failure onto a cohort that did not have
+ * it — which matters because the per-cohort grouping is what the report claims.
+ */
+
+import { resolveLocalTarget } from "./detect.js";
+import { formatDashboardLink } from "./link.js";
+import {
+  backendUrl,
+  callBackendJson,
+  cliInvocation,
+  describeFailure,
+  makeArgs,
+  nowMs,
+  resolveApiKey,
+} from "./shared.js";
+
+export const PROBE_HELP = `
+Probe options:
+  --visits <n>          How many simulated users to draw. Defaults to 100
+  --pack <key>          Which population to draw from. Defaults to the first
+  --seed <n>            Reuse a draw, so two runs are comparable. Defaults to 0
+  --port <port>         Add an explicit candidate port for the local target
+  --intent <text>       Why you are running this, kept verbatim on the report
+  --timeout <seconds>   Overall wall-clock budget. Defaults to 300
+  --json                Print the machine-readable result
+`;
+
+const DEFAULT_VISITS = 100;
+const DEFAULT_TIMEOUT_SECONDS = 300;
+const REQUEST_TIMEOUT_MS = 15_000;
+/** Matches the hosted executor's semaphore. A laptop is not a cluster. */
+const CONCURRENCY = 6;
+const INVENTORY_PAGE_SIZE = 200;
+const INVENTORY_MAX_ENDPOINTS = 2000;
+
+/** The only verbs that leave, checked again here. Mirrors READ_ONLY_METHODS. */
+const READ_METHODS = new Set(["GET", "HEAD", "OPTIONS"]);
+
+async function safeCall(args, method, routePath, options) {
+  try {
+    return await callBackendJson(args, method, routePath, options);
+  } catch (error) {
+    return {
+      ok: false,
+      status_code: 0,
+      detail: String(error?.message || error),
+      transport_error: true,
+    };
+  }
+}
+
+/**
+ * The saved endpoint inventory, which is what proves a candidate port is
+ * serving *this* repository rather than whatever else is listening.
+ *
+ * `resolveLocalTarget` needs it for that check, so a probe cannot be pointed
+ * at a stray dev server by accident.
+ */
+async function fetchInventory(args, token) {
+  const endpoints = [];
+  for (let offset = 0; offset < INVENTORY_MAX_ENDPOINTS; offset += INVENTORY_PAGE_SIZE) {
+    const result = await safeCall(args, "GET", "/cli/endpoints", {
+      token,
+      query: { limit: INVENTORY_PAGE_SIZE, offset },
+    });
+    if (!result.ok) {
+      return { endpoints, error: describeFailure(result, `status ${result.status_code}`) };
+    }
+    const payload = result.result ?? result.data ?? result;
+    const page = payload?.endpoints || (Array.isArray(payload) ? payload : []) || [];
+    const rows = Array.isArray(page) ? page : [];
+    endpoints.push(...rows);
+    const pagination = payload?.pagination || payload?.result?.pagination;
+    if (!pagination?.has_more || rows.length === 0) break;
+  }
+  return { endpoints };
+}
+
+function joinUrl(base, routePath) {
+  if (/^https?:\/\//i.test(routePath)) return routePath;
+  return `${String(base).replace(/\/+$/, "")}/${String(routePath).replace(/^\/+/, "")}`;
+}
+
+/** One call against the local build, as the report's three fields. */
+async function fireOne(origin, call) {
+  const method = String(call.method || "GET").toUpperCase();
+  if (!READ_METHODS.has(method)) {
+    // Should be unreachable: the server filters these out of the plan. If it
+    // ever happens, refusing locally is the behaviour that keeps the read-only
+    // promise true rather than merely intended.
+    return { index: call.index, status: null, latency_ms: null, error: "refused: not a read" };
+  }
+
+  const started = nowMs();
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+  try {
+    const response = await fetch(joinUrl(origin, call.path), {
+      method,
+      // Named distinctly from `PreMan-API-Tester`, matching SWARM_USER_AGENT.
+      // A customer excluding swarm traffic from their own analysis needs to
+      // exclude a swarm and not every PreMan caller.
+      headers: { "User-Agent": "PreMan-User-Simulation" },
+      redirect: "manual",
+      signal: controller.signal,
+    });
+    return {
+      index: call.index,
+      status: response.status,
+      latency_ms: Math.round(nowMs() - started),
+      error: "",
+    };
+  } catch (error) {
+    return {
+      index: call.index,
+      status: null,
+      latency_ms: Math.round(nowMs() - started),
+      error: String(error?.message || error).slice(0, 200),
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** A fixed-width worker pool, so a laptop is not asked to open 5,000 sockets. */
+async function runPool(items, worker, limit) {
+  const results = [];
+  let cursor = 0;
+  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    while (cursor < items.length) {
+      const index = cursor++;
+      results[index] = await worker(items[index]);
+    }
+  });
+  await Promise.all(runners);
+  return results;
+}
+
+export async function probeCommand(commandArgs = []) {
+  const args = makeArgs(commandArgs);
+  const asJson = args.has("--json");
+  const budgetMs =
+    Math.max(5, Number.parseInt(args.value("--timeout", String(DEFAULT_TIMEOUT_SECONDS)), 10) ||
+      DEFAULT_TIMEOUT_SECONDS) * 1000;
+  const deadline = Date.now() + budgetMs;
+
+  const finish = (outcome) => {
+    if (asJson) {
+      process.stdout.write(`${JSON.stringify(outcome, null, 2)}\n`);
+    } else if (outcome.message) {
+      process.stdout.write(`${outcome.message}\n`);
+    }
+    if (outcome.exitCode) process.exitCode = outcome.exitCode;
+    return outcome;
+  };
+
+  const token = resolveApiKey(args);
+  if (!token) {
+    return finish({
+      state: "skipped",
+      reason: "no_credentials",
+      message: `[preman] no API key. Run \`${cliInvocation()} login\`.`,
+      exitCode: 1,
+    });
+  }
+
+  // The inventory first, because it is what makes "is this my app" answerable.
+  // A probe aimed at a port that merely responds is a probe reporting on
+  // somebody else's dev server.
+  const inventory = await fetchInventory(args, token);
+  if (inventory.error) {
+    return finish({
+      state: "skipped",
+      reason: "inventory_unavailable",
+      message: `[preman] could not read your endpoint inventory: ${inventory.error}`,
+      exitCode: 1,
+    });
+  }
+
+  const extraPorts = [];
+  const port = Number.parseInt(args.value("--port", ""), 10);
+  if (Number.isFinite(port) && port > 0) extraPorts.push(port);
+
+  const local = await resolveLocalTarget(inventory.endpoints, { extraPorts });
+  if (!local.target) {
+    return finish({
+      state: "skipped",
+      reason: "no_local_target",
+      message:
+        `[preman] could not find your app running locally: ${local.reason}. ` +
+        `Start it, then run \`${cliInvocation()} probe --port <port>\`.`,
+      exitCode: 1,
+    });
+  }
+
+  const target = local.target;
+  const origin = String(target.url || "").replace(/\/+$/, "");
+
+  if (!asJson) {
+    process.stdout.write(
+      `[preman] target ${origin} (${target.confidence} match, ` +
+        `${target.matched}/${target.expected} routes via ${target.via})\n`
+    );
+  }
+
+  const opened = await safeCall(args, "POST", "/simulation/local-runs", {
+    token,
+    body: {
+      origin,
+      visits: Math.max(
+        1,
+        Number.parseInt(args.value("--visits", String(DEFAULT_VISITS)), 10) || DEFAULT_VISITS
+      ),
+      seed: Number.parseInt(args.value("--seed", "0"), 10) || 0,
+      intent: args.value("--intent", ""),
+      ...(args.value("--pack", "") ? { pack_key: args.value("--pack", "") } : {}),
+    },
+  });
+  if (!opened.ok) {
+    return finish({
+      state: "failed",
+      reason: "could_not_open_run",
+      message: `[preman] ${backendUrl(args)} refused the probe: ${describeFailure(
+        opened,
+        `status ${opened.status_code}`
+      )}`,
+      exitCode: 1,
+    });
+  }
+
+  const payload = opened.result ?? opened.data ?? opened;
+  const runId = String(payload.run_id || "");
+  const calls = Array.isArray(payload.calls) ? payload.calls : [];
+  const budget = payload.budget || {};
+
+  if (!asJson) {
+    process.stdout.write(
+      `[preman] probing ${origin} with ${budget.visits || 0} simulated users ` +
+        `(${calls.length} read-only requests)\n`
+    );
+  }
+
+  // Sliced to the deadline rather than abandoned at it. A run that stopped
+  // early still has findings worth writing, and the report says what was sent
+  // against what was planned -- so a partial probe reads as partial instead of
+  // as a smaller probe that went fine.
+  const results = [];
+  let stopped = "";
+  await runPool(
+    calls,
+    async (call) => {
+      if (Date.now() > deadline) {
+        stopped = stopped || "out_of_time";
+        return;
+      }
+      results.push(await fireOne(origin, call));
+    },
+    CONCURRENCY
+  );
+
+  const reported = await safeCall(args, "POST", `/simulation/local-runs/${encodeURIComponent(runId)}/report`, {
+    token,
+    body: { results, aborted_reason: stopped },
+  });
+  if (!reported.ok) {
+    return finish({
+      state: "failed",
+      reason: "could_not_report",
+      run_id: runId,
+      message: `[preman] fired ${results.length} requests but could not file the findings: ${describeFailure(
+        reported,
+        `status ${reported.status_code}`
+      )}`,
+      exitCode: 1,
+    });
+  }
+
+  const outcome = reported.result ?? reported.data ?? reported;
+  const link = formatDashboardLink({ args, label: "findings" });
+  return finish({
+    state: "completed",
+    run_id: runId,
+    origin,
+    verdict: outcome.verdict,
+    requests_sent: outcome.requests_sent,
+    requests_skipped: outcome.requests_skipped,
+    message:
+      `[preman] ${origin} ${outcome.verdict} — ${outcome.requests_sent} requests sent, ` +
+      `${outcome.requests_skipped} not attempted (writes and templated paths)` +
+      (link ? `\n${link}` : ""),
+    exitCode: 0,
+  });
+}

package/bin/runner.js

@@ -1136,6 +1136,29 @@ export async function runnerCommand(commandArgs = [], { resolveAgent = null } =
   const sub = commandArgs.find((value) => !value.startsWith("-")) || "status";
   let args = makeArgs(commandArgs);
 
+  // `runner work` is what `eval run` always was: this machine offering itself
+  // for queued work. It lives here now because that is the half of the product
+  // this file is, and because the old spelling sat next to `preman eval` --
+  // which a person types to measure their own agent -- reading as a synonym
+  // for the opposite role. Both spellings keep working; only one says what it
+  // does.
+  if (sub === "work") {
+    const { evalCommand } = await import("./eval.js");
+    return evalCommand(
+      ["run", ...commandArgs.filter((value) => value !== "work")],
+      {
+        makeArgs,
+        registerRunner,
+        readRunnerState,
+        saveRunnerState,
+        deviceId,
+        runnerLoop,
+        pairingIsLive,
+        resolveAgent,
+      }
+    );
+  }
+
   // Pairing needs an agent id, and the only place that can be asked for one is
   // here: `--background` hands the work to a detached child with no terminal, so
   // a question deferred to it is a question nobody ever sees.