premanmcp 1.1.16 → 1.1.17

package/bin/cli.js

@@ -36,6 +36,7 @@ import {
 import { HOSTED_HELP, linkCommand, runCommand, toolsCommand } from "./hosted.js";
 import { PREDICT_HELP, predictCommand } from "./predict.js";
 import { STATUS_HELP, statusCommand } from "./status.js";
+import { SWARM_HELP, swarmCommand } from "./swarm.js";
 import { HOOK_HELP, hookCommand, installHook, scheduleHookRepair } from "./hook.js";
 import { EVAL_HELP } from "./eval.js";
 import { TARGET_HELP, targetCommand } from "./target.js";
@@ -142,6 +143,7 @@ function printHelp() {
     ["", "Chat with the PreMan agent in this terminal"],
     ["chat [message] [--continue]", "The same session, named explicitly"],
     ["status", "Healthy / failing / recently fixed endpoints"],
+    ["swarm list|launch|watch|stop|bank", "Send real coding agents to see if they pick your MCP tools"],
     ["verify [options]", "Test endpoints against your local app"],
     ["probe [options]", "Fire a population of simulated users at your local app"],
     ["hook install|uninstall|status", "Manage the git pre-push hook"],
@@ -178,7 +180,7 @@ function printHelp() {
 
 Usage:
 ${usageLines}
-${AGENT_HELP}${INTEGRATIONS_HELP}${CONNECT_HELP}${DISPATCH_HELP}${STATUS_HELP}${VERIFY_HELP}${PROBE_HELP}${HOOK_HELP}${RUNNER_HELP}${EVAL_HELP}${TARGET_HELP}${ACCOUNT_HELP}${DESKTOP_HELP}${PREDICT_HELP}${ENDPOINTS_HELP}${TEST_HELP}${TESTS_HELP}
+${AGENT_HELP}${INTEGRATIONS_HELP}${CONNECT_HELP}${DISPATCH_HELP}${STATUS_HELP}${SWARM_HELP}${VERIFY_HELP}${PROBE_HELP}${HOOK_HELP}${RUNNER_HELP}${EVAL_HELP}${TARGET_HELP}${ACCOUNT_HELP}${DESKTOP_HELP}${PREDICT_HELP}${ENDPOINTS_HELP}${TEST_HELP}${TESTS_HELP}
 Login options:
   --email <email>       Pre-fill the email prompt
   --backend <url>       PreMan backend URL. Defaults to ${DEFAULT_BACKEND}
@@ -375,6 +377,8 @@ async function main() {
     await loginCommand();
   } else if (command === "status") {
     await statusCommand(commandArgs);
+  } else if (command === "swarm") {
+    await swarmCommand(commandArgs);
   } else if (command === "verify") {
     await verifyCommand(commandArgs);
   } else if (command === "probe") {

package/bin/swarm.js

@@ -0,0 +1,358 @@
+/**
+ * `preman swarm` — send real coding agents at a hosted MCP and see whether
+ * they pick its tools, from the terminal.
+ *
+ * A swarm is a harness matrix run, so every call here goes to `/matrix`; the
+ * dashboard's Swarms page reads the same runs. What this adds is the swarm's
+ * vocabulary: agents grouped by where developers run them (terminal, IDE,
+ * desktop app), each with how often it picked the tool, and the rival that
+ * won when it did not.
+ */
+
+import {
+  backendUrl,
+  callBackendJson,
+  cliInvocation,
+  describeFailure,
+  frontendUrl,
+  makeArgs,
+  resolveApiKey,
+  truncate,
+} from "./shared.js";
+
+export const SWARM_HELP = `
+Swarm options:
+  preman swarm list                       Recent swarms and what each found
+  preman swarm launch --server <slug|id> --ask "<what a developer asks>" [--ask ...]
+        [--expect tool,tool] [--agents claude_code,codex,...] [--rivals 0,5]
+        [--size 60] [--name <label>] [--dry-run]
+                                          Start a swarm (--dry-run only counts it)
+  preman swarm watch <id> [--once]        Follow a swarm until it finishes
+  preman swarm stop <id>                  Stop a swarm; unfinished agents do not run
+  preman swarm bank [--server <slug|id>]  Behaviors PreMan learned and you kept
+  --json                                  Print the raw JSON instead
+`;
+
+const ENVIRONMENTS = [
+  { id: "terminal", label: "Terminal", harnesses: ["claude_code", "codex", "cursor_cli"] },
+  { id: "ide", label: "IDE", harnesses: ["cursor_ide"] },
+  { id: "desktop", label: "Desktop app", harnesses: ["claude_desktop", "codex_desktop"] },
+];
+
+const AGENT_NAMES = {
+  claude_code: "Claude Code",
+  codex: "Codex CLI",
+  cursor_cli: "Cursor CLI",
+  cursor_ide: "Cursor",
+  claude_desktop: "Claude Desktop",
+  codex_desktop: "Codex Desktop",
+};
+
+const TERMINAL = new Set(["done", "failed", "cancelled"]);
+const POLL_MS = 5000;
+
+function environmentOf(harness) {
+  return ENVIRONMENTS.find((env) => env.harnesses.includes(harness))?.id || "other";
+}
+
+function paintFor(enabled) {
+  const wrap = (code) => (text) => (enabled ? `\u001b[${code}m${text}\u001b[0m` : String(text));
+  return { green: wrap("32"), red: wrap("31"), yellow: wrap("33"), dim: wrap("2"), bold: wrap("1") };
+}
+
+function colourEnabled(args) {
+  if (args.has("--no-color") || process.env.NO_COLOR) return false;
+  return Boolean(process.stdout.isTTY);
+}
+
+/** Every value of a repeatable flag, in order. */
+export function allValues(raw, name) {
+  const out = [];
+  for (let i = 0; i < raw.length; i += 1) {
+    if (raw[i] === name && raw[i + 1] !== undefined) out.push(raw[i + 1]);
+  }
+  return out;
+}
+
+/** Values not taken by a flag: `stop --backend <url> <id>` names the swarm, not the url. */
+export function positionals(raw) {
+  const out = [];
+  for (let i = 0; i < raw.length; i += 1) {
+    if (raw[i].startsWith("--")) {
+      if (raw[i + 1] !== undefined && !raw[i + 1].startsWith("--") && !BARE_FLAGS.has(raw[i])) i += 1;
+      continue;
+    }
+    out.push(raw[i]);
+  }
+  return out;
+}
+
+const BARE_FLAGS = new Set(["--json", "--once", "--dry-run", "--no-color"]);
+
+function listOf(value) {
+  return String(value || "")
+    .split(",")
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+/** A Wilson 95% interval, so a small swarm never reads as certain. */
+export function likelyRange(hits, n) {
+  if (!n) return null;
+  const z = 1.96;
+  const p = hits / n;
+  const d = 1 + (z * z) / n;
+  const c = p + (z * z) / (2 * n);
+  const m = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
+  return [Math.max(0, (c - m) / d), Math.min(1, (c + m) / d)];
+}
+
+const pct = (rate) => (rate === null || rate === undefined ? "—" : `${Math.round(rate * 100)}%`);
+
+/**
+ * The swarm by environment and agent, from its cells.
+ *
+ * The same rules as the dashboard: a cell with no verdict is not a miss, and
+ * two arms of one harness are two agents whose rates are never pooled.
+ */
+export function summarize(cells) {
+  const agents = new Map();
+  for (const cell of cells || []) {
+    const arm = cell.tool_visibility || "deferred";
+    const key = `${cell.harness}/${arm}`;
+    const agent = agents.get(key) || {
+      key,
+      harness: cell.harness,
+      arm,
+      environment: environmentOf(cell.harness),
+      picked: 0,
+      scored: 0,
+      running: 0,
+      queued: 0,
+      lostTo: new Map(),
+    };
+    if (cell.status === "running") agent.running += 1;
+    else if (cell.status === "queued") agent.queued += 1;
+    const unseen = cell.evidence?.tool_surfaced_by_search === false && arm === "deferred";
+    if ((cell.selected === true || cell.selected === false) && !unseen) {
+      agent.scored += 1;
+      if (cell.selected) agent.picked += 1;
+      else {
+        const expected = new Set([
+          ...(cell.evidence?.expected?.tools || []),
+          cell.evidence?.expected?.tool,
+          ...(cell.evidence?.expected?.alternates || []),
+        ].filter(Boolean));
+        for (const tool of cell.evidence?.tools_called || []) {
+          if (!expected.has(tool)) agent.lostTo.set(tool, (agent.lostTo.get(tool) || 0) + 1);
+        }
+      }
+    }
+    agents.set(key, agent);
+  }
+  const arms = new Map();
+  for (const agent of agents.values()) arms.set(agent.harness, (arms.get(agent.harness) || 0) + 1);
+  for (const agent of agents.values()) {
+    agent.name = AGENT_NAMES[agent.harness] || agent.harness;
+    if (arms.get(agent.harness) > 1) {
+      agent.name += agent.arm === "deferred" ? " (on demand)" : " (always loaded)";
+    }
+    agent.rate = agent.scored ? agent.picked / agent.scored : null;
+  }
+  const groups = [...ENVIRONMENTS, { id: "other", label: "Elsewhere" }]
+    .map((env) => ({ ...env, agents: [...agents.values()].filter((a) => a.environment === env.id) }))
+    .filter((env) => env.agents.length);
+  const picked = [...agents.values()].reduce((sum, a) => sum + a.picked, 0);
+  const scored = [...agents.values()].reduce((sum, a) => sum + a.scored, 0);
+  return { groups, picked, scored, total: (cells || []).length };
+}
+
+export function renderSwarm(run, cells, { colour = false } = {}) {
+  const paint = paintFor(colour);
+  const summary = summarize(cells);
+  const done = (cells || []).filter((cell) => cell.status === "done" || cell.status === "failed").length;
+  const lines = [];
+  lines.push(
+    `${paint.bold(run.release_label || run.id)}  ${paint.dim(run.hosted_mcp_name || run.hosted_mcp_slug || "")}  ${run.status}`
+  );
+  if (summary.scored) {
+    const range = likelyRange(summary.picked, summary.scored);
+    lines.push(
+      `${TERMINAL.has(run.status) ? "Agents" : "So far, agents"} picked your tool ${summary.picked} of ${summary.scored} times ` +
+        paint.dim(`(likely ${Math.round(range[0] * 100)}–${Math.round(range[1] * 100)}%)`)
+    );
+  } else {
+    lines.push(paint.dim(TERMINAL.has(run.status) ? "Nothing was scored." : "Agents are on their way."));
+  }
+  lines.push(paint.dim(`${done} of ${summary.total} attempts done`));
+  for (const group of summary.groups) {
+    lines.push("");
+    lines.push(paint.bold(group.label));
+    for (const agent of group.agents) {
+      const rate = agent.rate === null ? paint.dim("—") : agent.rate >= 0.75 ? paint.green(pct(agent.rate)) : agent.rate >= 0.4 ? paint.yellow(pct(agent.rate)) : paint.red(pct(agent.rate));
+      const working = agent.running ? paint.dim(` · ${agent.running} working`) : "";
+      const rival = [...agent.lostTo.entries()].sort((a, b) => b[1] - a[1])[0];
+      const lost = rival ? paint.dim(` · lost to ${rival[0]} ×${rival[1]}`) : "";
+      lines.push(`  ${truncate(agent.name, 30).padEnd(30)} ${String(rate).padStart(colour ? 13 : 4)}  ${paint.dim(`${agent.picked}/${agent.scored}`)}${working}${lost}`);
+    }
+  }
+  return `${lines.join("\n")}\n`;
+}
+
+function headlineFromRollup(run) {
+  const bucket = run?.rollup?.by_harness_visibility || {};
+  let hits = 0;
+  let n = 0;
+  for (const value of Object.values(bucket)) {
+    hits += value?.selected?.hits || 0;
+    n += value?.selected?.n || 0;
+  }
+  return n ? `picked ${hits} of ${n}` : "";
+}
+
+function requireToken(args) {
+  const token = resolveApiKey(args);
+  if (!token) {
+    throw new Error(`no PreMan API key found. Run \`${cliInvocation()} login\` first, or pass --api-key.`);
+  }
+  return token;
+}
+
+async function call(args, token, method, path, options = {}) {
+  const result = await callBackendJson(args, method, path, { token, ...options });
+  if (!result.ok) {
+    throw new Error(`${method} ${path} on ${backendUrl(args)}: ${result.status_code} ${describeFailure(result, "request failed")}`);
+  }
+  return result;
+}
+
+async function findServer(args, token, ref) {
+  if (/^[0-9a-f-]{36}$/i.test(ref)) return { id: ref, name: ref };
+  const result = await call(args, token, "GET", "/hosted-mcps");
+  const servers = result.hosted_mcps || result.list || [];
+  const match = servers.find((server) => [server.slug, server.name, server.id].includes(ref));
+  if (!match) {
+    const names = servers.map((server) => server.slug || server.name).filter(Boolean).join(", ");
+    throw new Error(`no hosted MCP called "${ref}"${names ? `. Yours: ${names}` : ""}`);
+  }
+  return match;
+}
+
+async function readSwarm(args, token, id) {
+  const run = await call(args, token, "GET", `/matrix/runs/${encodeURIComponent(id)}`);
+  const cells = await call(args, token, "GET", `/matrix/runs/${encodeURIComponent(id)}/cells`);
+  const { status_code, ok, ...runBody } = run;
+  return { run: runBody, cells: cells.cells || cells.list || [] };
+}
+
+function swarmLink(args, id) {
+  return `${frontendUrl(args)}/workbench/swarms/swarms/${encodeURIComponent(id)}`;
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export async function swarmCommand(commandArgs = []) {
+  const [sub = "list", ...rest] = commandArgs;
+  const args = makeArgs(rest);
+  const json = args.has("--json");
+  const colour = colourEnabled(args);
+  const token = requireToken(args);
+
+  if (sub === "list") {
+    const result = await call(args, token, "GET", "/matrix/runs");
+    const runs = result.runs || result.list || [];
+    if (json) return void process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
+    if (!runs.length) {
+      process.stdout.write(`No swarms yet. Start one with \`${cliInvocation()} swarm launch\`.\n`);
+      return;
+    }
+    for (const run of runs.slice(0, 20)) {
+      process.stdout.write(
+        `${String(run.id).slice(0, 8)}  ${truncate(run.release_label || "", 32).padEnd(32)}  ` +
+          `${truncate(run.hosted_mcp_name || run.hosted_mcp_slug || "", 20).padEnd(20)}  ${String(run.status).padEnd(9)}  ${headlineFromRollup(run)}\n`
+      );
+    }
+    return;
+  }
+
+  if (sub === "launch") {
+    const asks = allValues(rest, "--ask");
+    const serverRef = args.value("--server");
+    if (!serverRef || !asks.length) {
+      throw new Error(`a swarm needs --server and at least one --ask.${SWARM_HELP}`);
+    }
+    const server = await findServer(args, token, serverRef);
+    const expect = listOf(args.value("--expect"));
+    const payload = {
+      hosted_mcp_id: server.id,
+      asks: asks.map((prompt) => ({ prompt, expected_tools: expect })),
+      distractor_levels: listOf(args.value("--rivals", "0,5")).map(Number).filter((n) => Number.isFinite(n)),
+      target_size: Number(args.value("--size", "60")),
+      release_label: args.value("--name") || `cli ${new Date().toISOString().slice(0, 16).replace("T", " ")}`,
+    };
+    if (!Number.isInteger(payload.target_size) || payload.target_size < 1) {
+      throw new Error("--size must be a whole number of attempts, 1 or more.");
+    }
+    const agents = listOf(args.value("--agents"));
+    if (agents.length) payload.harnesses = agents;
+    if (args.has("--dry-run")) payload.dry_run = true;
+    const result = await call(args, token, "POST", "/matrix/runs", { json: payload });
+    const { status_code, ok, ...body } = result;
+    if (json) return void process.stdout.write(`${JSON.stringify(body, null, 2)}\n`);
+    if (payload.dry_run) {
+      process.stdout.write(`Would send ${body.cell_count ?? "?"} attempts (${body.asks_used ?? asks.length} asks, ${body.replicates ?? 1}× each).\n`);
+      return;
+    }
+    process.stdout.write(`Swarm ${body.id} launched against ${server.name || server.slug}.\n`);
+    process.stdout.write(`  Watch:  ${cliInvocation()} swarm watch ${body.id}\n  Open:   ${swarmLink(args, body.id)}\n`);
+    return;
+  }
+
+  if (sub === "watch" || sub === "show") {
+    const [id] = positionals(rest);
+    if (!id) throw new Error(`which swarm? ${cliInvocation()} swarm watch <id>`);
+    for (;;) {
+      const { run, cells } = await readSwarm(args, token, id);
+      if (json) {
+        process.stdout.write(`${JSON.stringify({ run, summary: summarize(cells) }, (k, v) => (v instanceof Map ? Object.fromEntries(v) : v), 2)}\n`);
+      } else {
+        if (process.stdout.isTTY && sub === "watch") process.stdout.write("\u001b[2J\u001b[H");
+        process.stdout.write(renderSwarm(run, cells, { colour }));
+      }
+      if (sub === "show" || args.has("--once") || TERMINAL.has(run.status)) {
+        if (!json) process.stdout.write(`\n${swarmLink(args, id)}\n`);
+        return;
+      }
+      await sleep(POLL_MS);
+    }
+  }
+
+  if (sub === "stop") {
+    const [id] = positionals(rest);
+    if (!id) throw new Error(`which swarm? ${cliInvocation()} swarm stop <id>`);
+    await call(args, token, "POST", `/matrix/runs/${encodeURIComponent(id)}/cancel`);
+    process.stdout.write(`Stopped ${id}. Agents that had not finished will not run.\n`);
+    return;
+  }
+
+  if (sub === "bank") {
+    const serverRef = args.value("--server");
+    const server = serverRef ? await findServer(args, token, serverRef) : null;
+    const result = await call(args, token, "GET", "/swarms/behaviors", {
+      query: server ? { hosted_mcp_id: server.id } : undefined,
+    });
+    const kept = (result.behaviors || []).filter((row) => row.status === "kept");
+    if (json) return void process.stdout.write(`${JSON.stringify(kept, null, 2)}\n`);
+    if (!kept.length) {
+      process.stdout.write("Nothing kept yet. After a swarm, keep the patterns PreMan finds in its misses.\n");
+      return;
+    }
+    for (const row of kept) {
+      process.stdout.write(`${row.hosted_mcp_name || ""}  ${row.title}  (${row.evidence?.count || 0} attempts)\n`);
+      for (const ask of (row.seed?.asks || []).slice(0, 3)) process.stdout.write(`    “${truncate(ask.prompt, 70)}”\n`);
+    }
+    return;
+  }
+
+  throw new Error(`unknown swarm command "${sub}".${SWARM_HELP}`);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "premanmcp",
-  "version": "1.1.16",
+  "version": "1.1.17",
   "description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
   "type": "module",
   "bin": {
@@ -16,7 +16,7 @@
     "test:proxy": "node --test scripts/smoke-proxy.mjs",
     "test:connect": "node scripts/smoke-connect.mjs",
     "test:connect-heartbeat": "node scripts/smoke-connect-heartbeat.mjs",
-    "test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-start.mjs scripts/smoke-eval-run-watch.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-eval-runtime.mjs scripts/smoke-eval-entry.mjs scripts/smoke-eval-entry-placeholder.mjs scripts/smoke-eval-discover.mjs scripts/smoke-eval-templates.mjs scripts/smoke-eval-decorated-capture.mjs scripts/smoke-eval-decorate-approval.mjs scripts/smoke-eval-approval-record.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-prompts.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs",
+    "test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-start.mjs scripts/smoke-eval-run-watch.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-eval-runtime.mjs scripts/smoke-eval-entry.mjs scripts/smoke-eval-entry-placeholder.mjs scripts/smoke-eval-discover.mjs scripts/smoke-eval-templates.mjs scripts/smoke-eval-decorated-capture.mjs scripts/smoke-eval-decorate-approval.mjs scripts/smoke-eval-approval-record.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-prompts.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs scripts/smoke-swarm.mjs",
     "test:dmg": "node --test --test-timeout=300000 scripts/smoke-install-desktop-volume.mjs"
   },
   "devDependencies": {