premanmcp 1.1.16 → 1.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/bin/agent.js +150 -0
- package/bin/cli.js +5 -1
- package/bin/swarm.js +373 -0
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -53,7 +53,7 @@ preman chat -p "list my endpoints" # one turn, printed, then exit
|
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
Inside a session: `/help`, `/new`, `/conversations`, `/resume <id>`, `/app`,
|
|
56
|
-
`/eval`, `/clear`, `/exit`. `/eval list` shows what your agent can be measured
|
|
56
|
+
`/swarm`, `/eval`, `/clear`, `/exit`. `/eval list` shows what your agent can be measured
|
|
57
57
|
on, numbered; `/eval on 3 12` or `/eval on sycophancy` switches those on (a
|
|
58
58
|
number, a `3-7` range, `all`, an id, or enough of one to be unambiguous), `/eval
|
|
59
59
|
off` the same in reverse, and `/eval` runs what is on. `/eval defaults` reads
|
|
@@ -65,6 +65,30 @@ link underneath opens the PreMan desktop app when it is installed (a build older
|
|
|
65
65
|
than 0.4.0 comes forward on whatever it was showing rather than on the run), and
|
|
66
66
|
the run's dashboard URL is always printed with it.
|
|
67
67
|
|
|
68
|
+
Ask the agent to launch a hosted-MCP swarm in plain language, just as in the
|
|
69
|
+
app: “Launch a 12-attempt Codex swarm against my tracker server; ask it to find
|
|
70
|
+
open issues and expect search_issues.” The terminal uses the same conversation
|
|
71
|
+
service and approval rules. A proposed launch shows its summary in the terminal;
|
|
72
|
+
choose “Approve exact action” to launch it, “Decline” to reject it, or Enter to
|
|
73
|
+
leave it unanswered. A swarm result prints its label, status, run ID, and link.
|
|
74
|
+
Piped sessions leave approvals unanswered for review in the app.
|
|
75
|
+
|
|
76
|
+
For explicit commands inside the session, `/swarm` uses the current session's
|
|
77
|
+
account, backend, and workspace:
|
|
78
|
+
|
|
79
|
+
```text
|
|
80
|
+
/swarm list
|
|
81
|
+
/swarm launch --server tracker --ask "Find open issues" --expect search_issues --agents codex --size 12 --rivals 0
|
|
82
|
+
/swarm watch <run-id>
|
|
83
|
+
/swarm stop <run-id>
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
`/swarm help` lists every option. Add `--dry-run` to preview a launch without
|
|
87
|
+
starting attempts. Ctrl+C while watching returns to the conversation and leaves
|
|
88
|
+
the swarm running; `/swarm stop <run-id>` cancels it explicitly. The same commands
|
|
89
|
+
work outside a session as `preman swarm list|launch|watch|stop|bank`. These run
|
|
90
|
+
real agents against hosted MCPs; the app's sample swarms remain a separate demo.
|
|
91
|
+
|
|
68
92
|
A session takes a screen of its own, the way `vim` does: nothing above it is
|
|
69
93
|
scrollable, and leaving hands your shell back exactly as it was with one line
|
|
70
94
|
saying how to resume. Ctrl+C stops the current turn (PreMan keeps working on it)
|
package/bin/agent.js
CHANGED
|
@@ -25,6 +25,7 @@ import { basename, join } from "node:path";
|
|
|
25
25
|
import { createInterface } from "node:readline/promises";
|
|
26
26
|
|
|
27
27
|
import { createSseParser } from "./runner.js";
|
|
28
|
+
import { SWARM_HELP, swarmCommand } from "./swarm.js";
|
|
28
29
|
import {
|
|
29
30
|
authenticateTerminal,
|
|
30
31
|
backendUrl,
|
|
@@ -59,6 +60,10 @@ const SLASH_HELP = `Commands:
|
|
|
59
60
|
/eval on|off <what> Switch behaviours on or off, by number or by id
|
|
60
61
|
/eval defaults Pick a few that suit this project, and switch them on
|
|
61
62
|
/eval doctor Check this machine can run evals before starting one
|
|
63
|
+
/swarm list List your hosted-MCP swarms
|
|
64
|
+
/swarm launch ... Launch with the same flags as preman swarm launch
|
|
65
|
+
/swarm watch <id> Follow a swarm; Ctrl+C returns to the conversation
|
|
66
|
+
/swarm help Show launch, watch, stop and behavior-bank options
|
|
62
67
|
/app Open this conversation in the PreMan app
|
|
63
68
|
/clear Clear the screen
|
|
64
69
|
/exit Leave (Ctrl+D does the same)
|
|
@@ -212,6 +217,13 @@ export function artifactLine(artifact) {
|
|
|
212
217
|
if (!artifact || typeof artifact !== "object") return "";
|
|
213
218
|
const type = String(artifact.type || "");
|
|
214
219
|
if (!type || SILENT_ARTIFACTS.has(type)) return "";
|
|
220
|
+
if (type === "swarm_run" && artifact.run?.id) {
|
|
221
|
+
const run = artifact.run;
|
|
222
|
+
const name = truncate(run.release_label || artifact.title || "Swarm", 60);
|
|
223
|
+
const line = [name, run.status, run.id].filter(Boolean).join(" · ");
|
|
224
|
+
const url = artifactUrl(artifact);
|
|
225
|
+
return url ? `${line} — ${url}` : line;
|
|
226
|
+
}
|
|
215
227
|
const label =
|
|
216
228
|
String(artifact.title || "").trim() || ARTIFACT_LABELS[type] || type.replace(/_/g, " ");
|
|
217
229
|
const url = artifactUrl(artifact);
|
|
@@ -757,10 +769,82 @@ async function runTurn({
|
|
|
757
769
|
ask,
|
|
758
770
|
say,
|
|
759
771
|
});
|
|
772
|
+
await askActionProposals({
|
|
773
|
+
args, token, workspaceId, artifacts: done.turn.artifacts, paint, ask, say, register,
|
|
774
|
+
});
|
|
760
775
|
}
|
|
761
776
|
return done;
|
|
762
777
|
}
|
|
763
778
|
|
|
779
|
+
/** Approve only a stored action on this session's backend; no tool-supplied URL is executed. */
|
|
780
|
+
async function askActionProposals({ args, token, workspaceId, artifacts, paint, ask, say, register }) {
|
|
781
|
+
if (!process.stdin.isTTY) return;
|
|
782
|
+
const seen = new Set();
|
|
783
|
+
const proposals = (Array.isArray(artifacts) ? artifacts : []).filter((item) => {
|
|
784
|
+
const proposal = item?.type === "action_proposal" ? item.proposal : null;
|
|
785
|
+
const id = proposal?.id;
|
|
786
|
+
if (typeof id !== "string" || !/^[\w-]{1,128}$/.test(id) || seen.has(id)) return false;
|
|
787
|
+
if ((proposal.status || "pending") !== "pending") return false;
|
|
788
|
+
if (proposal.expires_at && Date.parse(proposal.expires_at) <= Date.now()) return false;
|
|
789
|
+
seen.add(id);
|
|
790
|
+
return true;
|
|
791
|
+
});
|
|
792
|
+
if (!proposals.length) return;
|
|
793
|
+
const write = say || ((text) => process.stdout.write(text));
|
|
794
|
+
const borrowed = typeof ask === "function";
|
|
795
|
+
const rl = borrowed ? null : createInterface({ input: process.stdin, output: process.stdout });
|
|
796
|
+
const readAnswer = borrowed ? ask : (text) => rl.question(text);
|
|
797
|
+
try {
|
|
798
|
+
for (const artifact of proposals.slice(0, 10)) {
|
|
799
|
+
const proposal = artifact.proposal;
|
|
800
|
+
const title = truncate(String(proposal.title || artifact.title || "Confirm action"), 240);
|
|
801
|
+
write(`\n${paint.yellow("PreMan needs a decision")}\n ${title}\n`);
|
|
802
|
+
const summary = String(proposal.summary || artifact.summary || "").trim();
|
|
803
|
+
if (summary) write(` ${truncate(summary, 1000)}\n`);
|
|
804
|
+
write(` 1. Approve exact action\n 2. Decline\n${paint.dim(" Enter to skip\n")}`);
|
|
805
|
+
const answer = String((await readAnswer(" > ")) || "").trim().toLowerCase();
|
|
806
|
+
const action = ["1", "y", "yes", "approve"].includes(answer) ? "approve"
|
|
807
|
+
: ["2", "n", "no", "decline"].includes(answer) ? "decline" : null;
|
|
808
|
+
if (!action) {
|
|
809
|
+
write(paint.dim(" Left unanswered.\n"));
|
|
810
|
+
continue;
|
|
811
|
+
}
|
|
812
|
+
if (proposal.expires_at && Date.parse(proposal.expires_at) <= Date.now()) {
|
|
813
|
+
write(paint.dim(" This action expired. Ask PreMan to regenerate it.\n"));
|
|
814
|
+
continue;
|
|
815
|
+
}
|
|
816
|
+
const controller = new AbortController();
|
|
817
|
+
register?.({ abort: () => controller.abort() });
|
|
818
|
+
try {
|
|
819
|
+
const response = await backendCall(args, "POST",
|
|
820
|
+
`/workbench/action-proposals/${encodeURIComponent(proposal.id)}/${action}`, {
|
|
821
|
+
token, signal: controller.signal,
|
|
822
|
+
headers: workspaceId ? { "x-workspace-id": workspaceId } : undefined,
|
|
823
|
+
});
|
|
824
|
+
const result = response.result;
|
|
825
|
+
const status = response.proposal?.status || "unknown";
|
|
826
|
+
if (result?.ok === false || result?.error) {
|
|
827
|
+
write(` ${paint.red(describeFailure(result, "The action did not complete."))}\n`);
|
|
828
|
+
} else if (status === "completed" || status === "declined") {
|
|
829
|
+
write(` ${paint.green("✓")} ${status === "completed" ? "Completed" : "Declined"}: ${title}\n`);
|
|
830
|
+
if (result?.run?.id) write(` ${artifactLine({ type: "swarm_run", title, run: result.run, url: result.url })}\n`);
|
|
831
|
+
} else {
|
|
832
|
+
write(` Action ${String(status).replaceAll("_", " ")}. Use /app to check its result.\n`);
|
|
833
|
+
}
|
|
834
|
+
} catch (error) {
|
|
835
|
+
write(controller.signal.aborted
|
|
836
|
+
? " Stopped waiting for the action result. Check /app before trying again.\n"
|
|
837
|
+
: ` ${paint.red(error?.message || String(error))}\n`);
|
|
838
|
+
} finally {
|
|
839
|
+
register?.(null);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
if (proposals.length > 10) write(" More actions are awaiting a decision in /app.\n");
|
|
843
|
+
} finally {
|
|
844
|
+
if (rl) rl.close();
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
764
848
|
/**
|
|
765
849
|
* Put a permission confirmation to the person at the keyboard.
|
|
766
850
|
*
|
|
@@ -1956,6 +2040,64 @@ export async function runEval(rest, { args, token, dock, rl, paint, say, running
|
|
|
1956
2040
|
}
|
|
1957
2041
|
}
|
|
1958
2042
|
|
|
2043
|
+
/** Quoted flag values for a session command, without shell evaluation. */
|
|
2044
|
+
export function swarmArguments(rest) {
|
|
2045
|
+
const words = [];
|
|
2046
|
+
let word = "";
|
|
2047
|
+
let quote = "";
|
|
2048
|
+
let escaped = false;
|
|
2049
|
+
let active = false;
|
|
2050
|
+
for (const character of String(rest || "")) {
|
|
2051
|
+
if (escaped) {
|
|
2052
|
+
word += character;
|
|
2053
|
+
escaped = false;
|
|
2054
|
+
} else if (character === "\\" && quote !== "'") {
|
|
2055
|
+
escaped = true;
|
|
2056
|
+
active = true;
|
|
2057
|
+
} else if (quote) {
|
|
2058
|
+
if (character === quote) quote = "";
|
|
2059
|
+
else word += character;
|
|
2060
|
+
} else if (character === "'" || character === '"') {
|
|
2061
|
+
quote = character;
|
|
2062
|
+
active = true;
|
|
2063
|
+
} else if (/\s/.test(character)) {
|
|
2064
|
+
if (active) words.push(word);
|
|
2065
|
+
word = "";
|
|
2066
|
+
active = false;
|
|
2067
|
+
} else {
|
|
2068
|
+
word += character;
|
|
2069
|
+
active = true;
|
|
2070
|
+
}
|
|
2071
|
+
}
|
|
2072
|
+
if (quote || escaped) throw new Error("Finish the quoted value or escaped character in /swarm.");
|
|
2073
|
+
if (active) words.push(word);
|
|
2074
|
+
return words;
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
/** Reuse the CLI command inside the current authenticated conversation. */
|
|
2078
|
+
export async function runSwarm(rest, { args, token, workspaceId, say, paint, register }) {
|
|
2079
|
+
const controller = new AbortController();
|
|
2080
|
+
register?.({ abort: () => controller.abort() });
|
|
2081
|
+
try {
|
|
2082
|
+
const argv = swarmArguments(rest);
|
|
2083
|
+
if (["help", "--help", "-h"].includes(argv[0])) {
|
|
2084
|
+
say(`${SWARM_HELP}\n`);
|
|
2085
|
+
return;
|
|
2086
|
+
}
|
|
2087
|
+
await swarmCommand(argv, {
|
|
2088
|
+
sessionArgs: args, token, workspaceId, signal: controller.signal,
|
|
2089
|
+
write: say, clearWatch: false,
|
|
2090
|
+
});
|
|
2091
|
+
say("\n");
|
|
2092
|
+
} catch (error) {
|
|
2093
|
+
say(controller.signal.aborted
|
|
2094
|
+
? "Stopped waiting. The swarm may still be running; /swarm list to check.\n\n"
|
|
2095
|
+
: `${paint.red(error?.message || String(error))}\n\n`);
|
|
2096
|
+
} finally {
|
|
2097
|
+
register?.(null);
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
|
|
1959
2101
|
export async function agentCommand(commandArgs = [], { authenticate = authenticateTerminal } = {}) {
|
|
1960
2102
|
const args = makeArgs(commandArgs);
|
|
1961
2103
|
if (args.has("--help") || args.has("-h")) {
|
|
@@ -2067,6 +2209,7 @@ export async function agentCommand(commandArgs = [], { authenticate = authentica
|
|
|
2067
2209
|
// Only reaches anything when a run is open, and the import is already
|
|
2068
2210
|
// resolved by then -- the module was loaded to start the run.
|
|
2069
2211
|
if (evaluating) void import("./eval.js").then(({ cancelEvalRuns }) => cancelEvalRuns());
|
|
2212
|
+
turn?.abort();
|
|
2070
2213
|
});
|
|
2071
2214
|
hold.onTeardown(() => process.stdout.off("resize", onResize));
|
|
2072
2215
|
hold.onTeardown(() => dock.disable());
|
|
@@ -2286,6 +2429,13 @@ export async function agentCommand(commandArgs = [], { authenticate = authentica
|
|
|
2286
2429
|
});
|
|
2287
2430
|
continue;
|
|
2288
2431
|
}
|
|
2432
|
+
if (slash.name === "swarm") {
|
|
2433
|
+
await runSwarm(slash.rest, {
|
|
2434
|
+
args, token, workspaceId: workspace.id, say, paint,
|
|
2435
|
+
register: (handle) => { turn = handle; },
|
|
2436
|
+
});
|
|
2437
|
+
continue;
|
|
2438
|
+
}
|
|
2289
2439
|
if (slash.name === "app") {
|
|
2290
2440
|
const url = `${frontendUrl(args)}/workbench/${conversation.id}`;
|
|
2291
2441
|
openUrl(url);
|
package/bin/cli.js
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
import { HOSTED_HELP, linkCommand, runCommand, toolsCommand } from "./hosted.js";
|
|
37
37
|
import { PREDICT_HELP, predictCommand } from "./predict.js";
|
|
38
38
|
import { STATUS_HELP, statusCommand } from "./status.js";
|
|
39
|
+
import { SWARM_HELP, swarmCommand } from "./swarm.js";
|
|
39
40
|
import { HOOK_HELP, hookCommand, installHook, scheduleHookRepair } from "./hook.js";
|
|
40
41
|
import { EVAL_HELP } from "./eval.js";
|
|
41
42
|
import { TARGET_HELP, targetCommand } from "./target.js";
|
|
@@ -142,6 +143,7 @@ function printHelp() {
|
|
|
142
143
|
["", "Chat with the PreMan agent in this terminal"],
|
|
143
144
|
["chat [message] [--continue]", "The same session, named explicitly"],
|
|
144
145
|
["status", "Healthy / failing / recently fixed endpoints"],
|
|
146
|
+
["swarm list|launch|watch|stop|bank", "Send real coding agents to see if they pick your MCP tools"],
|
|
145
147
|
["verify [options]", "Test endpoints against your local app"],
|
|
146
148
|
["probe [options]", "Fire a population of simulated users at your local app"],
|
|
147
149
|
["hook install|uninstall|status", "Manage the git pre-push hook"],
|
|
@@ -178,7 +180,7 @@ function printHelp() {
|
|
|
178
180
|
|
|
179
181
|
Usage:
|
|
180
182
|
${usageLines}
|
|
181
|
-
${AGENT_HELP}${INTEGRATIONS_HELP}${CONNECT_HELP}${DISPATCH_HELP}${STATUS_HELP}${VERIFY_HELP}${PROBE_HELP}${HOOK_HELP}${RUNNER_HELP}${EVAL_HELP}${TARGET_HELP}${ACCOUNT_HELP}${DESKTOP_HELP}${PREDICT_HELP}${ENDPOINTS_HELP}${TEST_HELP}${TESTS_HELP}
|
|
183
|
+
${AGENT_HELP}${INTEGRATIONS_HELP}${CONNECT_HELP}${DISPATCH_HELP}${STATUS_HELP}${SWARM_HELP}${VERIFY_HELP}${PROBE_HELP}${HOOK_HELP}${RUNNER_HELP}${EVAL_HELP}${TARGET_HELP}${ACCOUNT_HELP}${DESKTOP_HELP}${PREDICT_HELP}${ENDPOINTS_HELP}${TEST_HELP}${TESTS_HELP}
|
|
182
184
|
Login options:
|
|
183
185
|
--email <email> Pre-fill the email prompt
|
|
184
186
|
--backend <url> PreMan backend URL. Defaults to ${DEFAULT_BACKEND}
|
|
@@ -375,6 +377,8 @@ async function main() {
|
|
|
375
377
|
await loginCommand();
|
|
376
378
|
} else if (command === "status") {
|
|
377
379
|
await statusCommand(commandArgs);
|
|
380
|
+
} else if (command === "swarm") {
|
|
381
|
+
await swarmCommand(commandArgs);
|
|
378
382
|
} else if (command === "verify") {
|
|
379
383
|
await verifyCommand(commandArgs);
|
|
380
384
|
} else if (command === "probe") {
|
package/bin/swarm.js
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `preman swarm` — send real coding agents at a hosted MCP and see whether
|
|
3
|
+
* they pick its tools, from the terminal.
|
|
4
|
+
*
|
|
5
|
+
* A swarm is a harness matrix run, so every call here goes to `/matrix`; the
|
|
6
|
+
* dashboard's Swarms page reads the same runs. What this adds is the swarm's
|
|
7
|
+
* vocabulary: agents grouped by where developers run them (terminal, IDE,
|
|
8
|
+
* desktop app), each with how often it picked the tool, and the rival that
|
|
9
|
+
* won when it did not.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { setTimeout as wait } from "node:timers/promises";
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
backendUrl,
|
|
16
|
+
callBackendJson,
|
|
17
|
+
cliInvocation,
|
|
18
|
+
describeFailure,
|
|
19
|
+
frontendUrl,
|
|
20
|
+
makeArgs,
|
|
21
|
+
resolveApiKey,
|
|
22
|
+
truncate,
|
|
23
|
+
} from "./shared.js";
|
|
24
|
+
|
|
25
|
+
export const SWARM_HELP = `
|
|
26
|
+
Swarm options:
|
|
27
|
+
preman swarm list Recent swarms and what each found
|
|
28
|
+
preman swarm launch --server <slug|id> --ask "<what a developer asks>" [--ask ...]
|
|
29
|
+
[--expect tool,tool] [--agents claude_code,codex,...] [--rivals 0,5]
|
|
30
|
+
[--size 60] [--name <label>] [--dry-run]
|
|
31
|
+
Start a swarm (--dry-run only counts it)
|
|
32
|
+
preman swarm watch <id> [--once] Follow a swarm until it finishes
|
|
33
|
+
preman swarm stop <id> Stop a swarm; unfinished agents do not run
|
|
34
|
+
preman swarm bank [--server <slug|id>] Behaviors PreMan learned and you kept
|
|
35
|
+
--json Print the raw JSON instead
|
|
36
|
+
`;
|
|
37
|
+
|
|
38
|
+
const ENVIRONMENTS = [
|
|
39
|
+
{ id: "terminal", label: "Terminal", harnesses: ["claude_code", "codex", "cursor_cli"] },
|
|
40
|
+
{ id: "ide", label: "IDE", harnesses: ["cursor_ide"] },
|
|
41
|
+
{ id: "desktop", label: "Desktop app", harnesses: ["claude_desktop", "codex_desktop"] },
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
const AGENT_NAMES = {
|
|
45
|
+
claude_code: "Claude Code",
|
|
46
|
+
codex: "Codex CLI",
|
|
47
|
+
cursor_cli: "Cursor CLI",
|
|
48
|
+
cursor_ide: "Cursor",
|
|
49
|
+
claude_desktop: "Claude Desktop",
|
|
50
|
+
codex_desktop: "Codex Desktop",
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const TERMINAL = new Set(["done", "failed", "cancelled"]);
|
|
54
|
+
const POLL_MS = 5000;
|
|
55
|
+
|
|
56
|
+
function environmentOf(harness) {
|
|
57
|
+
return ENVIRONMENTS.find((env) => env.harnesses.includes(harness))?.id || "other";
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function paintFor(enabled) {
|
|
61
|
+
const wrap = (code) => (text) => (enabled ? `\u001b[${code}m${text}\u001b[0m` : String(text));
|
|
62
|
+
return { green: wrap("32"), red: wrap("31"), yellow: wrap("33"), dim: wrap("2"), bold: wrap("1") };
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function colourEnabled(args) {
|
|
66
|
+
if (args.has("--no-color") || process.env.NO_COLOR) return false;
|
|
67
|
+
return Boolean(process.stdout.isTTY);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Every value of a repeatable flag, in order. */
|
|
71
|
+
export function allValues(raw, name) {
|
|
72
|
+
const out = [];
|
|
73
|
+
for (let i = 0; i < raw.length; i += 1) {
|
|
74
|
+
if (raw[i] === name && raw[i + 1] !== undefined) out.push(raw[i + 1]);
|
|
75
|
+
}
|
|
76
|
+
return out;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Values not taken by a flag: `stop --backend <url> <id>` names the swarm, not the url. */
|
|
80
|
+
export function positionals(raw) {
|
|
81
|
+
const out = [];
|
|
82
|
+
for (let i = 0; i < raw.length; i += 1) {
|
|
83
|
+
if (raw[i].startsWith("--")) {
|
|
84
|
+
if (raw[i + 1] !== undefined && !raw[i + 1].startsWith("--") && !BARE_FLAGS.has(raw[i])) i += 1;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
out.push(raw[i]);
|
|
88
|
+
}
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const BARE_FLAGS = new Set(["--json", "--once", "--dry-run", "--no-color"]);
|
|
93
|
+
|
|
94
|
+
function listOf(value) {
|
|
95
|
+
return String(value || "")
|
|
96
|
+
.split(",")
|
|
97
|
+
.map((item) => item.trim())
|
|
98
|
+
.filter(Boolean);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** A Wilson 95% interval, so a small swarm never reads as certain. */
|
|
102
|
+
export function likelyRange(hits, n) {
|
|
103
|
+
if (!n) return null;
|
|
104
|
+
const z = 1.96;
|
|
105
|
+
const p = hits / n;
|
|
106
|
+
const d = 1 + (z * z) / n;
|
|
107
|
+
const c = p + (z * z) / (2 * n);
|
|
108
|
+
const m = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
|
|
109
|
+
return [Math.max(0, (c - m) / d), Math.min(1, (c + m) / d)];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const pct = (rate) => (rate === null || rate === undefined ? "—" : `${Math.round(rate * 100)}%`);
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The swarm by environment and agent, from its cells.
|
|
116
|
+
*
|
|
117
|
+
* The same rules as the dashboard: a cell with no verdict is not a miss, and
|
|
118
|
+
* two arms of one harness are two agents whose rates are never pooled.
|
|
119
|
+
*/
|
|
120
|
+
export function summarize(cells) {
|
|
121
|
+
const agents = new Map();
|
|
122
|
+
for (const cell of cells || []) {
|
|
123
|
+
const arm = cell.tool_visibility || "deferred";
|
|
124
|
+
const key = `${cell.harness}/${arm}`;
|
|
125
|
+
const agent = agents.get(key) || {
|
|
126
|
+
key,
|
|
127
|
+
harness: cell.harness,
|
|
128
|
+
arm,
|
|
129
|
+
environment: environmentOf(cell.harness),
|
|
130
|
+
picked: 0,
|
|
131
|
+
scored: 0,
|
|
132
|
+
running: 0,
|
|
133
|
+
queued: 0,
|
|
134
|
+
lostTo: new Map(),
|
|
135
|
+
};
|
|
136
|
+
if (cell.status === "running") agent.running += 1;
|
|
137
|
+
else if (cell.status === "queued") agent.queued += 1;
|
|
138
|
+
const expected = new Set([
|
|
139
|
+
...(cell.evidence?.expected?.tools || []),
|
|
140
|
+
cell.evidence?.expected?.tool,
|
|
141
|
+
...(cell.evidence?.expected?.alternates || []),
|
|
142
|
+
].filter(Boolean));
|
|
143
|
+
const called = cell.evidence?.tools_called || [];
|
|
144
|
+
// Missing search telemetry cannot erase a recorded selection or call to
|
|
145
|
+
// the expected tool. Codex can call it without emitting a tool search.
|
|
146
|
+
const sawExpected = cell.selected === true || called.some((tool) => expected.has(tool));
|
|
147
|
+
const unseen = !sawExpected && cell.evidence?.tool_surfaced_by_search === false && arm === "deferred";
|
|
148
|
+
if ((cell.selected === true || cell.selected === false) && !unseen) {
|
|
149
|
+
agent.scored += 1;
|
|
150
|
+
if (cell.selected) agent.picked += 1;
|
|
151
|
+
else {
|
|
152
|
+
for (const tool of called) {
|
|
153
|
+
if (!expected.has(tool)) agent.lostTo.set(tool, (agent.lostTo.get(tool) || 0) + 1);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
agents.set(key, agent);
|
|
158
|
+
}
|
|
159
|
+
const arms = new Map();
|
|
160
|
+
for (const agent of agents.values()) arms.set(agent.harness, (arms.get(agent.harness) || 0) + 1);
|
|
161
|
+
for (const agent of agents.values()) {
|
|
162
|
+
agent.name = AGENT_NAMES[agent.harness] || agent.harness;
|
|
163
|
+
if (arms.get(agent.harness) > 1) {
|
|
164
|
+
agent.name += agent.arm === "deferred" ? " (on demand)" : " (always loaded)";
|
|
165
|
+
}
|
|
166
|
+
agent.rate = agent.scored ? agent.picked / agent.scored : null;
|
|
167
|
+
}
|
|
168
|
+
const groups = [...ENVIRONMENTS, { id: "other", label: "Elsewhere" }]
|
|
169
|
+
.map((env) => ({ ...env, agents: [...agents.values()].filter((a) => a.environment === env.id) }))
|
|
170
|
+
.filter((env) => env.agents.length);
|
|
171
|
+
const picked = [...agents.values()].reduce((sum, a) => sum + a.picked, 0);
|
|
172
|
+
const scored = [...agents.values()].reduce((sum, a) => sum + a.scored, 0);
|
|
173
|
+
return { groups, picked, scored, total: (cells || []).length };
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function renderSwarm(run, cells, { colour = false } = {}) {
|
|
177
|
+
const paint = paintFor(colour);
|
|
178
|
+
const summary = summarize(cells);
|
|
179
|
+
const done = (cells || []).filter((cell) => cell.status === "done" || cell.status === "failed").length;
|
|
180
|
+
const lines = [];
|
|
181
|
+
lines.push(
|
|
182
|
+
`${paint.bold(run.release_label || run.id)} ${paint.dim(run.hosted_mcp_name || run.hosted_mcp_slug || "")} ${run.status}`
|
|
183
|
+
);
|
|
184
|
+
if (summary.scored) {
|
|
185
|
+
const range = likelyRange(summary.picked, summary.scored);
|
|
186
|
+
lines.push(
|
|
187
|
+
`${TERMINAL.has(run.status) ? "Agents" : "So far, agents"} picked your tool ${summary.picked} of ${summary.scored} times ` +
|
|
188
|
+
paint.dim(`(likely ${Math.round(range[0] * 100)}–${Math.round(range[1] * 100)}%)`)
|
|
189
|
+
);
|
|
190
|
+
} else {
|
|
191
|
+
lines.push(paint.dim(TERMINAL.has(run.status) ? "Nothing was scored." : "Agents are on their way."));
|
|
192
|
+
}
|
|
193
|
+
lines.push(paint.dim(`${done} of ${summary.total} attempts done`));
|
|
194
|
+
for (const group of summary.groups) {
|
|
195
|
+
lines.push("");
|
|
196
|
+
lines.push(paint.bold(group.label));
|
|
197
|
+
for (const agent of group.agents) {
|
|
198
|
+
const rate = agent.rate === null ? paint.dim("—") : agent.rate >= 0.75 ? paint.green(pct(agent.rate)) : agent.rate >= 0.4 ? paint.yellow(pct(agent.rate)) : paint.red(pct(agent.rate));
|
|
199
|
+
const working = agent.running ? paint.dim(` · ${agent.running} working`) : "";
|
|
200
|
+
const rival = [...agent.lostTo.entries()].sort((a, b) => b[1] - a[1])[0];
|
|
201
|
+
const lost = rival ? paint.dim(` · lost to ${rival[0]} ×${rival[1]}`) : "";
|
|
202
|
+
lines.push(` ${truncate(agent.name, 30).padEnd(30)} ${String(rate).padStart(colour ? 13 : 4)} ${paint.dim(`${agent.picked}/${agent.scored}`)}${working}${lost}`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return `${lines.join("\n")}\n`;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function headlineFromRollup(run) {
|
|
209
|
+
const bucket = run?.rollup?.by_harness_visibility || {};
|
|
210
|
+
let hits = 0;
|
|
211
|
+
let n = 0;
|
|
212
|
+
for (const value of Object.values(bucket)) {
|
|
213
|
+
hits += value?.selected?.hits || 0;
|
|
214
|
+
n += value?.selected?.n || 0;
|
|
215
|
+
}
|
|
216
|
+
return n ? `picked ${hits} of ${n}` : "";
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function requireToken(args) {
|
|
220
|
+
const token = resolveApiKey(args);
|
|
221
|
+
if (!token) {
|
|
222
|
+
throw new Error(`no PreMan API key found. Run \`${cliInvocation()} login\` first, or pass --api-key.`);
|
|
223
|
+
}
|
|
224
|
+
return token;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async function call(args, token, method, path, options = {}) {
|
|
228
|
+
const result = await callBackendJson(args, method, path, { token, ...options });
|
|
229
|
+
if (!result.ok) {
|
|
230
|
+
throw new Error(`${method} ${path} on ${backendUrl(args)}: ${result.status_code} ${describeFailure(result, "request failed")}`);
|
|
231
|
+
}
|
|
232
|
+
return result;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async function findServer(request, ref) {
|
|
236
|
+
if (/^[0-9a-f-]{36}$/i.test(ref)) return { id: ref, name: ref };
|
|
237
|
+
const result = await request("GET", "/hosted-mcps");
|
|
238
|
+
const servers = result.hosted_mcps || result.list || [];
|
|
239
|
+
const match = servers.find((server) => [server.slug, server.name, server.id].includes(ref));
|
|
240
|
+
if (!match) {
|
|
241
|
+
const names = servers.map((server) => server.slug || server.name).filter(Boolean).join(", ");
|
|
242
|
+
throw new Error(`no hosted MCP called "${ref}"${names ? `. Yours: ${names}` : ""}`);
|
|
243
|
+
}
|
|
244
|
+
return match;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
async function readSwarm(request, id) {
|
|
248
|
+
const run = await request("GET", `/matrix/runs/${encodeURIComponent(id)}`);
|
|
249
|
+
const cells = await request("GET", `/matrix/runs/${encodeURIComponent(id)}/cells`);
|
|
250
|
+
const { status_code, ok, ...runBody } = run;
|
|
251
|
+
return { run: runBody, cells: cells.cells || cells.list || [] };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function swarmLink(args, id) {
|
|
255
|
+
return `${frontendUrl(args)}/workbench/swarms/swarms/${encodeURIComponent(id)}`;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export async function swarmCommand(commandArgs = [], {
|
|
259
|
+
sessionArgs, token: sessionToken, workspaceId, signal,
|
|
260
|
+
write = (text) => process.stdout.write(text), clearWatch = true,
|
|
261
|
+
} = {}) {
|
|
262
|
+
const [sub = "list", ...rest] = commandArgs;
|
|
263
|
+
// A session's credential stays with its backend even when slash-command
|
|
264
|
+
// arguments contain transport overrides. First occurrence wins in makeArgs.
|
|
265
|
+
const args = makeArgs(sessionArgs ? [
|
|
266
|
+
"--backend", backendUrl(sessionArgs), "--frontend", frontendUrl(sessionArgs),
|
|
267
|
+
...(sessionArgs.has("--no-color") || sessionArgs.has("--no-colour") ? ["--no-color"] : []), ...rest,
|
|
268
|
+
] : rest);
|
|
269
|
+
const json = args.has("--json");
|
|
270
|
+
const colour = colourEnabled(args);
|
|
271
|
+
const token = sessionToken || requireToken(args);
|
|
272
|
+
const request = (method, path, options = {}) => call(args, token, method, path, {
|
|
273
|
+
...options, signal, ...(workspaceId ? { headers: { "x-workspace-id": workspaceId } } : {}),
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
if (sub === "list") {
|
|
277
|
+
const result = await request("GET", "/matrix/runs");
|
|
278
|
+
const runs = result.runs || result.list || [];
|
|
279
|
+
if (json) return void write(`${JSON.stringify(runs, null, 2)}\n`);
|
|
280
|
+
if (!runs.length) {
|
|
281
|
+
write(`No swarms yet. Start one with \`${cliInvocation()} swarm launch\`.\n`);
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
for (const run of runs.slice(0, 20)) {
|
|
285
|
+
write(
|
|
286
|
+
`${String(run.id).slice(0, 8)} ${truncate(run.release_label || "", 32).padEnd(32)} ` +
|
|
287
|
+
`${truncate(run.hosted_mcp_name || run.hosted_mcp_slug || "", 20).padEnd(20)} ${String(run.status).padEnd(9)} ${headlineFromRollup(run)}\n`
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
if (sub === "launch") {
|
|
294
|
+
const asks = allValues(rest, "--ask");
|
|
295
|
+
const serverRef = args.value("--server");
|
|
296
|
+
if (!serverRef || !asks.length) {
|
|
297
|
+
throw new Error(`a swarm needs --server and at least one --ask.${SWARM_HELP}`);
|
|
298
|
+
}
|
|
299
|
+
const server = await findServer(request, serverRef);
|
|
300
|
+
const expect = listOf(args.value("--expect"));
|
|
301
|
+
const payload = {
|
|
302
|
+
hosted_mcp_id: server.id,
|
|
303
|
+
asks: asks.map((prompt) => ({ prompt, expected_tools: expect })),
|
|
304
|
+
distractor_levels: listOf(args.value("--rivals", "0,5")).map(Number).filter((n) => Number.isFinite(n)),
|
|
305
|
+
target_size: Number(args.value("--size", "60")),
|
|
306
|
+
release_label: args.value("--name") || `cli ${new Date().toISOString().slice(0, 16).replace("T", " ")}`,
|
|
307
|
+
};
|
|
308
|
+
if (!Number.isInteger(payload.target_size) || payload.target_size < 1) {
|
|
309
|
+
throw new Error("--size must be a whole number of attempts, 1 or more.");
|
|
310
|
+
}
|
|
311
|
+
const agents = listOf(args.value("--agents"));
|
|
312
|
+
if (agents.length) payload.harnesses = agents;
|
|
313
|
+
if (args.has("--dry-run")) payload.dry_run = true;
|
|
314
|
+
const result = await request("POST", "/matrix/runs", { json: payload });
|
|
315
|
+
const { status_code, ok, ...body } = result;
|
|
316
|
+
if (json) return void write(`${JSON.stringify(body, null, 2)}\n`);
|
|
317
|
+
if (payload.dry_run) {
|
|
318
|
+
write(`Would send ${body.cell_count ?? "?"} attempts (${body.asks_used ?? asks.length} asks, ${body.replicates ?? 1}× each).\n`);
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
write(`Swarm ${body.id} launched against ${server.name || server.slug}.\n`);
|
|
322
|
+
write(` Watch: ${cliInvocation()} swarm watch ${body.id}\n Open: ${swarmLink(args, body.id)}\n`);
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
if (sub === "watch" || sub === "show") {
|
|
327
|
+
const [id] = positionals(rest);
|
|
328
|
+
if (!id) throw new Error(`which swarm? ${cliInvocation()} swarm watch <id>`);
|
|
329
|
+
for (;;) {
|
|
330
|
+
const { run, cells } = await readSwarm(request, id);
|
|
331
|
+
if (json) {
|
|
332
|
+
write(`${JSON.stringify({ run, summary: summarize(cells) }, (k, v) => (v instanceof Map ? Object.fromEntries(v) : v), 2)}\n`);
|
|
333
|
+
} else {
|
|
334
|
+
if (clearWatch && process.stdout.isTTY && sub === "watch") write("\u001b[2J\u001b[H");
|
|
335
|
+
write(renderSwarm(run, cells, { colour }));
|
|
336
|
+
}
|
|
337
|
+
if (sub === "show" || args.has("--once") || TERMINAL.has(run.status)) {
|
|
338
|
+
if (!json) write(`\n${swarmLink(args, id)}\n`);
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
await wait(POLL_MS, undefined, { signal });
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
if (sub === "stop") {
|
|
346
|
+
const [id] = positionals(rest);
|
|
347
|
+
if (!id) throw new Error(`which swarm? ${cliInvocation()} swarm stop <id>`);
|
|
348
|
+
await request("POST", `/matrix/runs/${encodeURIComponent(id)}/cancel`);
|
|
349
|
+
write(`Stopped ${id}. Agents that had not finished will not run.\n`);
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
if (sub === "bank") {
|
|
354
|
+
const serverRef = args.value("--server");
|
|
355
|
+
const server = serverRef ? await findServer(request, serverRef) : null;
|
|
356
|
+
const result = await request("GET", "/swarms/behaviors", {
|
|
357
|
+
query: server ? { hosted_mcp_id: server.id } : undefined,
|
|
358
|
+
});
|
|
359
|
+
const kept = (result.behaviors || []).filter((row) => row.status === "kept");
|
|
360
|
+
if (json) return void write(`${JSON.stringify(kept, null, 2)}\n`);
|
|
361
|
+
if (!kept.length) {
|
|
362
|
+
write("Nothing kept yet. After a swarm, keep the patterns PreMan finds in its misses.\n");
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
for (const row of kept) {
|
|
366
|
+
write(`${row.hosted_mcp_name || ""} ${row.title} (${row.evidence?.count || 0} attempts)\n`);
|
|
367
|
+
for (const ask of (row.seed?.asks || []).slice(0, 3)) write(` “${truncate(ask.prompt, 70)}”\n`);
|
|
368
|
+
}
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
throw new Error(`unknown swarm command "${sub}".${SWARM_HELP}`);
|
|
373
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "premanmcp",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.18",
|
|
4
4
|
"description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"test:proxy": "node --test scripts/smoke-proxy.mjs",
|
|
17
17
|
"test:connect": "node scripts/smoke-connect.mjs",
|
|
18
18
|
"test:connect-heartbeat": "node scripts/smoke-connect-heartbeat.mjs",
|
|
19
|
-
"test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-start.mjs scripts/smoke-eval-run-watch.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-eval-runtime.mjs scripts/smoke-eval-entry.mjs scripts/smoke-eval-entry-placeholder.mjs scripts/smoke-eval-discover.mjs scripts/smoke-eval-templates.mjs scripts/smoke-eval-decorated-capture.mjs scripts/smoke-eval-decorate-approval.mjs scripts/smoke-eval-approval-record.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-prompts.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs",
|
|
19
|
+
"test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-start.mjs scripts/smoke-eval-run-watch.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-eval-runtime.mjs scripts/smoke-eval-entry.mjs scripts/smoke-eval-entry-placeholder.mjs scripts/smoke-eval-discover.mjs scripts/smoke-eval-templates.mjs scripts/smoke-eval-decorated-capture.mjs scripts/smoke-eval-decorate-approval.mjs scripts/smoke-eval-approval-record.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-prompts.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs scripts/smoke-swarm.mjs",
|
|
20
20
|
"test:dmg": "node --test --test-timeout=300000 scripts/smoke-install-desktop-volume.mjs"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|