premanmcp 1.1.7 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/agent.js +315 -36
- package/bin/desktop.js +2 -2
- package/bin/detect.js +29 -0
- package/bin/eval.js +430 -22
- package/bin/eval_entry.js +762 -0
- package/bin/eval_harness.py +166 -5
- package/bin/eval_runtime.js +156 -0
- package/bin/eval_target.js +160 -113
- package/bin/eval_templates.js +684 -0
- package/bin/runner.js +42 -3
- package/bin/shared.js +2 -1
- package/bin/target.js +105 -3
- package/package.json +2 -2
package/bin/runner.js
CHANGED
|
@@ -738,7 +738,10 @@ export async function sendHeartbeat(args, state, { busy = false, log = () => {}
|
|
|
738
738
|
async function serveEvalTarget(args, { log }) {
|
|
739
739
|
let target;
|
|
740
740
|
try {
|
|
741
|
-
|
|
741
|
+
// Cache-only, and explicitly so rather than by accident of having no
|
|
742
|
+
// terminal: `preman runner` is sometimes started from one, and a daemon
|
|
743
|
+
// that stopped to ask a question mid-heartbeat would hang the device.
|
|
744
|
+
target = await discoverTarget(args, { log: () => {}, allowSearch: false });
|
|
742
745
|
} catch (error) {
|
|
743
746
|
log(`no agent to evaluate on this machine (${error.message}); eval runs will not be offered`);
|
|
744
747
|
return null;
|
|
@@ -758,7 +761,21 @@ async function serveEvalTarget(args, { log }) {
|
|
|
758
761
|
log(`offering ${target.label} for evals`);
|
|
759
762
|
advertise({
|
|
760
763
|
agent_eval_v1: true,
|
|
761
|
-
eval_target: {
|
|
764
|
+
eval_target: {
|
|
765
|
+
// A callable has no address to advertise; what makes it runnable is
|
|
766
|
+
// that a wrapper was approved here, which the digest names.
|
|
767
|
+
...(target.kind === "callable"
|
|
768
|
+
? {
|
|
769
|
+
callable: target.ref,
|
|
770
|
+
entry_file: target.entryFile,
|
|
771
|
+
entry_sha256: target.sha256,
|
|
772
|
+
language: target.language,
|
|
773
|
+
framework: target.framework,
|
|
774
|
+
}
|
|
775
|
+
: { url: adapter.url }),
|
|
776
|
+
label: target.label,
|
|
777
|
+
how: target.how,
|
|
778
|
+
},
|
|
762
779
|
});
|
|
763
780
|
return adapter;
|
|
764
781
|
} catch (error) {
|
|
@@ -787,6 +804,13 @@ export async function runnerLoop(
|
|
|
787
804
|
// daemon; only the interactive `preman eval run` has somebody in front of it
|
|
788
805
|
// to hand a run over to.
|
|
789
806
|
headless = true,
|
|
807
|
+
// Lets a caller cut a stream that is open but idle. `stopWhen` cannot do
|
|
808
|
+
// that on its own: it is read in the `while` condition and after a job, and
|
|
809
|
+
// an idle stream is inside neither -- the loop is parked in `for await` on
|
|
810
|
+
// the response body, which no predicate reaches. A caller that has learnt
|
|
811
|
+
// the run is over out of band aborts this, the read throws, and the loop
|
|
812
|
+
// comes back around to `stopWhen` and leaves.
|
|
813
|
+
signal = null,
|
|
790
814
|
} = {}
|
|
791
815
|
) {
|
|
792
816
|
let stopped = false;
|
|
@@ -812,8 +836,23 @@ export async function runnerLoop(
|
|
|
812
836
|
try {
|
|
813
837
|
response = await fetch(eventsUrl, {
|
|
814
838
|
headers: { Authorization: `Bearer ${state.runner_token}`, Accept: "text/event-stream" },
|
|
839
|
+
...(signal ? { signal } : {}),
|
|
815
840
|
});
|
|
816
841
|
} catch (error) {
|
|
842
|
+
// An abort is this loop being told to stop, not a network problem.
|
|
843
|
+
// Reporting it as one would put "stream failed" in front of somebody
|
|
844
|
+
// who is about to be told the real reason the run ended.
|
|
845
|
+
//
|
|
846
|
+
// `stopped` and not just `continue`: without it this depends on the
|
|
847
|
+
// caller's `stopWhen` already answering true, and a caller that passes
|
|
848
|
+
// a signal without one gets a hot loop -- the fetch throws at once, the
|
|
849
|
+
// backoff sleep below is skipped, and the `while` condition is still
|
|
850
|
+
// true. Aborting is a decision to stop, so record it here and let the
|
|
851
|
+
// two options be correct apart as well as together.
|
|
852
|
+
if (signal?.aborted) {
|
|
853
|
+
stopped = true;
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
817
856
|
log(`stream failed: ${error.message}; retrying in ${Math.round(backoff / 1000)}s`);
|
|
818
857
|
await sleep(backoff);
|
|
819
858
|
backoff = Math.min(backoff * 2, RECONNECT_MAX_MS);
|
|
@@ -894,7 +933,7 @@ export async function runnerLoop(
|
|
|
894
933
|
if (terminal) break;
|
|
895
934
|
}
|
|
896
935
|
} catch (error) {
|
|
897
|
-
log(`stream dropped: ${error.message}`);
|
|
936
|
+
if (!signal?.aborted) log(`stream dropped: ${error.message}`);
|
|
898
937
|
}
|
|
899
938
|
|
|
900
939
|
if (terminal === "revoked") {
|
package/bin/shared.js
CHANGED
|
@@ -506,7 +506,7 @@ export async function callBackendJson(
|
|
|
506
506
|
args,
|
|
507
507
|
method,
|
|
508
508
|
routePath,
|
|
509
|
-
{ json, form, token, query, headers: extraHeaders } = {}
|
|
509
|
+
{ json, form, token, query, headers: extraHeaders, signal } = {}
|
|
510
510
|
) {
|
|
511
511
|
const url = new URL(routePath.replace(/^\/+/, ""), `${backendUrl(args)}/`);
|
|
512
512
|
if (query) {
|
|
@@ -533,6 +533,7 @@ export async function callBackendJson(
|
|
|
533
533
|
method,
|
|
534
534
|
headers,
|
|
535
535
|
body: form !== undefined && form !== null ? form : hasBody ? JSON.stringify(json) : undefined,
|
|
536
|
+
signal,
|
|
536
537
|
});
|
|
537
538
|
} catch (error) {
|
|
538
539
|
// `fetch failed` is what Node says when nothing accepted the connection,
|
package/bin/target.js
CHANGED
|
@@ -68,6 +68,27 @@ export function findConfigRoot(from = process.cwd()) {
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
/**
|
|
72
|
+
* The repository `from` sits in, by its VCS marker.
|
|
73
|
+
*
|
|
74
|
+
* `findConfigRoot` answers a different question and answers it correctly: it
|
|
75
|
+
* stops at `.git` and returns null, because a `.preman.json` *above* the
|
|
76
|
+
* repository belongs to a different project. But a first run has no
|
|
77
|
+
* `.preman.json` to find, so a caller that falls back to `from` treats whatever
|
|
78
|
+
* directory somebody happened to be standing in as the project root -- and a
|
|
79
|
+
* customer who `cd`s into `services/bot` before pasting a command gets "does not
|
|
80
|
+
* exist" for a path that plainly does.
|
|
81
|
+
*/
|
|
82
|
+
export function findRepoRoot(from = process.cwd()) {
|
|
83
|
+
let dir = path.resolve(from);
|
|
84
|
+
for (;;) {
|
|
85
|
+
if (existsSync(path.join(dir, ".git"))) return dir;
|
|
86
|
+
const parent = path.dirname(dir);
|
|
87
|
+
if (parent === dir) return null;
|
|
88
|
+
dir = parent;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
71
92
|
export function readConfig(root = process.cwd()) {
|
|
72
93
|
try {
|
|
73
94
|
const parsed = JSON.parse(readFileSync(configPath(root), "utf8"));
|
|
@@ -104,6 +125,58 @@ export function savedAgent(root = process.cwd()) {
|
|
|
104
125
|
};
|
|
105
126
|
}
|
|
106
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Record the code entry point somebody approved, beside the HTTP one.
|
|
130
|
+
*
|
|
131
|
+
* A sibling of `agent` rather than a replacement for it: a repository can
|
|
132
|
+
* legitimately have both, and which one an eval uses is a precedence question
|
|
133
|
+
* answered at discovery time, not a reason to discard the other.
|
|
134
|
+
*/
|
|
135
|
+
export function saveEntry(entry, root = process.cwd()) {
|
|
136
|
+
const config = readConfig(root);
|
|
137
|
+
const next = { ...config, entry };
|
|
138
|
+
writeFileSync(configPath(root), `${JSON.stringify(next, null, 2)}\n`, "utf8");
|
|
139
|
+
return next;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The approved shim, or null.
|
|
144
|
+
*
|
|
145
|
+
* `sha256` is returned unvalidated -- proving it still matches means reading
|
|
146
|
+
* the file, which is the caller's job because the caller is the one that knows
|
|
147
|
+
* whether a mismatch should prompt or refuse. What is enforced here is only
|
|
148
|
+
* that the record is complete enough to act on.
|
|
149
|
+
*/
|
|
150
|
+
export function savedEntry(root = process.cwd()) {
|
|
151
|
+
const configRoot = findConfigRoot(root) || root;
|
|
152
|
+
const entry = readConfig(configRoot).entry;
|
|
153
|
+
if (!entry || typeof entry !== "object") return null;
|
|
154
|
+
const file = String(entry.file || "").trim();
|
|
155
|
+
const ref = String(entry.ref || "").trim();
|
|
156
|
+
if (!file || !ref) return null;
|
|
157
|
+
return {
|
|
158
|
+
root: configRoot,
|
|
159
|
+
file,
|
|
160
|
+
ref,
|
|
161
|
+
sysPath: String(entry.sysPath || ""),
|
|
162
|
+
sha256: String(entry.sha256 || ""),
|
|
163
|
+
// Additional approved files, each with its own digest. Absent on a record
|
|
164
|
+
// written before two-file targets existed, which is why the caller treats
|
|
165
|
+
// an empty list as "only the wrapper" rather than as "nothing to check".
|
|
166
|
+
files: (Array.isArray(entry.files) ? entry.files : [])
|
|
167
|
+
.map((item) => ({
|
|
168
|
+
path: String(item?.path || ""),
|
|
169
|
+
sha256: String(item?.sha256 || ""),
|
|
170
|
+
}))
|
|
171
|
+
.filter((item) => item.path && item.sha256),
|
|
172
|
+
shape: String(entry.shape || ""),
|
|
173
|
+
language: String(entry.language || "python"),
|
|
174
|
+
framework: String(entry.framework || ""),
|
|
175
|
+
entryFile: String(entry.entryFile || ""),
|
|
176
|
+
approvedAt: String(entry.approvedAt || ""),
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
107
180
|
/**
|
|
108
181
|
* Where this project's agent might be listening.
|
|
109
182
|
*
|
|
@@ -271,18 +344,47 @@ export async function startProject(command, { root = process.cwd(), waitMs = 60_
|
|
|
271
344
|
}
|
|
272
345
|
};
|
|
273
346
|
|
|
347
|
+
/**
|
|
348
|
+
* Is anything in the group still running?
|
|
349
|
+
*
|
|
350
|
+
* `exited` cannot answer this, and trusting it was the bug. `shell: true`
|
|
351
|
+
* means the command runs under `/bin/sh -c`, which does not always exec, so
|
|
352
|
+
* the wrapper and the server it launched are two processes. The wrapper traps
|
|
353
|
+
* nothing, dies on the first SIGTERM and sets `exited` -- measured at 102ms --
|
|
354
|
+
* while the server that *does* trap SIGTERM keeps running. The escalation
|
|
355
|
+
* below was then skipped, and a dev server survived `stop()` still holding its
|
|
356
|
+
* port, which is exactly what the caller asked to have stopped.
|
|
357
|
+
*
|
|
358
|
+
* Signal 0 asks the kernel about the group without touching it. Negative pid
|
|
359
|
+
* for the group, matching `signal` above.
|
|
360
|
+
*/
|
|
361
|
+
const groupAlive = () => {
|
|
362
|
+
try {
|
|
363
|
+
process.kill(-child.pid, 0);
|
|
364
|
+
return true;
|
|
365
|
+
} catch (error) {
|
|
366
|
+
// Somebody else's now, which is not this function's business, but it is
|
|
367
|
+
// certainly not gone.
|
|
368
|
+
return error?.code === "EPERM";
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
|
|
274
372
|
const stop = async () => {
|
|
275
|
-
if (
|
|
373
|
+
if (!groupAlive()) return;
|
|
276
374
|
signal("SIGTERM");
|
|
277
375
|
// Then SIGKILL, and the pipes closed behind it. A dev server that ignores
|
|
278
376
|
// SIGTERM keeps this process alive forever otherwise: its stdout is still
|
|
279
377
|
// open here, so Node has a reason to stay, and `preman target detect`
|
|
280
378
|
// simply never returns.
|
|
379
|
+
//
|
|
380
|
+
// The grace is not spent on a well-behaved server: its whole group is gone
|
|
381
|
+
// about as fast as the wrapper is, measured at 101ms, so this loop leaves on
|
|
382
|
+
// the first pass.
|
|
281
383
|
const deadline = Date.now() + STOP_GRACE_MS;
|
|
282
|
-
while (
|
|
384
|
+
while (groupAlive() && Date.now() < deadline) {
|
|
283
385
|
await new Promise((resume) => setTimeout(resume, 100));
|
|
284
386
|
}
|
|
285
|
-
if (
|
|
387
|
+
if (groupAlive()) {
|
|
286
388
|
log(`${command.split(" ")[0]} did not stop when asked; killing it`);
|
|
287
389
|
signal("SIGKILL");
|
|
288
390
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "premanmcp",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.9",
|
|
4
4
|
"description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"test:proxy": "node --test scripts/smoke-proxy.mjs",
|
|
17
17
|
"test:connect": "node scripts/smoke-connect.mjs",
|
|
18
18
|
"test:connect-heartbeat": "node scripts/smoke-connect-heartbeat.mjs",
|
|
19
|
-
"test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs",
|
|
19
|
+
"test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-run-watch.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-eval-runtime.mjs scripts/smoke-eval-entry.mjs scripts/smoke-eval-entry-placeholder.mjs scripts/smoke-eval-discover.mjs scripts/smoke-eval-templates.mjs scripts/smoke-eval-approval-record.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs",
|
|
20
20
|
"test:dmg": "node --test --test-timeout=300000 scripts/smoke-install-desktop-volume.mjs"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|