premanmcp 1.1.6 → 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/agent.js +416 -25
- package/bin/connect/pairing.js +26 -0
- package/bin/connect.js +8 -2
- package/bin/eval.js +94 -5
- package/bin/runner.js +13 -0
- package/package.json +2 -1
package/bin/agent.js
CHANGED
|
@@ -84,12 +84,13 @@ const THINK_MS = 220;
|
|
|
84
84
|
const CLEAR_LINE = "\r\u001b[2K";
|
|
85
85
|
const CLEAR_SCREEN = "\u001b[2J\u001b[H";
|
|
86
86
|
|
|
87
|
-
// The alternate screen buffer
|
|
88
|
-
//
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
|
|
92
|
-
|
|
87
|
+
// The alternate screen buffer is deliberately not used, and the sequences for
|
|
88
|
+
// it are deliberately not defined, so that reaching for one is a decision
|
|
89
|
+
// rather than an autocomplete. It gave a session a screen of its own -- a
|
|
90
|
+
// second `preman` could not be scrolled back into the first -- by holding
|
|
91
|
+
// exactly one screen and keeping no history, which meant a reply that ran past
|
|
92
|
+
// the top of the window could not be scrolled back to either. A transcript is
|
|
93
|
+
// the wrong thing to put in a buffer that forgets.
|
|
93
94
|
const SHOW_CURSOR = "\u001b[?25h";
|
|
94
95
|
const RELEASE_REGION = "\u001b[r";
|
|
95
96
|
|
|
@@ -217,6 +218,188 @@ export function artifactLine(artifact) {
|
|
|
217
218
|
return url ? `${truncate(label, 72)} — ${url}` : truncate(label, 100);
|
|
218
219
|
}
|
|
219
220
|
|
|
221
|
+
/**
|
|
222
|
+
* The route an endpoint sits on, normalised so two saved requests that hit the
|
|
223
|
+
* same place collapse onto one line.
|
|
224
|
+
*
|
|
225
|
+
* The same rule the app groups by, deliberately: a workspace that reads as 84
|
|
226
|
+
* routes in the browser and 284 rows here is two different answers to one
|
|
227
|
+
* question. A saved URL is often a template (`{{baseUrl}}/v1/users`) rather
|
|
228
|
+
* than something `URL` will parse, which is why the fallback strips the
|
|
229
|
+
* template head by hand instead of giving up.
|
|
230
|
+
*/
|
|
231
|
+
export function endpointRoutePath(endpoint) {
|
|
232
|
+
const raw = String(endpoint?.url || endpoint?.name || "/").trim();
|
|
233
|
+
let path = raw;
|
|
234
|
+
try {
|
|
235
|
+
path = new URL(raw).pathname || "/";
|
|
236
|
+
} catch {
|
|
237
|
+
path = raw.replace(/^\{\{[^}]+\}\}/, "").split(/[?#]/, 1)[0] || "/";
|
|
238
|
+
}
|
|
239
|
+
if (!path.startsWith("/")) path = `/${path}`;
|
|
240
|
+
return path.length > 1 ? path.replace(/\/+$/, "") : path;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// The three buckets the server counts in and the model writes its reply from.
|
|
244
|
+
// Matching them exactly is what keeps "87 failing" in the prose and the heading
|
|
245
|
+
// above the rows from being two different numbers.
|
|
246
|
+
const FAILING_RUNS = new Set(["failed", "error", "fail"]);
|
|
247
|
+
const PASSING_RUNS = new Set(["passed", "pass", "ok", "success"]);
|
|
248
|
+
|
|
249
|
+
/** Which of the three buckets a last-run status falls in. */
|
|
250
|
+
export function endpointBucket(lastRun) {
|
|
251
|
+
const status = String(lastRun || "").trim().toLowerCase();
|
|
252
|
+
if (!status) return "untested";
|
|
253
|
+
if (FAILING_RUNS.has(status)) return "failing";
|
|
254
|
+
if (PASSING_RUNS.has(status)) return "passing";
|
|
255
|
+
return "untested";
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Failing first because it is the only bucket anybody acts on, then untested,
|
|
259
|
+
// then passing -- the order the model already writes its own summary in.
|
|
260
|
+
const BUCKET_ORDER = ["failing", "untested", "passing"];
|
|
261
|
+
const BUCKET_LABELS = { failing: "Failing", untested: "Untested", passing: "Passing" };
|
|
262
|
+
const METHOD_ORDER = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"];
|
|
263
|
+
|
|
264
|
+
const methodRank = (method) => {
|
|
265
|
+
const at = METHOD_ORDER.indexOf(String(method || "GET").trim().toUpperCase());
|
|
266
|
+
return at === -1 ? METHOD_ORDER.length : at;
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Every endpoint the turn returned, once, as routes.
|
|
271
|
+
*
|
|
272
|
+
* A list of 284 arrives as thirteen separate tool calls, because the tool caps
|
|
273
|
+
* a page at 25 rows and the model pages through it. Ids are deduplicated
|
|
274
|
+
* because overlapping pages are a model's decision and not a thing to render
|
|
275
|
+
* twice; routes are collapsed because that is what turns request history back
|
|
276
|
+
* into an API. A route whose saved requests disagree about their last run
|
|
277
|
+
* takes the worst of them, since "one of these is failing" is the fact worth
|
|
278
|
+
* putting on a line somebody is scanning for failures.
|
|
279
|
+
*/
|
|
280
|
+
export function mergeEndpoints(artifacts = []) {
|
|
281
|
+
const seen = new Set();
|
|
282
|
+
const endpoints = [];
|
|
283
|
+
// The furthest page reached in each sweep the model made. A turn that lists
|
|
284
|
+
// everything is several sweeps -- one per status -- and every page but the
|
|
285
|
+
// last of each carries has_more, so believing all of them would claim the
|
|
286
|
+
// catalogue was cut short on exactly the turns that read all of it.
|
|
287
|
+
const sweeps = new Map();
|
|
288
|
+
let saved = 0;
|
|
289
|
+
|
|
290
|
+
for (const artifact of artifacts) {
|
|
291
|
+
saved = Math.max(saved, Number(artifact?.total_count) || 0);
|
|
292
|
+
const key = `${artifact?.status ?? ""}|${artifact?.query ?? ""}`;
|
|
293
|
+
const offset = Number(artifact?.offset) || 0;
|
|
294
|
+
const reached = offset + (artifact?.endpoints?.length || 0);
|
|
295
|
+
const furthest = sweeps.get(key);
|
|
296
|
+
if (!furthest || offset >= furthest.offset) {
|
|
297
|
+
sweeps.set(key, {
|
|
298
|
+
offset,
|
|
299
|
+
more: Boolean(artifact?.has_more) || (Number(artifact?.matched_count) || 0) > reached,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
for (const endpoint of artifact?.endpoints || []) {
|
|
303
|
+
const id = String(endpoint?.id ?? "");
|
|
304
|
+
if (id && seen.has(id)) continue;
|
|
305
|
+
if (id) seen.add(id);
|
|
306
|
+
endpoints.push(endpoint);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const routes = new Map();
|
|
311
|
+
const counts = { failing: 0, untested: 0, passing: 0 };
|
|
312
|
+
for (const endpoint of endpoints) {
|
|
313
|
+
const method = String(endpoint?.method || "GET").trim().toUpperCase();
|
|
314
|
+
const path = endpointRoutePath(endpoint);
|
|
315
|
+
const bucket = endpointBucket(endpoint?.last_run);
|
|
316
|
+
counts[bucket] += 1;
|
|
317
|
+
const key = `${method} ${path}`;
|
|
318
|
+
const found = routes.get(key);
|
|
319
|
+
if (!found) {
|
|
320
|
+
routes.set(key, { method, path, bucket, count: 1 });
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
found.count += 1;
|
|
324
|
+
if (BUCKET_ORDER.indexOf(bucket) < BUCKET_ORDER.indexOf(found.bucket)) found.bucket = bucket;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const ordered = [...routes.values()].sort(
|
|
328
|
+
(left, right) => methodRank(left.method) - methodRank(right.method) || left.path.localeCompare(right.path)
|
|
329
|
+
);
|
|
330
|
+
return { endpoints, routes: ordered, counts, saved, more: [...sweeps.values()].some((s) => s.more) };
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* The endpoints of a turn, drawn for a terminal.
|
|
335
|
+
*
|
|
336
|
+
* Until now this surface printed the artifact's title and dropped its payload,
|
|
337
|
+
* so a request to list 284 endpoints produced thirteen lines reading "showing
|
|
338
|
+
* 51-75 of 197" and not one endpoint. The model's reply said the list was
|
|
339
|
+
* above, and in the app it is -- the card renders the rows. Here there is no
|
|
340
|
+
* card and no page to send anybody to: the transcript is the whole surface, so
|
|
341
|
+
* the rows have to land in it.
|
|
342
|
+
*
|
|
343
|
+
* Piped output is a different contract and gets a different shape: one line per
|
|
344
|
+
* endpoint, ungrouped and never truncated, because on the other end of a pipe
|
|
345
|
+
* this is something being grepped rather than read.
|
|
346
|
+
*/
|
|
347
|
+
export function endpointsBlock(artifacts, { columns = 80, live = true, paint } = {}) {
|
|
348
|
+
const ink = paint || makePaint(false);
|
|
349
|
+
const { endpoints, routes, counts, saved, more } = mergeEndpoints(artifacts);
|
|
350
|
+
if (!endpoints.length) return "";
|
|
351
|
+
|
|
352
|
+
if (!live) {
|
|
353
|
+
const lines = endpoints.map((endpoint) => {
|
|
354
|
+
const method = String(endpoint?.method || "GET").trim().toUpperCase();
|
|
355
|
+
const where = String(endpoint?.url || "").trim() || endpointRoutePath(endpoint);
|
|
356
|
+
return ` ${endpointBucket(endpoint?.last_run).padEnd(8)} ${method.padEnd(7)} ${where}`;
|
|
357
|
+
});
|
|
358
|
+
return `${lines.join("\n")}\n`;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
const shown = endpoints.length;
|
|
362
|
+
const head = [`${shown} endpoint${shown === 1 ? "" : "s"}`];
|
|
363
|
+
if (saved > shown) head[0] = `${shown} of ${saved} endpoints`;
|
|
364
|
+
if (routes.length !== shown) head.push(`${routes.length} route${routes.length === 1 ? "" : "s"}`);
|
|
365
|
+
for (const bucket of BUCKET_ORDER) {
|
|
366
|
+
if (counts[bucket]) head.push(`${counts[bucket]} ${bucket}`);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const width = Math.max(24, columns);
|
|
370
|
+
// Truncated before it is painted, not after: the colour runs are bytes that
|
|
371
|
+
// `truncate` would count and cut through, and half an escape sequence on a
|
|
372
|
+
// narrow window is a terminal left in the wrong colour.
|
|
373
|
+
const lines = [
|
|
374
|
+
` ${ink.bold("Endpoints")} ${ink.dim(`\u00b7 ${truncate(head.join(" \u00b7 "), Math.max(8, width - 14))}`)}`,
|
|
375
|
+
];
|
|
376
|
+
const method = Math.min(7, Math.max(...routes.map((route) => route.method.length)));
|
|
377
|
+
const room = Math.max(8, width - 6 - method - 2);
|
|
378
|
+
|
|
379
|
+
for (const bucket of BUCKET_ORDER) {
|
|
380
|
+
const group = routes.filter((route) => route.bucket === bucket);
|
|
381
|
+
if (!group.length) continue;
|
|
382
|
+
lines.push("", ` ${ink.bold(BUCKET_LABELS[bucket])} ${ink.dim(`(${counts[bucket]})`)}`);
|
|
383
|
+
for (const route of group) {
|
|
384
|
+
// A hollow mark for untested, because "no run yet" is an absence rather
|
|
385
|
+
// than a result, and a filled dot in any colour reads as one.
|
|
386
|
+
const dot =
|
|
387
|
+
bucket === "failing" ? ink.red("\u25cf") : bucket === "passing" ? ink.green("\u25cf") : ink.dim("\u25cb");
|
|
388
|
+
const also = route.count > 1 ? ink.dim(` (${route.count})`) : "";
|
|
389
|
+
lines.push(` ${dot} ${ink.dim(route.method.padEnd(method))} ${truncate(route.path, room)}${also}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Said once, at the end, rather than thirteen times on the way: the model
|
|
394
|
+
// stopped paging before the catalogue ran out, and the rest is a command
|
|
395
|
+
// away rather than a page in a browser this session cannot open.
|
|
396
|
+
if (more) {
|
|
397
|
+
const rest = `More were not fetched. ${cliInvocation()} endpoints list for the rest.`;
|
|
398
|
+
lines.push("", ` ${ink.dim(truncate(rest, width - 2))}`);
|
|
399
|
+
}
|
|
400
|
+
return `${lines.join("\n")}\n`;
|
|
401
|
+
}
|
|
402
|
+
|
|
220
403
|
/** The permission questions in a finished turn, in the order they were asked. */
|
|
221
404
|
export function permissionQuestions(artifacts = []) {
|
|
222
405
|
return (Array.isArray(artifacts) ? artifacts : []).filter(
|
|
@@ -268,6 +451,13 @@ function makePaint(enabled) {
|
|
|
268
451
|
*/
|
|
269
452
|
export function createRenderer({ stream = process.stdout, paint, live } = {}) {
|
|
270
453
|
const isLive = live ?? Boolean(stream.isTTY);
|
|
454
|
+
// Whether there is a window here, which is a different question from whether
|
|
455
|
+
// it may be coloured. `NO_COLOR=1` is a widespread way of saying "no colour",
|
|
456
|
+
// and it used to reach layout as "this is a pipe" -- so somebody who set it
|
|
457
|
+
// and sat down at a terminal got the shape meant for `grep`. Colour is the
|
|
458
|
+
// palette's business; how wide the window is and whether rows may be grouped
|
|
459
|
+
// and clipped is this.
|
|
460
|
+
const isTerminal = Boolean(stream.isTTY);
|
|
271
461
|
const ink = paint || makePaint(isLive);
|
|
272
462
|
let timer = null;
|
|
273
463
|
let frame = 0;
|
|
@@ -277,6 +467,8 @@ export function createRenderer({ stream = process.stdout, paint, live } = {}) {
|
|
|
277
467
|
let stallTick = 0;
|
|
278
468
|
let wroteReply = false;
|
|
279
469
|
let atLineStart = true;
|
|
470
|
+
// Endpoint pages, held until the turn has something else to say.
|
|
471
|
+
let pages = [];
|
|
280
472
|
|
|
281
473
|
function stopStall() {
|
|
282
474
|
if (!stall) return false;
|
|
@@ -312,6 +504,26 @@ export function createRenderer({ stream = process.stdout, paint, live } = {}) {
|
|
|
312
504
|
stream.write(`${CLEAR_LINE}${tint(`${word}${runOn} ${dots}`)}`);
|
|
313
505
|
}
|
|
314
506
|
|
|
507
|
+
/**
|
|
508
|
+
* Draw the endpoint pages collected so far, as one table.
|
|
509
|
+
*
|
|
510
|
+
* Held rather than printed on arrival because a single "list my endpoints"
|
|
511
|
+
* is thirteen tool calls, and thirteen tables of 25 rows is not a list --
|
|
512
|
+
* it is the pagination read aloud. Flushed by whatever speaks next, so the
|
|
513
|
+
* rows still land *above* the reply that refers to them.
|
|
514
|
+
*/
|
|
515
|
+
function flushPages() {
|
|
516
|
+
if (!pages.length) return;
|
|
517
|
+
const held = pages;
|
|
518
|
+
pages = [];
|
|
519
|
+
const block = endpointsBlock(held, { columns: stream.columns || 80, live: isTerminal, paint: ink });
|
|
520
|
+
if (!block) return;
|
|
521
|
+
clearSpinner();
|
|
522
|
+
if (!atLineStart) stream.write("\n");
|
|
523
|
+
stream.write(block);
|
|
524
|
+
atLineStart = true;
|
|
525
|
+
}
|
|
526
|
+
|
|
315
527
|
function draw() {
|
|
316
528
|
if (!isLive || !label) return;
|
|
317
529
|
const spin = FRAMES[frame % FRAMES.length];
|
|
@@ -347,8 +559,16 @@ export function createRenderer({ stream = process.stdout, paint, live } = {}) {
|
|
|
347
559
|
}
|
|
348
560
|
},
|
|
349
561
|
artifact(item) {
|
|
562
|
+
// Endpoint pages are the one artifact whose payload is the point, and
|
|
563
|
+
// they arrive one page at a time, so they are collected rather than
|
|
564
|
+
// drawn. Everything else is a card this surface summarises in a line.
|
|
565
|
+
if (item && item.type === "endpoints" && Array.isArray(item.endpoints)) {
|
|
566
|
+
pages.push(item);
|
|
567
|
+
return;
|
|
568
|
+
}
|
|
350
569
|
const line = artifactLine(item);
|
|
351
570
|
if (!line) return;
|
|
571
|
+
flushPages();
|
|
352
572
|
clearSpinner();
|
|
353
573
|
if (!atLineStart) stream.write("\n");
|
|
354
574
|
stream.write(` ${ink.cyan("●")} ${line}\n`);
|
|
@@ -357,6 +577,7 @@ export function createRenderer({ stream = process.stdout, paint, live } = {}) {
|
|
|
357
577
|
delta(text) {
|
|
358
578
|
const chunk = String(text ?? "");
|
|
359
579
|
if (!chunk) return;
|
|
580
|
+
flushPages();
|
|
360
581
|
clearSpinner();
|
|
361
582
|
if (!wroteReply) {
|
|
362
583
|
stream.write("\n");
|
|
@@ -373,18 +594,23 @@ export function createRenderer({ stream = process.stdout, paint, live } = {}) {
|
|
|
373
594
|
this.delta(body);
|
|
374
595
|
},
|
|
375
596
|
note(text) {
|
|
597
|
+
flushPages();
|
|
376
598
|
clearSpinner();
|
|
377
599
|
if (!atLineStart) stream.write("\n");
|
|
378
600
|
stream.write(`${ink.dim(String(text))}\n`);
|
|
379
601
|
atLineStart = true;
|
|
380
602
|
},
|
|
381
603
|
error(text) {
|
|
604
|
+
flushPages();
|
|
382
605
|
clearSpinner();
|
|
383
606
|
if (!atLineStart) stream.write("\n");
|
|
384
607
|
stream.write(`${ink.red(String(text))}\n`);
|
|
385
608
|
atLineStart = true;
|
|
386
609
|
},
|
|
387
610
|
end() {
|
|
611
|
+
// The backstop: a turn that listed endpoints and then said nothing at
|
|
612
|
+
// all still has to put them on the screen.
|
|
613
|
+
flushPages();
|
|
388
614
|
clearSpinner();
|
|
389
615
|
if (!atLineStart) stream.write("\n");
|
|
390
616
|
if (wroteReply) stream.write("\n");
|
|
@@ -705,7 +931,14 @@ export function holdTerminal({ stream = process.stdout, footer = () => "" } = {}
|
|
|
705
931
|
if (restored) return;
|
|
706
932
|
restored = true;
|
|
707
933
|
for (const fn of teardown.splice(0)) safely(fn);
|
|
708
|
-
|
|
934
|
+
// No `ESC[?1049l` here any more, and its absence is load-bearing rather
|
|
935
|
+
// than tidying. The session never enters the alternate screen, so leaving
|
|
936
|
+
// it would be addressed to a buffer nobody is on -- and the sequence is
|
|
937
|
+
// not inert: it restores the cursor as DECRC does, from the same saved
|
|
938
|
+
// slot the dock writes the transcript's position into with `ESC7`. Sent on
|
|
939
|
+
// the shell's own screen it would drag the cursor back up into the
|
|
940
|
+
// transcript and print the resume footer over the conversation.
|
|
941
|
+
if (live) safely(() => stream.write(`${RELEASE_REGION}${SHOW_CURSOR}${CLEAR_TITLE}`));
|
|
709
942
|
safely(() => {
|
|
710
943
|
const text = footer();
|
|
711
944
|
if (text) stream.write(text);
|
|
@@ -829,6 +1062,11 @@ export function createDock({
|
|
|
829
1062
|
// old rows before painting new ones, and after the event they are no longer
|
|
830
1063
|
// derivable from the terminal -- it has already reflowed.
|
|
831
1064
|
let painted = null;
|
|
1065
|
+
// Where the transcript has got to, in screen coordinates. The terminal keeps
|
|
1066
|
+
// this too, in its saved cursor, and that copy is the one the writes use --
|
|
1067
|
+
// but a resize destroys it, so it is mirrored here to be put back. Null means
|
|
1068
|
+
// nobody knows: before `enable`, and after any stretch the dock did not paint.
|
|
1069
|
+
let at = null;
|
|
832
1070
|
|
|
833
1071
|
const size = () => ({
|
|
834
1072
|
columns: stream.columns || 80,
|
|
@@ -838,6 +1076,82 @@ export function createDock({
|
|
|
838
1076
|
/** Last row the transcript may use; everything below belongs to the dock. */
|
|
839
1077
|
const floor = () => Math.max(1, size().rows - DOCK_ROWS);
|
|
840
1078
|
|
|
1079
|
+
/**
|
|
1080
|
+
* Follow the transcript's cursor through a chunk.
|
|
1081
|
+
*
|
|
1082
|
+
* The terminal is tracking the same thing in its saved cursor and does it
|
|
1083
|
+
* perfectly, so this exists for one moment only: a resize, after which that
|
|
1084
|
+
* saved position no longer means anything and the dock has to name a row to
|
|
1085
|
+
* carry on from. It used to name the floor, which is why a session that had
|
|
1086
|
+
* printed ten lines onto a fifty-row window jumped to the bottom of it and
|
|
1087
|
+
* scrolled the banner away to say the eleventh.
|
|
1088
|
+
*
|
|
1089
|
+
* Only the finals that move the cursor are read. The erases and the colour
|
|
1090
|
+
* runs that make up most of a transcript leave it exactly where it was, and
|
|
1091
|
+
* a sequence this does not recognise is likelier to be one of those than a
|
|
1092
|
+
* jump -- so the unknown case is "no movement" rather than a guess.
|
|
1093
|
+
*/
|
|
1094
|
+
function advance(chunk) {
|
|
1095
|
+
if (!at) return;
|
|
1096
|
+
const { columns } = size();
|
|
1097
|
+
const limit = floor();
|
|
1098
|
+
let { row, column } = at;
|
|
1099
|
+
// At the floor the region scrolls under the cursor rather than moving it:
|
|
1100
|
+
// everything already printed goes up a row and the cursor stays put.
|
|
1101
|
+
const down = () => {
|
|
1102
|
+
if (row < limit) row += 1;
|
|
1103
|
+
};
|
|
1104
|
+
const text = String(chunk);
|
|
1105
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
1106
|
+
const ch = text[i];
|
|
1107
|
+
if (ch === "\u001b") {
|
|
1108
|
+
const next = text[i + 1];
|
|
1109
|
+
if (next === "[") {
|
|
1110
|
+
let j = i + 2;
|
|
1111
|
+
while (j < text.length && !/[@-~]/.test(text[j])) j += 1;
|
|
1112
|
+
const params = text.slice(i + 2, j);
|
|
1113
|
+
const final = text[j];
|
|
1114
|
+
if (final === "H" || final === "f") {
|
|
1115
|
+
const [r, c] = params.split(";");
|
|
1116
|
+
row = Math.min(Math.max(Number(r) || 1, 1), limit);
|
|
1117
|
+
column = Math.min(Math.max(Number(c) || 1, 1), columns);
|
|
1118
|
+
} else if (final === "G") {
|
|
1119
|
+
column = Math.min(Math.max(Number(params) || 1, 1), columns);
|
|
1120
|
+
}
|
|
1121
|
+
i = j;
|
|
1122
|
+
continue;
|
|
1123
|
+
}
|
|
1124
|
+
if (next === "]") {
|
|
1125
|
+
while (i < text.length && text[i] !== "\u0007") i += 1;
|
|
1126
|
+
continue;
|
|
1127
|
+
}
|
|
1128
|
+
i += 1;
|
|
1129
|
+
continue;
|
|
1130
|
+
}
|
|
1131
|
+
if (ch === "\r") {
|
|
1132
|
+
column = 1;
|
|
1133
|
+
continue;
|
|
1134
|
+
}
|
|
1135
|
+
if (ch === "\n") {
|
|
1136
|
+
// ONLCR: stdout to a terminal turns a bare newline into CR+LF, so the
|
|
1137
|
+
// column goes back to one. Modelling it as a pure index down would
|
|
1138
|
+
// stair-step every line of the transcript to the right.
|
|
1139
|
+
column = 1;
|
|
1140
|
+
down();
|
|
1141
|
+
continue;
|
|
1142
|
+
}
|
|
1143
|
+
if (ch === "\u0007") continue;
|
|
1144
|
+
// Deferred wrap, the way a terminal does it: the glyph in the last column
|
|
1145
|
+
// leaves the cursor on that column, and the *next* one moves the line on.
|
|
1146
|
+
if (column > columns) {
|
|
1147
|
+
column = 1;
|
|
1148
|
+
down();
|
|
1149
|
+
}
|
|
1150
|
+
column += 1;
|
|
1151
|
+
}
|
|
1152
|
+
at = { row, column };
|
|
1153
|
+
}
|
|
1154
|
+
|
|
841
1155
|
/**
|
|
842
1156
|
* Is there a window here to dock in at all?
|
|
843
1157
|
*
|
|
@@ -983,16 +1297,32 @@ export function createDock({
|
|
|
983
1297
|
return {
|
|
984
1298
|
live,
|
|
985
1299
|
|
|
986
|
-
/**
|
|
987
|
-
|
|
1300
|
+
/**
|
|
1301
|
+
* Reserve the bottom rows and remember where the transcript is.
|
|
1302
|
+
*
|
|
1303
|
+
* `row` is where the transcript has got to on a screen the caller knows the
|
|
1304
|
+
* state of -- a session that has just cleared the screen and printed a
|
|
1305
|
+
* banner knows exactly that. Without it the screen is assumed to be full,
|
|
1306
|
+
* which is the honest reading when the dock is coming back up after
|
|
1307
|
+
* something else owned the terminal: scroll to make room for the furniture
|
|
1308
|
+
* and carry on at the floor.
|
|
1309
|
+
*/
|
|
1310
|
+
enable({ row } = {}) {
|
|
988
1311
|
if (!live || open) return;
|
|
989
1312
|
open = true;
|
|
990
1313
|
listen();
|
|
991
1314
|
stream.write(SET_TITLE);
|
|
1315
|
+
// Whatever a previous life left here says nothing about this screen.
|
|
1316
|
+
at = null;
|
|
992
1317
|
if (!roomy()) return;
|
|
993
|
-
|
|
1318
|
+
if (row == null) {
|
|
1319
|
+
stream.write("\n".repeat(DOCK_ROWS));
|
|
1320
|
+
at = { row: floor(), column: 1 };
|
|
1321
|
+
} else {
|
|
1322
|
+
at = { row: Math.min(Math.max(1, row), floor()), column: 1 };
|
|
1323
|
+
}
|
|
994
1324
|
stream.write(`\u001b[1;${floor()}r`);
|
|
995
|
-
stream.write(`\u001b[${
|
|
1325
|
+
stream.write(`\u001b[${at.row};${at.column}H`);
|
|
996
1326
|
stream.write("\u001b7");
|
|
997
1327
|
frame();
|
|
998
1328
|
},
|
|
@@ -1023,12 +1353,18 @@ export function createDock({
|
|
|
1023
1353
|
*/
|
|
1024
1354
|
write(chunk) {
|
|
1025
1355
|
if (!open || !roomy()) {
|
|
1356
|
+
// Nothing is following the cursor down a screen the dock does not own,
|
|
1357
|
+
// so whatever was tracked is now a guess. Said rather than kept: a
|
|
1358
|
+
// window that grows back is better off assuming the transcript filled
|
|
1359
|
+
// it than resuming at a row from before it went blind.
|
|
1360
|
+
at = null;
|
|
1026
1361
|
stream.write(chunk);
|
|
1027
1362
|
return;
|
|
1028
1363
|
}
|
|
1029
1364
|
// One write rather than three: the restore, the chunk and the save are a
|
|
1030
1365
|
// single sequence to the terminal, so nothing can land between them.
|
|
1031
1366
|
stream.write(`\u001b8${chunk}\u001b7`);
|
|
1367
|
+
advance(chunk);
|
|
1032
1368
|
repaint();
|
|
1033
1369
|
},
|
|
1034
1370
|
|
|
@@ -1096,10 +1432,18 @@ export function createDock({
|
|
|
1096
1432
|
painted = null;
|
|
1097
1433
|
if (!roomy()) return;
|
|
1098
1434
|
stream.write(`\u001b[1;${floor()}r`);
|
|
1099
|
-
//
|
|
1100
|
-
//
|
|
1101
|
-
//
|
|
1102
|
-
|
|
1435
|
+
// Where the transcript actually is, clamped into the new window -- not
|
|
1436
|
+
// the floor, which is what this used to say. Naming the floor meant every
|
|
1437
|
+
// drag of a window edge moved the transcript to the bottom of it, so the
|
|
1438
|
+
// next line printed scrolled the region and the banner climbed a row
|
|
1439
|
+
// towards the top and off. On a session that had barely started, that is
|
|
1440
|
+
// the whole screen going blank above a reply pinned to the bottom.
|
|
1441
|
+
const { columns } = size();
|
|
1442
|
+
at = {
|
|
1443
|
+
row: Math.min(Math.max(1, at?.row ?? floor()), floor()),
|
|
1444
|
+
column: Math.min(Math.max(1, at?.column ?? 1), columns),
|
|
1445
|
+
};
|
|
1446
|
+
stream.write(`\u001b[${at.row};${at.column}H`);
|
|
1103
1447
|
stream.write("\u001b7");
|
|
1104
1448
|
frame();
|
|
1105
1449
|
},
|
|
@@ -1122,9 +1466,35 @@ export function createDock({
|
|
|
1122
1466
|
}
|
|
1123
1467
|
if (!open) return;
|
|
1124
1468
|
open = false;
|
|
1469
|
+
at = null;
|
|
1125
1470
|
const { rows } = size();
|
|
1126
1471
|
stream.write("\u001b[r");
|
|
1127
|
-
|
|
1472
|
+
// Every row the furniture owned, not just the last one. On the alternate
|
|
1473
|
+
// screen clearing one row was enough because leaving the buffer threw the
|
|
1474
|
+
// whole screen away; on the shell's own screen the chrome is real output,
|
|
1475
|
+
// and a composer box and a status strip left sitting under the final
|
|
1476
|
+
// prompt are the last thing the session shows.
|
|
1477
|
+
//
|
|
1478
|
+
// Guarded on having painted, because that is the only thing that says
|
|
1479
|
+
// these rows were ever the dock's. In a window too short to dock in
|
|
1480
|
+
// nothing is drawn at all -- the session is a plain prompt -- and erasing
|
|
1481
|
+
// four rows of a five-row window there would take somebody's transcript
|
|
1482
|
+
// with it. Clamped to the live window for the same reason `resize` is:
|
|
1483
|
+
// after a shrink, `painted.rows` names rows that no longer exist.
|
|
1484
|
+
if (painted) {
|
|
1485
|
+
const bottom = Math.min(painted.rows, rows);
|
|
1486
|
+
const top = Math.max(1, bottom - DOCK_ROWS + 1);
|
|
1487
|
+
for (let row = top; row <= bottom; row += 1) {
|
|
1488
|
+
stream.write(`\u001b[${row};1H\u001b[2K`);
|
|
1489
|
+
}
|
|
1490
|
+
// Carry on immediately below the transcript rather than at the bottom
|
|
1491
|
+
// of the window, so the resume footer and the shell prompt follow the
|
|
1492
|
+
// conversation instead of a gap where the dock used to be.
|
|
1493
|
+
stream.write(`\u001b[${top};1H`);
|
|
1494
|
+
} else {
|
|
1495
|
+
stream.write(`\u001b[${rows};1H\u001b[2K`);
|
|
1496
|
+
}
|
|
1497
|
+
painted = null;
|
|
1128
1498
|
stream.write(CLEAR_TITLE);
|
|
1129
1499
|
},
|
|
1130
1500
|
};
|
|
@@ -1626,10 +1996,27 @@ export async function agentCommand(commandArgs = [], { authenticate = authentica
|
|
|
1626
1996
|
if (args.has("--print") || args.has("-p") || !process.stdin.isTTY) return;
|
|
1627
1997
|
}
|
|
1628
1998
|
|
|
1629
|
-
// The session's
|
|
1630
|
-
//
|
|
1631
|
-
//
|
|
1632
|
-
|
|
1999
|
+
// The session shares the shell's screen rather than taking one of its own.
|
|
2000
|
+
//
|
|
2001
|
+
// It used to enter the alternate screen, the way a full-screen app does, and
|
|
2002
|
+
// that cost the one thing a chat transcript needs: the alternate buffer keeps
|
|
2003
|
+
// no history, so a reply that ran past the top of the window was not scrolled
|
|
2004
|
+
// away, it was discarded. There was nothing to scroll back to in any
|
|
2005
|
+
// terminal, because the buffer holds exactly one screen.
|
|
2006
|
+
//
|
|
2007
|
+
// On the shell's own screen the transcript is ordinary output, so the wheel,
|
|
2008
|
+
// Page Up, find and select-to-copy all work on a conversation the way they
|
|
2009
|
+
// work on any other command's output. Keeping the composer docked costs none
|
|
2010
|
+
// of it: the dock's scrolling region starts at row 1, and a region whose top
|
|
2011
|
+
// margin is the first row is the case where a terminal pushes the lines it
|
|
2012
|
+
// scrolls past into the scrollback instead of dropping them.
|
|
2013
|
+
//
|
|
2014
|
+
// What this gives up is what the alternate screen was for -- a new session
|
|
2015
|
+
// can now be scrolled back into a finished one. That is the trade every
|
|
2016
|
+
// other command makes with its own output, and it is the cheaper way round:
|
|
2017
|
+
// scrolling into an old session is something a reader can see and scroll out
|
|
2018
|
+
// of, and a transcript that cannot be scrolled at all is not.
|
|
2019
|
+
const interactive = Boolean(process.stdout.isTTY);
|
|
1633
2020
|
|
|
1634
2021
|
// Watching before the screen is taken, not after: the handlers have to be up
|
|
1635
2022
|
// for every moment the terminal is in a state somebody else has to live with.
|
|
@@ -1643,15 +2030,11 @@ export async function agentCommand(commandArgs = [], { authenticate = authentica
|
|
|
1643
2030
|
// A conversation whose id never arrived stringifies to "undefined", and a
|
|
1644
2031
|
// resume line naming that is worse than no resume line.
|
|
1645
2032
|
const id = String(conversation?.id || "");
|
|
1646
|
-
if (!
|
|
2033
|
+
if (!interactive || !id || id === "undefined") return "";
|
|
1647
2034
|
return `${paint.dim("Resume this session with:")}\n ${cliInvocation()} --conversation ${id}\n`;
|
|
1648
2035
|
},
|
|
1649
2036
|
}).watch();
|
|
1650
2037
|
|
|
1651
|
-
if (fullScreen) process.stdout.write(`${ENTER_ALT}${CLEAR_SCREEN}`);
|
|
1652
|
-
|
|
1653
|
-
process.stdout.write(banner({ paint, workspace, backend: backendUrl(args), conversation }));
|
|
1654
|
-
|
|
1655
2038
|
const dock = createDock({
|
|
1656
2039
|
stream: process.stdout,
|
|
1657
2040
|
paint,
|
|
@@ -1660,7 +2043,15 @@ export async function agentCommand(commandArgs = [], { authenticate = authentica
|
|
|
1660
2043
|
const prompt = `${paint.caret("\u276f")} `;
|
|
1661
2044
|
const rl = createInterface({ input: process.stdin, output: process.stdout, prompt });
|
|
1662
2045
|
dock.attach(rl);
|
|
2046
|
+
// No start row, because there is no cleared screen to start at the top of any
|
|
2047
|
+
// more. Told nothing, the dock makes room by scrolling the shell up by its
|
|
2048
|
+
// own height and carries on at the floor -- which keeps whatever was on the
|
|
2049
|
+
// screen before, the prompt the session was launched from included, instead
|
|
2050
|
+
// of erasing it. The banner still goes through the dock rather than around
|
|
2051
|
+
// it, which is what keeps the two in step.
|
|
1663
2052
|
dock.enable();
|
|
2053
|
+
|
|
2054
|
+
dock.write(banner({ paint, workspace, backend: backendUrl(args), conversation }));
|
|
1664
2055
|
const onResize = () => dock.resize();
|
|
1665
2056
|
process.stdout.on("resize", onResize);
|
|
1666
2057
|
|
package/bin/connect/pairing.js
CHANGED
|
@@ -51,6 +51,32 @@ async function verifyKey(args, apiKey) {
|
|
|
51
51
|
return { pairCode: "", pairingId: "", stale: result.status_code === 401 };
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Redeem the pair code from this terminal.
|
|
56
|
+
*
|
|
57
|
+
* The link used to be closed by the agent calling `preman_status`, but the MCP
|
|
58
|
+
* server is a pure bridge now and defines no such tool, so nothing has made this
|
|
59
|
+
* call since. This terminal holds both halves anyway — the key and the code it
|
|
60
|
+
* just minted — so it closes the link itself rather than waiting for a call that
|
|
61
|
+
* no longer happens. Failure is not fatal: the config is already written.
|
|
62
|
+
*/
|
|
63
|
+
export async function redeemPairCode(args, agent, apiKey, pairCode) {
|
|
64
|
+
if (!pairCode) return false;
|
|
65
|
+
const result = await callBackendJson(args, "POST", "/workbench/coding-agent/heartbeat", {
|
|
66
|
+
token: apiKey,
|
|
67
|
+
json: {
|
|
68
|
+
pair_code: pairCode,
|
|
69
|
+
agent: agent.id,
|
|
70
|
+
project_path: process.cwd(),
|
|
71
|
+
source: "premanmcp",
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
if (!result.ok) {
|
|
75
|
+
process.stdout.write(`Note: could not close the link (${result.status_code}).\n`);
|
|
76
|
+
}
|
|
77
|
+
return Boolean(result.ok);
|
|
78
|
+
}
|
|
79
|
+
|
|
54
80
|
/** Whichever of the two ways to learn the backend still accepts this key. */
|
|
55
81
|
export async function checkKeyAndPair(args, agent, apiKey) {
|
|
56
82
|
return args.has("--no-pair")
|
package/bin/connect.js
CHANGED
|
@@ -49,7 +49,7 @@ import {
|
|
|
49
49
|
renderCodexToml,
|
|
50
50
|
verifyWrittenConfig,
|
|
51
51
|
} from "./connect/configs.js";
|
|
52
|
-
import { checkKeyAndPair, refreshStaleCredentials } from "./connect/pairing.js";
|
|
52
|
+
import { checkKeyAndPair, redeemPairCode, refreshStaleCredentials } from "./connect/pairing.js";
|
|
53
53
|
import {
|
|
54
54
|
autoCheckIn,
|
|
55
55
|
lastLine,
|
|
@@ -258,6 +258,8 @@ export async function connectCommand(commandArgs) {
|
|
|
258
258
|
);
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
+
const redeemed = await redeemPairCode(args, agent, apiKey, pairCode);
|
|
262
|
+
|
|
261
263
|
// A non-interactive run can still be handed the credential up front, so this
|
|
262
264
|
// stays reachable; the prompt inside only fires when there is a TTY, and by
|
|
263
265
|
// then onboarding is done.
|
|
@@ -273,6 +275,7 @@ export async function connectCommand(commandArgs) {
|
|
|
273
275
|
serverConfig,
|
|
274
276
|
projectInstall,
|
|
275
277
|
pairingId,
|
|
278
|
+
preLinked: redeemed,
|
|
276
279
|
});
|
|
277
280
|
// The agent goes back to the caller because `onboard` runs steps after this one
|
|
278
281
|
// that need to know which agent to drive, and asking twice is a question we
|
|
@@ -394,7 +397,7 @@ async function establishCheckIn(
|
|
|
394
397
|
args,
|
|
395
398
|
agent,
|
|
396
399
|
apiKey,
|
|
397
|
-
{ serverName, written, serverConfig, projectInstall = false, pairingId = "" }
|
|
400
|
+
{ serverName, written, serverConfig, projectInstall = false, pairingId = "", preLinked = false }
|
|
398
401
|
) {
|
|
399
402
|
const notes = [];
|
|
400
403
|
// Both set only when this directory redirects the agent: where it can be
|
|
@@ -402,6 +405,9 @@ async function establishCheckIn(
|
|
|
402
405
|
let elsewhere = null;
|
|
403
406
|
let blockedHere = "";
|
|
404
407
|
const done = (linked) => ({ linked, blockedHere });
|
|
408
|
+
// Already linked by the terminal that wrote the config: nothing to self-test
|
|
409
|
+
// into, no window to open, and no check-in to wait for.
|
|
410
|
+
if (preLinked) return done(true);
|
|
405
411
|
const ticker = pollTicker();
|
|
406
412
|
const say = (text) => {
|
|
407
413
|
ticker.end();
|
package/bin/eval.js
CHANGED
|
@@ -470,6 +470,18 @@ export function runDir(cwd, job) {
|
|
|
470
470
|
return path.join(cwd, "artifacts", "results", String(job.suite), String(job.run));
|
|
471
471
|
}
|
|
472
472
|
|
|
473
|
+
/**
|
|
474
|
+
* Where the suite's own output lands — the parent of every run directory.
|
|
475
|
+
*
|
|
476
|
+
* Only ever joined with a name from `SUITE_ARTIFACTS`. The same directory also
|
|
477
|
+
* holds assert-ai's versioned `artifacts/<stage>/v0001/` tree, which nothing
|
|
478
|
+
* reads back and which a device walking the directory would start uploading;
|
|
479
|
+
* naming the files explicitly is what keeps that out.
|
|
480
|
+
*/
|
|
481
|
+
export function suiteDir(cwd, job) {
|
|
482
|
+
return path.join(cwd, "artifacts", "results", String(job.suite));
|
|
483
|
+
}
|
|
484
|
+
|
|
473
485
|
/**
|
|
474
486
|
* The four stages as one activity list, current stage active, earlier ones done.
|
|
475
487
|
*
|
|
@@ -522,6 +534,21 @@ export const RUN_ARTIFACTS = [
|
|
|
522
534
|
"artifacts.json",
|
|
523
535
|
];
|
|
524
536
|
|
|
537
|
+
/**
|
|
538
|
+
* The files that describe the bank rather than one run.
|
|
539
|
+
*
|
|
540
|
+
* These are the cases themselves and the rubric they are scored against, and
|
|
541
|
+
* they live one level up from the run directory because every run of the suite
|
|
542
|
+
* shares them. Without them the store holds verdicts with no record of what was
|
|
543
|
+
* asked: the dashboard can say "case 3 failed" and cannot say what case 3 was.
|
|
544
|
+
*
|
|
545
|
+
* No scope is sent with an upload. The server derives it from the filename
|
|
546
|
+
* (`eval_runner_artifacts.scope_of`), files these under the suite rather than
|
|
547
|
+
* the run, and applies its own write-once rule — so a second run of the same
|
|
548
|
+
* suite offers them again and the server declines to rewrite history.
|
|
549
|
+
*/
|
|
550
|
+
export const SUITE_ARTIFACTS = ["taxonomy.json", "test_set.jsonl"];
|
|
551
|
+
|
|
525
552
|
/**
|
|
526
553
|
* Upload the artifacts that changed since last time.
|
|
527
554
|
*
|
|
@@ -536,10 +563,15 @@ export const RUN_ARTIFACTS = [
|
|
|
536
563
|
* success and the next tick retries.
|
|
537
564
|
*/
|
|
538
565
|
export async function syncArtifacts(job, cwd, sent, { call, log = () => {}, lease } = {}) {
|
|
539
|
-
|
|
566
|
+
// Both sets on the same tick, not the suite files at the end: `systematize`
|
|
567
|
+
// and `test_set` finish before the first case does, so a run that dies
|
|
568
|
+
// halfway would otherwise publish scores for cases it never published.
|
|
569
|
+
const sources = [
|
|
570
|
+
...RUN_ARTIFACTS.map((name) => [name, path.join(runDir(cwd, job), name)]),
|
|
571
|
+
...SUITE_ARTIFACTS.map((name) => [name, path.join(suiteDir(cwd, job), name)]),
|
|
572
|
+
];
|
|
540
573
|
let uploaded = 0;
|
|
541
|
-
for (const name of
|
|
542
|
-
const file = path.join(dir, name);
|
|
574
|
+
for (const [name, file] of sources) {
|
|
543
575
|
let mark;
|
|
544
576
|
try {
|
|
545
577
|
const stat = statSync(file);
|
|
@@ -561,7 +593,18 @@ export async function syncArtifacts(job, cwd, sent, { call, log = () => {}, leas
|
|
|
561
593
|
form.set("lease_token", lease);
|
|
562
594
|
form.set("name", name);
|
|
563
595
|
form.set("file", new Blob([body]), name);
|
|
564
|
-
|
|
596
|
+
// A refused upload and an unsendable one are the same situation to this
|
|
597
|
+
// loop -- the file is still on disk, unrecorded in `sent`, and the next
|
|
598
|
+
// tick will offer it again. They are not the same to `callBackendJson`,
|
|
599
|
+
// which answers the first and *throws* the second, so without this a
|
|
600
|
+
// backend that blinks mid-run takes the run's process down with it.
|
|
601
|
+
let result;
|
|
602
|
+
try {
|
|
603
|
+
result = await call(`/workbench/coding-agent/local-runner/evals/${job.id}/artifacts`, form);
|
|
604
|
+
} catch (error) {
|
|
605
|
+
log(`upload of ${name} could not be sent: ${error.message}; will retry`);
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
565
608
|
if (result.status_code === 409) return { uploaded, lost: true };
|
|
566
609
|
if (!result.ok) {
|
|
567
610
|
log(`upload of ${name} answered ${result.status_code}; will retry`);
|
|
@@ -629,6 +672,39 @@ export function readProgress(cwd, job) {
|
|
|
629
672
|
* Returns "" when there is nothing wrong, so the caller can tell "fine" from
|
|
630
673
|
* "unreadable".
|
|
631
674
|
*/
|
|
675
|
+
/**
|
|
676
|
+
* What this run alone did, out of counters that belong to the whole invocation.
|
|
677
|
+
*
|
|
678
|
+
* The adapter is opened once and every run in the batch shares it, so its
|
|
679
|
+
* counters are lifetime totals -- which is the honest thing for them to be, and
|
|
680
|
+
* the wrong thing to hand a guard asking about one run. Read as totals they
|
|
681
|
+
* fail a run for the run before it: a batch where the first eval made one tool
|
|
682
|
+
* call and the second legitimately made none reported "the agent made 1 tool
|
|
683
|
+
* call(s) and none of them reached the transcript" against the second, quoting
|
|
684
|
+
* the first one's number. The same arithmetic hides the opposite fault --
|
|
685
|
+
* `reached` never returns to zero after any run reaches the agent, so a later
|
|
686
|
+
* run where every single turn failed cannot be caught at all.
|
|
687
|
+
*
|
|
688
|
+
* Subtracting rather than resetting: `adapterStats()` is the adapter's own
|
|
689
|
+
* lifetime record and other readers rely on it being exactly that. Zeroing
|
|
690
|
+
* shared state from inside one run would also be wrong the day two overlap.
|
|
691
|
+
*
|
|
692
|
+
* `lastFailure` is passed through rather than differenced. It is a string, and
|
|
693
|
+
* the most recent one is the useful one whenever `failures` moved at all.
|
|
694
|
+
*/
|
|
695
|
+
export function statsSince(baseline, stats) {
|
|
696
|
+
if (!stats) return stats;
|
|
697
|
+
if (!baseline) return stats;
|
|
698
|
+
const since = (key) => Math.max(0, Number(stats[key] || 0) - Number(baseline[key] || 0));
|
|
699
|
+
return {
|
|
700
|
+
...stats,
|
|
701
|
+
turns: since("turns"),
|
|
702
|
+
reached: since("reached"),
|
|
703
|
+
failures: since("failures"),
|
|
704
|
+
toolCalls: since("toolCalls"),
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
|
|
632
708
|
export function missingToolEvidence(dir, stats) {
|
|
633
709
|
const forwarded = Number(stats?.toolCalls);
|
|
634
710
|
if (!Number.isFinite(forwarded) || forwarded <= 0) return "";
|
|
@@ -903,6 +979,10 @@ export async function executeEvalRun(
|
|
|
903
979
|
// Before the harness starts, not after it ends: a run that dies mid-way would
|
|
904
980
|
// otherwise leave its last steps to appear under whichever run came next.
|
|
905
981
|
stepFeed.reset();
|
|
982
|
+
// Same reason, for the counters that cannot be reset because the adapter they
|
|
983
|
+
// belong to is shared with every other run in this batch. See `statsSince`.
|
|
984
|
+
const adapterBaseline = adapterStats();
|
|
985
|
+
const baselineAt = adapterBaseline ? { ...adapterBaseline } : null;
|
|
906
986
|
|
|
907
987
|
const timeout = Math.min(RUN_TIMEOUT_MS, RUN_TIMEOUT_CAP_MS);
|
|
908
988
|
const child = spawn(python.command, [...python.argv, HARNESS_PATH, "run", "--config", laid.configPath], {
|
|
@@ -965,6 +1045,15 @@ export async function executeEvalRun(
|
|
|
965
1045
|
// before it spends an uplink on artifacts nobody will accept.
|
|
966
1046
|
const synced = await syncArtifacts(job, laid.cwd, sent, { call, log, lease });
|
|
967
1047
|
if (synced.lost) return lose("lease rejected an upload");
|
|
1048
|
+
} catch (error) {
|
|
1049
|
+
// Nothing this tick reports is worth the run for. It is liveness: the
|
|
1050
|
+
// stages and steps somebody watching sees, and a copy of artifacts the
|
|
1051
|
+
// final sync sends again anyway. Nobody awaits this callback, so an
|
|
1052
|
+
// escaping rejection is an *unhandled* one, and Node's answer to that
|
|
1053
|
+
// is to end the process -- killing a harness mid-stage, skipping the
|
|
1054
|
+
// completion callback, and leaving the run to sit until its lease
|
|
1055
|
+
// lapses. A logged tick and a live run is the better trade.
|
|
1056
|
+
log(`eval ${job.id}: progress tick failed (${error.message}); the run continues`);
|
|
968
1057
|
} finally {
|
|
969
1058
|
ticking = false;
|
|
970
1059
|
}
|
|
@@ -1022,7 +1111,7 @@ export async function executeEvalRun(
|
|
|
1022
1111
|
// Turns are counted rather than failures, because a run where the agent
|
|
1023
1112
|
// answered some of the time is a real measurement of a flaky agent and
|
|
1024
1113
|
// deciding otherwise here would throw away the finding.
|
|
1025
|
-
const stats = adapterStats();
|
|
1114
|
+
const stats = statsSince(baselineAt, adapterStats());
|
|
1026
1115
|
// `reached` rather than `turns`: the adapter counts a turn when it starts
|
|
1027
1116
|
// one, so a run where every turn failed has as many turns as cases.
|
|
1028
1117
|
if (stats && (stats.reached ?? stats.turns) === 0 && stats.failures > 0) {
|
package/bin/runner.js
CHANGED
|
@@ -869,6 +869,19 @@ export async function runnerLoop(
|
|
|
869
869
|
try {
|
|
870
870
|
if (kind === "eval") await runLeasedEval(args, state, job, { log, headless });
|
|
871
871
|
else await executeJob(args, state, job, { log, fullAccess });
|
|
872
|
+
} catch (error) {
|
|
873
|
+
// Attribution, not recovery. Both executors report their own
|
|
874
|
+
// failures and return; a throw escaping one is a bug in it, and
|
|
875
|
+
// letting it reach the stream's catch below labels that bug
|
|
876
|
+
// `stream dropped` -- blaming the network for something that
|
|
877
|
+
// happened on this machine, which is the wrong place to look and
|
|
878
|
+
// the reason the last one took all evening to find.
|
|
879
|
+
//
|
|
880
|
+
// Counted as an attempt below rather than halting the batch. A
|
|
881
|
+
// crash that stopped the loop silently is what the caller then
|
|
882
|
+
// reports as a finished run, which is the failure this is here
|
|
883
|
+
// to stop being invisible.
|
|
884
|
+
log(`${kind} ${job.id} failed unexpectedly: ${error?.stack || error.message}`);
|
|
872
885
|
} finally {
|
|
873
886
|
busy = false;
|
|
874
887
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "premanmcp",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.8",
|
|
4
4
|
"description": "PreMan CLI and stdio proxy for PreMan's hosted MCP server",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
"test": "npm run build && npm run test:proxy",
|
|
16
16
|
"test:proxy": "node --test scripts/smoke-proxy.mjs",
|
|
17
17
|
"test:connect": "node scripts/smoke-connect.mjs",
|
|
18
|
+
"test:connect-heartbeat": "node scripts/smoke-connect-heartbeat.mjs",
|
|
18
19
|
"test:node": "node --test --test-timeout=90000 scripts/smoke-account.mjs scripts/smoke-agent-session.mjs scripts/smoke-agent-dock.mjs scripts/smoke-terminal-lifecycle.mjs scripts/smoke-cli-entrypoint.mjs scripts/smoke-cli-update.mjs scripts/smoke-launcher-config.mjs scripts/smoke-runner.mjs scripts/smoke-eval.mjs scripts/smoke-eval-behaviors.mjs scripts/smoke-repo-config.mjs scripts/smoke-onboard.mjs scripts/smoke-onboard-opening.mjs scripts/smoke-local-detect.mjs scripts/smoke-agent-target.mjs scripts/smoke-prepush-hook.mjs scripts/smoke-cli-identity.mjs scripts/smoke-runner-heartbeat.mjs scripts/smoke-verify-prepush.mjs scripts/smoke-push-diff.mjs scripts/smoke-progress-reporter.mjs scripts/smoke-verify-plan.mjs scripts/smoke-install-desktop.mjs scripts/smoke-desktop-session.mjs scripts/smoke-api-tools.mjs scripts/smoke-tests-workbench.mjs scripts/smoke-bin-scope.mjs scripts/smoke-shared-errors.mjs scripts/smoke-process-title.mjs scripts/smoke-predict.mjs",
|
|
19
20
|
"test:dmg": "node --test --test-timeout=300000 scripts/smoke-install-desktop-volume.mjs"
|
|
20
21
|
},
|