pi-gauntlet 8.4.4 → 8.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/extensions/lib/tool-result-warnings.test.ts +105 -0
- package/extensions/lib/tool-result-warnings.ts +35 -0
- package/extensions/phase-tracker.test.ts +135 -26
- package/extensions/phase-tracker.ts +12 -26
- package/extensions/telemetry.test.ts +68 -3
- package/extensions/telemetry.ts +8 -4
- package/extensions/test-support/merge-tool-result.mjs +9 -0
- package/extensions/verify-before-ship.test.ts +72 -5
- package/extensions/verify-before-ship.ts +12 -20
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v8.4.5 - 2026-11-08
|
|
4
|
+
|
|
5
|
+
- Guard, ship-verification, and post-ship spec warnings preserve tool `structuredContent`; nested-call warnings reach the parent codemode result once without changing the child's value or re-arming warn-once guards. (#133)
|
|
6
|
+
|
|
3
7
|
## v8.4.4 - 2026-11-08
|
|
4
8
|
|
|
5
9
|
- Block scripted `subagent` dispatches in every entered gauntlet phase and exclude nested results from closure latch, fix-round counter, and cadence updates. Use direct tool calls instead; disable with `closureReview.enforce: false`. The configuration guide documents codemode `on` support with direct-only dispatch, why `only` is unsupported in an entered flow, and how to read literal skill dispatch examples (#132, #134).
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { test } from "node:test";
|
|
3
|
+
import { createPendingWarnings, prependWarning } from "./tool-result-warnings.ts";
|
|
4
|
+
import { mergeToolResult } from "../test-support/merge-tool-result.mjs";
|
|
5
|
+
|
|
6
|
+
for (const [name, toolName, structuredContent] of [
|
|
7
|
+
["bash payload", "bash", { output: "ok", exit_code: 0 }],
|
|
8
|
+
["write null", "write", null],
|
|
9
|
+
["edit false", "edit", false],
|
|
10
|
+
["zero", "codemode", 0],
|
|
11
|
+
["empty string", "read", ""],
|
|
12
|
+
["absent", "read", undefined],
|
|
13
|
+
] as const) {
|
|
14
|
+
test(`prependWarning preserves the runner-merged result (${name})`, () => {
|
|
15
|
+
const event = {
|
|
16
|
+
toolName, toolCallId: "c1", input: {}, isError: true,
|
|
17
|
+
...(structuredContent !== undefined ? { structuredContent } : {}),
|
|
18
|
+
content: [{ type: "text", text: "original" }, { type: "image", data: "AA==", mimeType: "image/png" }],
|
|
19
|
+
details: { sentinel: "details" }, usage: { sentinel: "usage" },
|
|
20
|
+
};
|
|
21
|
+
const original = structuredClone(event);
|
|
22
|
+
const ret = prependWarning(event as any, "warning");
|
|
23
|
+
assert.deepEqual(ret, { content: [{ type: "text", text: "warning" }, ...event.content], structuredContent });
|
|
24
|
+
assert.deepEqual(mergeToolResult(event, ret), { ...event, content: ret.content });
|
|
25
|
+
assert.deepEqual(event, original);
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
test("nested warning re-keys without changing the child or the fired latch, then delivers once", () => {
|
|
30
|
+
const store = createPendingWarnings();
|
|
31
|
+
const firedGuards = new Map([["guard", true]]);
|
|
32
|
+
const child = {
|
|
33
|
+
toolName: "bash", toolCallId: "c1/0", parentToolCallId: "c1", input: {}, isError: false,
|
|
34
|
+
content: [{ type: "text", text: "ok" }], structuredContent: { output: "ok", exit_code: 0 },
|
|
35
|
+
};
|
|
36
|
+
const original = structuredClone(child);
|
|
37
|
+
store.add(child.toolCallId, "warning");
|
|
38
|
+
const ret = store.deliver(child as any, store.take(child.toolCallId)!);
|
|
39
|
+
assert.equal(ret, undefined);
|
|
40
|
+
assert.deepEqual(mergeToolResult(child, ret), original);
|
|
41
|
+
assert.deepEqual(child, original);
|
|
42
|
+
assert.equal(firedGuards.get("guard"), true);
|
|
43
|
+
assert.equal(store.take(child.toolCallId), undefined);
|
|
44
|
+
|
|
45
|
+
const parent = { ...child, toolName: "codemode", toolCallId: "c1", parentToolCallId: undefined, structuredContent: { value: 0 } };
|
|
46
|
+
const text = store.take(parent.toolCallId)!;
|
|
47
|
+
assert.equal(text, "warning");
|
|
48
|
+
const delivered = store.deliver(parent as any, text)!;
|
|
49
|
+
assert.deepEqual(mergeToolResult(parent, delivered), { ...parent, content: [{ type: "text", text }, ...parent.content] });
|
|
50
|
+
assert.equal(store.take(parent.toolCallId), undefined);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("two nested guards reach one parent once in arrival order", () => {
|
|
54
|
+
const store = createPendingWarnings();
|
|
55
|
+
for (const [id, text] of [["c1/0", "first guard"], ["c1/1", "second guard"]]) {
|
|
56
|
+
store.add(id, text);
|
|
57
|
+
assert.equal(store.deliver({ toolCallId: id, parentToolCallId: "c1" } as any, store.take(id)!), undefined);
|
|
58
|
+
assert.equal(store.take(id), undefined);
|
|
59
|
+
}
|
|
60
|
+
const parent = { toolName: "codemode", toolCallId: "c1", isError: false, content: [], structuredContent: null };
|
|
61
|
+
const ret = store.deliver(parent as any, store.take("c1")!)!;
|
|
62
|
+
const merged = mergeToolResult(parent, ret);
|
|
63
|
+
assert.deepEqual(merged.content, [{ type: "text", text: "first guard\n\nsecond guard" }]);
|
|
64
|
+
assert.deepEqual(merged.structuredContent, parent.structuredContent);
|
|
65
|
+
assert.equal(store.take("c1"), undefined);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("depth-two nesting re-keys c1/0/2 -> c1/0 -> c1", () => {
|
|
69
|
+
const store = createPendingWarnings();
|
|
70
|
+
store.add("c1/0/2", "warning");
|
|
71
|
+
for (const [toolCallId, parentToolCallId] of [["c1/0/2", "c1/0"], ["c1/0", "c1"]]) {
|
|
72
|
+
const event = { toolName: "codemode", toolCallId, parentToolCallId, content: [], structuredContent: { value: 0 } };
|
|
73
|
+
const ret = store.deliver(event as any, store.take(toolCallId)!);
|
|
74
|
+
assert.equal(ret, undefined);
|
|
75
|
+
assert.deepEqual(mergeToolResult(event, ret), event);
|
|
76
|
+
assert.equal(store.take(toolCallId), undefined);
|
|
77
|
+
}
|
|
78
|
+
const parent = { toolCallId: "c1", content: [], structuredContent: { value: 0 } };
|
|
79
|
+
const ret = store.deliver(parent as any, store.take("c1")!);
|
|
80
|
+
const merged = mergeToolResult(parent, ret);
|
|
81
|
+
assert.deepEqual(merged.content, [{ type: "text", text: "warning" }]);
|
|
82
|
+
assert.deepEqual(merged.structuredContent, parent.structuredContent);
|
|
83
|
+
assert.equal(store.take("c1"), undefined);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("warning delivery tolerates missing content and preserves structuredContent", () => {
|
|
87
|
+
const store = createPendingWarnings();
|
|
88
|
+
const event = { toolName: "bash", toolCallId: "c1", isError: false, structuredContent: { output: "ok", exit_code: 0 } };
|
|
89
|
+
store.add(event.toolCallId, "warning");
|
|
90
|
+
const ret = store.deliver(event as any, store.take(event.toolCallId)!);
|
|
91
|
+
assert.deepEqual(mergeToolResult(event, ret), { ...event, content: [{ type: "text", text: "warning" }] });
|
|
92
|
+
assert.equal(store.take(event.toolCallId), undefined);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("add concatenates warnings and clear discards all entries", () => {
|
|
96
|
+
const store = createPendingWarnings();
|
|
97
|
+
store.add("c1", "first");
|
|
98
|
+
store.add("c1", "second");
|
|
99
|
+
assert.equal(store.take("c1"), "first\n\nsecond");
|
|
100
|
+
store.add("c1", "pending");
|
|
101
|
+
store.add("c2", "another");
|
|
102
|
+
store.clear();
|
|
103
|
+
assert.equal(store.take("c1"), undefined);
|
|
104
|
+
assert.equal(store.take("c2"), undefined);
|
|
105
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { ToolResultEvent, ToolResultEventResult } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
|
|
3
|
+
export function prependWarning(event: ToolResultEvent, text: string): ToolResultEventResult {
|
|
4
|
+
return {
|
|
5
|
+
content: [{ type: "text", text }, ...(event.content ?? [])],
|
|
6
|
+
// Pi drops structuredContent when a content replacement omits it.
|
|
7
|
+
structuredContent: event.structuredContent,
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function createPendingWarnings() {
|
|
12
|
+
const pending = new Map<string, string>();
|
|
13
|
+
const add = (id: string, text: string) => {
|
|
14
|
+
const prior = pending.get(id);
|
|
15
|
+
pending.set(id, prior ? `${prior}\n\n${text}` : text);
|
|
16
|
+
};
|
|
17
|
+
return {
|
|
18
|
+
add,
|
|
19
|
+
take(id: string): string | undefined {
|
|
20
|
+
const text = pending.get(id);
|
|
21
|
+
pending.delete(id);
|
|
22
|
+
return text;
|
|
23
|
+
},
|
|
24
|
+
deliver(event: ToolResultEvent, text: string): ToolResultEventResult | undefined {
|
|
25
|
+
if (event.parentToolCallId) {
|
|
26
|
+
add(event.parentToolCallId, text);
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
return prependWarning(event, text);
|
|
30
|
+
},
|
|
31
|
+
clear() {
|
|
32
|
+
pending.clear();
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
@@ -9,6 +9,7 @@ import { tmpdir } from "node:os";
|
|
|
9
9
|
import { isAbsolute, join } from "node:path";
|
|
10
10
|
import registerPhaseTracker from "./phase-tracker.ts";
|
|
11
11
|
import registerPlanTracker from "./plan-tracker.ts";
|
|
12
|
+
import { mergeToolResult } from "./test-support/merge-tool-result.mjs";
|
|
12
13
|
|
|
13
14
|
const originalSubagentDepth = process.env.PI_SUBAGENT_DEPTH;
|
|
14
15
|
process.env.PI_SUBAGENT_DEPTH = "0";
|
|
@@ -391,19 +392,105 @@ test("implement-write guard: silent when unarmed, when enforce is false, and in
|
|
|
391
392
|
}
|
|
392
393
|
});
|
|
393
394
|
|
|
394
|
-
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
395
|
+
for (const toolName of ["bash", "write", "edit"]) {
|
|
396
|
+
for (const nested of [false, true]) {
|
|
397
|
+
test(`brainstorm ${toolName} warning preserves results (${nested ? "nested" : "direct"}) and stays fired`, async () => {
|
|
398
|
+
const h = harness({ branch: [phaseResult("start", phases({ brainstorm: "in_progress" }))] });
|
|
399
|
+
await h.emit("session_start");
|
|
400
|
+
const toolCallId = nested ? "parent/0" : "direct";
|
|
401
|
+
const nesting = nested ? { parentToolCallId: "parent" } : {};
|
|
402
|
+
const call = {
|
|
403
|
+
toolName, toolCallId, ...nesting,
|
|
404
|
+
input: toolName === "bash" ? { command: "echo x > src/a.ts" } : { path: "src/a.ts" },
|
|
405
|
+
};
|
|
406
|
+
await h.emitEvent("tool_call", call);
|
|
407
|
+
const event = {
|
|
408
|
+
...writeResult(toolCallId), toolName, ...nesting, isError: true,
|
|
409
|
+
details: { sentinel: "guard details" }, usage: { sentinel: "guard usage" },
|
|
410
|
+
structuredContent: toolName === "bash" ? { output: "ok", exit_code: 0 } : toolName === "write" ? null : { edited: true },
|
|
411
|
+
};
|
|
412
|
+
const original = structuredClone(event);
|
|
413
|
+
const [ret] = await h.emitEvent("tool_result", event);
|
|
414
|
+
let delivered = event;
|
|
415
|
+
let merged = mergeToolResult(event, ret);
|
|
416
|
+
if (nested) {
|
|
417
|
+
assert.equal(ret, undefined);
|
|
418
|
+
assert.deepEqual(merged, original);
|
|
419
|
+
const parent = {
|
|
420
|
+
...event, toolName: "codemode", toolCallId: "parent", parentToolCallId: undefined,
|
|
421
|
+
content: [{ type: "text", text: "script value: 0" }], structuredContent: { value: 0 },
|
|
422
|
+
};
|
|
423
|
+
delivered = parent;
|
|
424
|
+
const [parentRet] = await h.emitEvent("tool_result", parent);
|
|
425
|
+
merged = mergeToolResult(parent, parentRet);
|
|
426
|
+
}
|
|
427
|
+
assert.deepEqual(event, original);
|
|
428
|
+
assert.match(merged.content[0].text, /Writing outside the spec directory/);
|
|
429
|
+
assert.equal(merged.content[0].text.split("Writing outside the spec directory").length - 1, 1);
|
|
430
|
+
assert.deepEqual(merged, { ...delivered, content: [merged.content[0], ...delivered.content] });
|
|
431
|
+
assert.equal((await h.emitEvent("tool_result", delivered))[0], undefined);
|
|
432
|
+
await h.emitEvent("tool_call", writeCall("later", "src/b.ts"));
|
|
433
|
+
assert.equal((await h.emitEvent("tool_result", writeResult("later")))[0], undefined);
|
|
434
|
+
});
|
|
405
435
|
}
|
|
406
|
-
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
for (const toolName of ["bash", "write", "edit"]) {
|
|
439
|
+
for (const nested of [false, true]) {
|
|
440
|
+
test(`implement-write and cadence warnings preserve ${toolName} results (${nested ? "nested" : "direct"})`, async () => {
|
|
441
|
+
const h = harness({ branch: implementBranch([subagentResult(["implementer"])]) });
|
|
442
|
+
await h.emit("session_start");
|
|
443
|
+
const nesting = nested ? { parentToolCallId: "parent" } : {};
|
|
444
|
+
const write = { ...writeCall("write", "src/a.ts"), toolName: toolName === "edit" ? "edit" : "write" };
|
|
445
|
+
for (const [index, call] of [write, commitCall("commit")].entries()) {
|
|
446
|
+
const toolCallId = nested ? `parent/${index}` : call.toolCallId;
|
|
447
|
+
await h.emitEvent("tool_call", { ...call, toolCallId, ...nesting });
|
|
448
|
+
// Deferred cadence delivery is keyed by call ID, not the result's tool name.
|
|
449
|
+
const event = { ...writeResult(toolCallId), toolName: index === 0 ? write.toolName : toolName, ...nesting, structuredContent: { exit_code: 0 } };
|
|
450
|
+
const [ret] = await h.emitEvent("tool_result", event);
|
|
451
|
+
if (nested) {
|
|
452
|
+
assert.equal(ret, undefined);
|
|
453
|
+
assert.deepEqual(mergeToolResult(event, ret), event);
|
|
454
|
+
} else {
|
|
455
|
+
const merged = mergeToolResult(event, ret);
|
|
456
|
+
assert.match(merged.content[0].text, index === 0 ? /Parent write/ : /no wave-reviewer observed/);
|
|
457
|
+
assert.deepEqual(merged, { ...event, content: [merged.content[0], ...event.content] });
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
if (nested) {
|
|
461
|
+
const parent = { ...commitResult("parent"), toolName: "codemode", structuredContent: { value: "done" } };
|
|
462
|
+
const [ret] = await h.emitEvent("tool_result", parent);
|
|
463
|
+
const merged = mergeToolResult(parent, ret);
|
|
464
|
+
assert.match(merged.content[0].text, /Parent write[\s\S]*no wave-reviewer observed/);
|
|
465
|
+
assert.equal(merged.content[0].text.split("Parent write").length - 1, 1);
|
|
466
|
+
assert.equal(merged.content[0].text.split("no wave-reviewer observed").length - 1, 1);
|
|
467
|
+
assert.deepEqual(merged, { ...parent, content: [merged.content[0], ...parent.content] });
|
|
468
|
+
assert.equal((await h.emitEvent("tool_result", parent))[0], undefined);
|
|
469
|
+
}
|
|
470
|
+
await h.emitEvent("tool_call", writeCall("later", "src/b.ts"));
|
|
471
|
+
assert.equal((await h.emitEvent("tool_result", writeResult("later")))[0], undefined);
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
for (const transition of ["reset", "session_start"]) {
|
|
477
|
+
test(`${transition} clears a deferred nested guard warning before the parent result`, async () => {
|
|
478
|
+
const h = harness({ branch: [phaseResult("start", phases({ brainstorm: "in_progress" }))] });
|
|
479
|
+
await h.emit("session_start");
|
|
480
|
+
await h.emitEvent("tool_call", { ...writeCall("parent/0", "src/a.ts"), parentToolCallId: "parent" });
|
|
481
|
+
assert.equal((await h.emitEvent("tool_result", { ...writeResult("parent/0"), parentToolCallId: "parent" }))[0], undefined);
|
|
482
|
+
if (transition === "reset") {
|
|
483
|
+
const tool = h.tools.find((t) => t.name === "phase_tracker")!;
|
|
484
|
+
await tool.execute("reset", { action: "reset" }, undefined, undefined, h.ctx);
|
|
485
|
+
} else {
|
|
486
|
+
await h.emit("session_start");
|
|
487
|
+
}
|
|
488
|
+
const parent = { ...writeResult("parent"), toolName: "codemode", structuredContent: { value: 0 } };
|
|
489
|
+
const [ret] = await h.emitEvent("tool_result", parent);
|
|
490
|
+
assert.equal(ret, undefined);
|
|
491
|
+
assert.deepEqual(mergeToolResult(parent, ret), parent);
|
|
492
|
+
});
|
|
493
|
+
}
|
|
407
494
|
|
|
408
495
|
test("brainstorm write confinement: a docs/specs write under the default dirs does not warn", async () => {
|
|
409
496
|
const priorDepth = process.env.PI_SUBAGENT_DEPTH;
|
|
@@ -1660,7 +1747,7 @@ test("nested dispatch: allowed without flow entry", async () => {
|
|
|
1660
1747
|
});
|
|
1661
1748
|
|
|
1662
1749
|
for (const malformed of [false, true]) {
|
|
1663
|
-
test(`nested dispatch: management passes and its
|
|
1750
|
+
test(`nested dispatch: management passes and its parent consumes queued warnings; malformed=${malformed}`, async () => {
|
|
1664
1751
|
const cwd = tempCwd({});
|
|
1665
1752
|
if (malformed) writeFileSync(join(cwd, ".pi/settings.json"), "{");
|
|
1666
1753
|
const h = harness({ cwd, branch: verifyBranch() });
|
|
@@ -1668,13 +1755,27 @@ for (const malformed of [false, true]) {
|
|
|
1668
1755
|
assert.deepEqual(await h.emitEvent("tool_call", nestedCall("script/status", { action: "status" })), [undefined]);
|
|
1669
1756
|
const content = [{ type: "text", text: "status result" }];
|
|
1670
1757
|
const result = { ...waveResult("script/status", []), parentToolCallId: "script", content };
|
|
1671
|
-
const
|
|
1758
|
+
const original = structuredClone(result);
|
|
1759
|
+
const [ret] = await h.emitEvent("tool_result", result);
|
|
1760
|
+
assert.equal(ret, undefined);
|
|
1761
|
+
assert.deepEqual(result, original);
|
|
1762
|
+
assert.deepEqual(mergeToolResult(result, ret), original);
|
|
1763
|
+
assert.deepEqual(await h.emitEvent("tool_result", result), [undefined]);
|
|
1764
|
+
const parent = {
|
|
1765
|
+
...result, toolName: "codemode", toolCallId: "script", parentToolCallId: undefined,
|
|
1766
|
+
content: [{ type: "text", text: "script status result" }], structuredContent: { value: "status" },
|
|
1767
|
+
};
|
|
1768
|
+
const [warning] = await h.emitEvent("tool_result", parent);
|
|
1672
1769
|
assert.equal(warning !== undefined, malformed);
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
assert.
|
|
1770
|
+
const merged = mergeToolResult(parent, warning);
|
|
1771
|
+
if (malformed) {
|
|
1772
|
+
assert.match(merged.content[0].text, /gauntlet settings load error \(using defaults\)/);
|
|
1773
|
+
assert.equal(merged.content[0].text.split("gauntlet settings load error").length - 1, 1);
|
|
1774
|
+
assert.deepEqual(merged, { ...parent, content: [merged.content[0], ...parent.content] });
|
|
1775
|
+
} else {
|
|
1776
|
+
assert.deepEqual(merged, parent);
|
|
1676
1777
|
}
|
|
1677
|
-
assert.deepEqual(await h.emitEvent("tool_result",
|
|
1778
|
+
assert.deepEqual(await h.emitEvent("tool_result", parent), [undefined]);
|
|
1678
1779
|
});
|
|
1679
1780
|
}
|
|
1680
1781
|
|
|
@@ -2433,18 +2534,26 @@ test("open-task nudge: fires on every qualifying wave and leaves fix-round count
|
|
|
2433
2534
|
});
|
|
2434
2535
|
|
|
2435
2536
|
// The closure-model mismatch warning is stashed at tool_call; it must render first.
|
|
2436
|
-
test("open-task nudge
|
|
2537
|
+
test("open-task nudge and mismatch preserve results and warning order (direct)", async () => {
|
|
2437
2538
|
const h = harness({
|
|
2438
|
-
cwd: tempCwd({ piGauntlet: { closureReview: { enforce: true, model: "p/configured" } } }),
|
|
2439
|
-
branch: verifyBranch([taskSnapshot(eightTasks([3]))]),
|
|
2539
|
+
cwd: tempCwd({ piGauntlet: { closureReview: { enforce: true, model: "p/configured", maxFixRounds: 1 } } }),
|
|
2540
|
+
branch: verifyBranch([taskSnapshot(eightTasks([3])), subagentResult(["conformance-reviewer"])]),
|
|
2440
2541
|
});
|
|
2441
2542
|
await h.emit("session_start");
|
|
2442
2543
|
const call = { tasks: [{ agent: "conformance-reviewer", model: "p/other", task: "audit" }, { agent: "implementer", task: "fix", worktree: true }] };
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
const
|
|
2446
|
-
|
|
2447
|
-
|
|
2544
|
+
const toolCallId = "m1";
|
|
2545
|
+
assert.equal((await h.emitEvent("tool_call", { toolName: "subagent", toolCallId, input: call }))[0], undefined);
|
|
2546
|
+
const event = {
|
|
2547
|
+
...waveResult(toolCallId, [{ agent: "implementer", exitCode: 0 }]),
|
|
2548
|
+
structuredContent: { results: [{ value: "repair" }] }, usage: { sentinel: "subagent usage" },
|
|
2549
|
+
};
|
|
2550
|
+
const original = structuredClone(event);
|
|
2551
|
+
const [ret] = await h.emitEvent("tool_result", event);
|
|
2552
|
+
const merged = mergeToolResult(event, ret);
|
|
2553
|
+
assert.deepEqual(event, original);
|
|
2554
|
+
assert.match(merged.content[0].text, /p\/other[\s\S]*Plan tasks still open after this repair wave:/);
|
|
2555
|
+
assert.deepEqual(merged, { ...event, content: [merged.content[0], ...event.content] });
|
|
2556
|
+
assert.equal((await firstCallResult(h, "next", implementerWave(1)))?.block, true, "repair still spends one fix round");
|
|
2448
2557
|
});
|
|
2449
2558
|
|
|
2450
2559
|
test("open-task nudge: silent outside the verify window and without open rows", async () => {
|
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
settingsErrorWarning,
|
|
29
29
|
} from "./lib/gauntlet-settings.ts";
|
|
30
30
|
import { loadGauntletSettings } from "./lib/gauntlet-settings-loader.ts";
|
|
31
|
+
import { createPendingWarnings } from "./lib/tool-result-warnings.ts";
|
|
31
32
|
import { checkoutOf, gitSync } from "./lib/checkout.ts";
|
|
32
33
|
import { checkPlan, renderManifest, legOwnership, sha256, type FsPort, type PlanCheckFinding } from "./lib/plan-check.ts";
|
|
33
34
|
import {
|
|
@@ -497,11 +498,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
497
498
|
const firedGuards = new Map<string, boolean>();
|
|
498
499
|
// Warnings stashed at tool_call, prepended at tool_result (verify-before-ship pattern).
|
|
499
500
|
// Relies on tool_result firing for every tool_call; reconstructState clears any stragglers.
|
|
500
|
-
const
|
|
501
|
-
const addGuardWarning = (id: string, text: string) => {
|
|
502
|
-
const prior = pendingGuardWarnings.get(id);
|
|
503
|
-
pendingGuardWarnings.set(id, prior ? prior + "\n\n" + text : text);
|
|
504
|
-
};
|
|
501
|
+
const pendingWarnings = createPendingWarnings();
|
|
505
502
|
|
|
506
503
|
const activeGuardPhase = (): Phase | undefined =>
|
|
507
504
|
GUARD_PHASES.find((p) => phases[p].status === "in_progress");
|
|
@@ -554,7 +551,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
554
551
|
planCheckStamp = undefined;
|
|
555
552
|
attemptedRecoveryEdges.clear();
|
|
556
553
|
firedGuards.clear();
|
|
557
|
-
|
|
554
|
+
pendingWarnings.clear();
|
|
558
555
|
cadenceSeq = 0;
|
|
559
556
|
cadenceLastSeen.implementer = 0;
|
|
560
557
|
cadenceLastSeen["wave-reviewer"] = 0;
|
|
@@ -672,7 +669,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
672
669
|
const g = () => {
|
|
673
670
|
if (!gauntletCache) {
|
|
674
671
|
gauntletCache = loadGauntletSettings(ctx.cwd);
|
|
675
|
-
if (gauntletCache.errors.length > 0)
|
|
672
|
+
if (gauntletCache.errors.length > 0) pendingWarnings.add(event.toolCallId, settingsErrorWarning(gauntletCache.errors));
|
|
676
673
|
}
|
|
677
674
|
return gauntletCache.gauntlet;
|
|
678
675
|
};
|
|
@@ -721,7 +718,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
721
718
|
}
|
|
722
719
|
const mismatched = [...new Set((models as string[]).filter((m) => m !== configured))];
|
|
723
720
|
if (mismatched.length > 0) {
|
|
724
|
-
|
|
721
|
+
pendingWarnings.add(event.toolCallId, closureModelMismatchWarning(configured, mismatched));
|
|
725
722
|
}
|
|
726
723
|
}
|
|
727
724
|
}
|
|
@@ -753,13 +750,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
753
750
|
if (firedGuards.get("brainstorm-write")) return undefined;
|
|
754
751
|
if (typeof p !== "string" || pathInSpecDirs(p, specDirs())) return undefined;
|
|
755
752
|
firedGuards.set("brainstorm-write", true);
|
|
756
|
-
|
|
753
|
+
pendingWarnings.add(event.toolCallId, brainstormWriteWarning(specDirs()));
|
|
757
754
|
return undefined;
|
|
758
755
|
}
|
|
759
756
|
if (firedGuards.get("implement-write")) return undefined;
|
|
760
757
|
if (typeof p !== "string" || pathInSpecDirs(p, implementExemptDirs(specDirs()))) return undefined;
|
|
761
758
|
firedGuards.set("implement-write", true);
|
|
762
|
-
|
|
759
|
+
pendingWarnings.add(event.toolCallId, implementWriteWarning());
|
|
763
760
|
return undefined;
|
|
764
761
|
}
|
|
765
762
|
|
|
@@ -848,7 +845,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
848
845
|
}
|
|
849
846
|
}
|
|
850
847
|
|
|
851
|
-
if (warnings.length > 0)
|
|
848
|
+
if (warnings.length > 0) pendingWarnings.add(event.toolCallId, warnings.join("\n\n"));
|
|
852
849
|
return undefined;
|
|
853
850
|
});
|
|
854
851
|
|
|
@@ -868,26 +865,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
868
865
|
const loaded = loadGauntletSettings(ctx.cwd);
|
|
869
866
|
if (loaded.errors.length === 0 && resolveFlowGuards(loaded.gauntlet).enforce) {
|
|
870
867
|
const open = unfinishedTaskLines(latestPlanSnapshot(ctx));
|
|
871
|
-
if (open.length)
|
|
868
|
+
if (open.length) pendingWarnings.add(event.toolCallId, openTaskNudge(open));
|
|
872
869
|
}
|
|
873
870
|
}
|
|
874
871
|
}
|
|
875
|
-
const warning = pendingGuardWarnings.get(event.toolCallId);
|
|
876
|
-
if (warning) {
|
|
877
|
-
pendingGuardWarnings.delete(event.toolCallId);
|
|
878
|
-
return { content: [{ type: "text" as const, text: warning }, ...event.content] };
|
|
879
|
-
}
|
|
880
|
-
return undefined;
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
// Guards 2/3 — prepend the stashed advisory warning to the bash/write/edit result.
|
|
884
|
-
if (event.toolName === "bash" || event.toolName === "write" || event.toolName === "edit") {
|
|
885
|
-
const warning = pendingGuardWarnings.get(event.toolCallId);
|
|
886
|
-
if (!warning) return undefined;
|
|
887
|
-
pendingGuardWarnings.delete(event.toolCallId);
|
|
888
|
-
return { content: [{ type: "text" as const, text: warning }, ...event.content] };
|
|
889
872
|
}
|
|
890
873
|
|
|
874
|
+
const text = pendingWarnings.take(event.toolCallId);
|
|
875
|
+
if (text) return pendingWarnings.deliver(event, text);
|
|
891
876
|
return undefined;
|
|
892
877
|
});
|
|
893
878
|
|
|
@@ -1349,6 +1334,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
1349
1334
|
fixRoundCredits = 0;
|
|
1350
1335
|
gauntletEntered = nextGauntletEntered(gauntletEntered, "reset", phases.brainstorm.status);
|
|
1351
1336
|
firedGuards.clear();
|
|
1337
|
+
pendingWarnings.clear();
|
|
1352
1338
|
updateWidget(ctx);
|
|
1353
1339
|
return {
|
|
1354
1340
|
content: [{ type: "text", text: "Phase tracker reset. All phases pending." }],
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import { DEFAULT_TELEMETRY_BUCKETS, planDirsFor, resolveFlowGuards } from "./lib/gauntlet-settings.ts";
|
|
8
8
|
import { emptyAccumulators, parseRecord, serializeRecord } from "./lib/telemetry-record.ts";
|
|
9
9
|
import { guardReason } from "./lib/telemetry-ship.ts";
|
|
10
|
+
import { mergeToolResult } from "./test-support/merge-tool-result.mjs";
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
const originalDepth = process.env.PI_SUBAGENT_DEPTH;
|
|
@@ -35,6 +36,7 @@
|
|
|
35
36
|
const specDirs = resolveFlowGuards({}).specDirs;
|
|
36
37
|
const fetchedRecords = new Map<string, string>();
|
|
37
38
|
let clock = Date.parse("2026-09-17T10:00:00Z");
|
|
39
|
+
let enabled = o.enabled ?? true;
|
|
38
40
|
const deps: Deps = {
|
|
39
41
|
telemetryRepo: () => "test/repo",
|
|
40
42
|
readTelemetryBranch: () => ({ records: [...fetchedRecords].map(([path, bytes]) => ({ path, bytes })) }),
|
|
@@ -61,7 +63,7 @@
|
|
|
61
63
|
if (args[0] === "ls-files") return { code: 0, stdout: "", stderr: "" };
|
|
62
64
|
return { code: 0, stdout: "", stderr: "" };
|
|
63
65
|
},
|
|
64
|
-
settings: () => ({ specDirs, planDirs: planDirsFor(specDirs), telemetry: { enabled
|
|
66
|
+
settings: () => ({ specDirs, planDirs: planDirsFor(specDirs), telemetry: { enabled, buckets: DEFAULT_TELEMETRY_BUCKETS, warning: o.telemetryWarning }, errors: [], agentOverrides: { implementer: { model: "p/x" } }, versions: { pi: "0.85.1" } }),
|
|
65
67
|
// A plain jj workspace: `jj root` answers the temp root while git rev-parse fails.
|
|
66
68
|
jj: async (args, cwd) => {
|
|
67
69
|
jjCalls.push(args);
|
|
@@ -100,7 +102,7 @@
|
|
|
100
102
|
await emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: rel, content: body } });
|
|
101
103
|
return emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: rel, content: body }, content: [], isError: false, details: undefined });
|
|
102
104
|
};
|
|
103
|
-
return { fetchedRecords, root, handlers, emit, gitCalls, gitCwds, jjCalls, readFileCalls, gitWrites, excludeFile, bash, recordPath, readRecord, phaseResult, writeSpec, setBranch: (b: unknown[]) => (branch = b), ctx };
|
|
105
|
+
return { setEnabled: (value: boolean) => (enabled = value), fetchedRecords, root, handlers, emit, gitCalls, gitCwds, jjCalls, readFileCalls, gitWrites, excludeFile, bash, recordPath, readRecord, phaseResult, writeSpec, setBranch: (b: unknown[]) => (branch = b), ctx };
|
|
104
106
|
}
|
|
105
107
|
|
|
106
108
|
const P = (over: Record<string, string> = {}) => Object.fromEntries(["brainstorm", "plan", "implement", "verify", "ship"].map((p) => [p, { status: over[p] ?? "pending" }]));
|
|
@@ -1222,6 +1224,69 @@
|
|
|
1222
1224
|
return h;
|
|
1223
1225
|
}
|
|
1224
1226
|
|
|
1227
|
+
for (const toolName of ["write", "edit"]) {
|
|
1228
|
+
for (const nested of [false, true]) {
|
|
1229
|
+
test(`post-ship ${toolName} warning preserves results (${nested ? "nested" : "direct"})`, async () => {
|
|
1230
|
+
const h = await boundInJjShip();
|
|
1231
|
+
const before = readFileSync(h.recordPath(), "utf8");
|
|
1232
|
+
const event = {
|
|
1233
|
+
toolName, toolCallId: nested ? "parent/0" : "direct",
|
|
1234
|
+
...(nested ? { parentToolCallId: "parent" } : {}),
|
|
1235
|
+
input: { path: "doc/specs/a.md", content: "# Spec\n", newText: "updated" },
|
|
1236
|
+
content: [{ type: "text", text: "spec updated" }], isError: false,
|
|
1237
|
+
structuredContent: toolName === "write" ? { written: true } : null,
|
|
1238
|
+
details: { sentinel: "telemetry details" }, usage: { sentinel: "telemetry usage" },
|
|
1239
|
+
};
|
|
1240
|
+
const original = structuredClone(event);
|
|
1241
|
+
const [ret] = await h.emit("tool_result", event);
|
|
1242
|
+
let delivered = event;
|
|
1243
|
+
let merged = mergeToolResult(event, ret);
|
|
1244
|
+
if (nested) {
|
|
1245
|
+
assert.equal(ret, undefined);
|
|
1246
|
+
assert.deepEqual(merged, original);
|
|
1247
|
+
const parent = {
|
|
1248
|
+
...event, toolName: "codemode", toolCallId: "parent", parentToolCallId: undefined,
|
|
1249
|
+
content: [{ type: "text", text: "script complete" }], structuredContent: { value: true },
|
|
1250
|
+
};
|
|
1251
|
+
delivered = parent;
|
|
1252
|
+
const [parentRet] = await h.emit("tool_result", parent);
|
|
1253
|
+
merged = mergeToolResult(parent, parentRet);
|
|
1254
|
+
assert.equal((await h.emit("tool_result", parent))[0], undefined);
|
|
1255
|
+
}
|
|
1256
|
+
assert.deepEqual(event, original);
|
|
1257
|
+
assert.match(merged.content[0].text, /spec shipped at .*write a follow-up spec/);
|
|
1258
|
+
assert.equal(merged.content[0].text.split("spec shipped at").length - 1, 1);
|
|
1259
|
+
assert.deepEqual(merged, { ...delivered, content: [merged.content[0], ...delivered.content] });
|
|
1260
|
+
assert.equal(readFileSync(h.recordPath(), "utf8"), before, "sealed record stays frozen");
|
|
1261
|
+
});
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
|
|
1265
|
+
for (const transition of ["disabled", "session_tree"] as const) {
|
|
1266
|
+
test(`deferred shipped-spec warning survives ${transition} before its parent result`, async () => {
|
|
1267
|
+
const h = await boundInJjShip();
|
|
1268
|
+
const before = readFileSync(h.recordPath(), "utf8");
|
|
1269
|
+
const child = {
|
|
1270
|
+
toolName: "edit", toolCallId: "parent/0", parentToolCallId: "parent",
|
|
1271
|
+
input: { path: "doc/specs/a.md", newText: "updated" },
|
|
1272
|
+
content: [{ type: "text", text: "updated" }], structuredContent: { edited: true }, isError: false,
|
|
1273
|
+
};
|
|
1274
|
+
assert.equal((await h.emit("tool_result", child))[0], undefined);
|
|
1275
|
+
if (transition === "disabled") h.setEnabled(false);
|
|
1276
|
+
else await h.emit("session_tree", { type: "session_tree" });
|
|
1277
|
+
const parent = {
|
|
1278
|
+
toolName: "codemode", toolCallId: "parent", input: {}, isError: false,
|
|
1279
|
+
content: [{ type: "text", text: "done" }], structuredContent: { value: 0 },
|
|
1280
|
+
};
|
|
1281
|
+
const [ret] = await h.emit("tool_result", parent);
|
|
1282
|
+
const merged = mergeToolResult(parent, ret);
|
|
1283
|
+
assert.match(merged.content[0].text, /spec shipped at .*write a follow-up spec/);
|
|
1284
|
+
assert.deepEqual(merged, { ...parent, content: [merged.content[0], ...parent.content] });
|
|
1285
|
+
assert.equal((await h.emit("tool_result", parent))[0], undefined);
|
|
1286
|
+
assert.equal(readFileSync(h.recordPath(), "utf8"), before, "sealed record stays frozen");
|
|
1287
|
+
});
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1225
1290
|
const diffWarnings = (h: { readRecord: () => ReturnType<typeof parseRecord> }) =>
|
|
1226
1291
|
h.readRecord()!.events.filter((e) => e.kind === "warning").map((e: any) => e.message as string).filter((m) => m.startsWith("diff omitted"));
|
|
1227
1292
|
|
|
@@ -1234,7 +1299,7 @@
|
|
|
1234
1299
|
assert.equal(rec.derived.gates.ship_option, "keep");
|
|
1235
1300
|
const beforeWrite = readFileSync(h.recordPath(), "utf8");
|
|
1236
1301
|
const out = await h.writeSpec("doc/specs/a.md");
|
|
1237
|
-
assert.deepEqual(out.filter(Boolean), [{ content: [{ type: "text", text: `⚠️ spec shipped at ${rec.shipped_at}; write a follow-up spec that supersedes it` }] }]);
|
|
1302
|
+
assert.deepEqual(out.filter(Boolean), [{ content: [{ type: "text", text: `⚠️ spec shipped at ${rec.shipped_at}; write a follow-up spec that supersedes it` }], structuredContent: undefined }]);
|
|
1238
1303
|
assert.equal(readFileSync(h.recordPath(), "utf8"), beforeWrite);
|
|
1239
1304
|
assert.equal(h.readRecord().derived.spec_edits_after_ship, 0);
|
|
1240
1305
|
assert.deepEqual(rec.derived.modified_files, ["src/a.ts", "test/a.test.ts"]);
|
package/extensions/telemetry.ts
CHANGED
|
@@ -14,9 +14,10 @@
|
|
|
14
14
|
import { dirname, isAbsolute, join, posix, resolve } from "node:path";
|
|
15
15
|
import * as piRuntime from "@earendil-works/pi-coding-agent";
|
|
16
16
|
import { SettingsManager, getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
17
|
-
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
17
|
+
import type { ExtensionAPI, ExtensionContext, ToolResultEvent } from "@earendil-works/pi-coding-agent";
|
|
18
18
|
import { DEFAULT_TELEMETRY_DIR, DEFAULT_TEST_COMMANDS, planDirsFor, resolveFlowGuards, resolveTelemetry, settingsErrorWarning, type TelemetryResolved } from "./lib/gauntlet-settings.ts";
|
|
19
19
|
import { loadGauntletSettings } from "./lib/gauntlet-settings-loader.ts";
|
|
20
|
+
import { createPendingWarnings } from "./lib/tool-result-warnings.ts";
|
|
20
21
|
import { checkoutOf } from "./lib/checkout.ts";
|
|
21
22
|
import { CONTEXT_DRAFT_MARKER, nextGauntletEntered } from "./lib/phase-tracker-helpers.ts";
|
|
22
23
|
import { sha256 } from "./lib/plan-check.ts";
|
|
@@ -544,6 +545,8 @@
|
|
|
544
545
|
});
|
|
545
546
|
|
|
546
547
|
pi.on("tool_result", async (event, ctx) => {
|
|
548
|
+
const text = pendingWarnings.take(event.toolCallId);
|
|
549
|
+
if (text) return pendingWarnings.deliver(event, text);
|
|
547
550
|
lastCtx = ctx;
|
|
548
551
|
const snap = enabledSettings(ctx);
|
|
549
552
|
if (!snap) return undefined;
|
|
@@ -605,7 +608,7 @@
|
|
|
605
608
|
});
|
|
606
609
|
|
|
607
610
|
// Binding + spec-write bookkeeping; the bound-spec hooks are defined in block 2.
|
|
608
|
-
const onSpecInteraction = async (tool: "write" | "edit" | "read", loc: { toplevel: string; rel: string; via: "git" | "jj" }, event:
|
|
611
|
+
const onSpecInteraction = async (tool: "write" | "edit" | "read", loc: { toplevel: string; rel: string; via: "git" | "jj" }, event: ToolResultEvent, snap: SettingsSnapshot): Promise<unknown> => {
|
|
609
612
|
const rel = loc.rel;
|
|
610
613
|
if (!boundSpec) {
|
|
611
614
|
if ((tool === "write" || tool === "edit") && isSpecPath(rel, snap.specDirs)) {
|
|
@@ -655,12 +658,13 @@
|
|
|
655
658
|
};
|
|
656
659
|
|
|
657
660
|
// Defined in block 2.
|
|
658
|
-
let onBoundSpecWrite: (tool: "write" | "edit", rel: string, event:
|
|
661
|
+
let onBoundSpecWrite: (tool: "write" | "edit", rel: string, event: ToolResultEvent) => unknown;
|
|
659
662
|
let onOtherSpecEdit: (rel: string, event: { input: unknown }, specDirs: string[]) => void;
|
|
660
663
|
|
|
661
664
|
let amendmentOpen = false;
|
|
662
665
|
let lastPlanTasks: { name?: string; status: string }[] | undefined;
|
|
663
666
|
const pendingTest = new Map<string, string>(); // toolCallId -> matched statement
|
|
667
|
+
const pendingWarnings = createPendingWarnings();
|
|
664
668
|
|
|
665
669
|
const phaseAcc = (k: PhaseKey = phaseNow()): PhaseAcc => (live().phases[k] ??= {});
|
|
666
670
|
|
|
@@ -877,7 +881,7 @@
|
|
|
877
881
|
refreshLinks();
|
|
878
882
|
if (record.shipped_at) {
|
|
879
883
|
const warning = `⚠️ spec shipped at ${record.shipped_at}; write a follow-up spec that supersedes it`;
|
|
880
|
-
return
|
|
884
|
+
return pendingWarnings.deliver(event, warning);
|
|
881
885
|
}
|
|
882
886
|
if (record.approved_at && !amendmentOpen) {
|
|
883
887
|
acc.amendments += 1;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// Mirror the runner's replacement rule so a raw hook return cannot hide data loss.
|
|
2
|
+
export function mergeToolResult(event, ret) {
|
|
3
|
+
if (ret === undefined) return event;
|
|
4
|
+
const merged = { ...event, ...ret };
|
|
5
|
+
if (ret.content !== undefined && ret.structuredContent === undefined) {
|
|
6
|
+
delete merged.structuredContent;
|
|
7
|
+
}
|
|
8
|
+
return merged;
|
|
9
|
+
}
|
|
@@ -4,6 +4,7 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs";
|
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import registerVerifyBeforeShip from "./verify-before-ship.ts";
|
|
7
|
+
import { mergeToolResult } from "./test-support/merge-tool-result.mjs";
|
|
7
8
|
import { execFileSync } from "node:child_process";
|
|
8
9
|
|
|
9
10
|
const tempDirs: string[] = [];
|
|
@@ -57,12 +58,78 @@ test("git commit is not watched: no warning even when unverified", async () => {
|
|
|
57
58
|
assert.equal((await h.emitEvent("tool_result", bashResult("c1")))[0], undefined);
|
|
58
59
|
});
|
|
59
60
|
|
|
60
|
-
|
|
61
|
+
for (const { name, toolName, resultFields } of [
|
|
62
|
+
{ name: "bash", toolName: "bash", resultFields: { structuredContent: { output: "ok", exit_code: 0 } } },
|
|
63
|
+
{ name: "write", toolName: "write", resultFields: { structuredContent: null } },
|
|
64
|
+
{ name: "edit", toolName: "edit", resultFields: { structuredContent: { sentinel: "edit payload" } } },
|
|
65
|
+
{ name: "edit without structuredContent", toolName: "edit", resultFields: {} },
|
|
66
|
+
]) {
|
|
67
|
+
test(`direct ${name} ship warning preserves the result fields`, async () => {
|
|
68
|
+
const h = harness();
|
|
69
|
+
await editSource(h);
|
|
70
|
+
await h.emitEvent("tool_call", bashCall("p1", "git push origin main"));
|
|
71
|
+
// Synthetic write/edit results exercise the tool-independent pending drain.
|
|
72
|
+
const event = {
|
|
73
|
+
...bashResult("p1", true), toolName,
|
|
74
|
+
...resultFields,
|
|
75
|
+
details: { sentinel: "ship details" }, usage: { sentinel: "ship usage" },
|
|
76
|
+
};
|
|
77
|
+
const [ret] = await h.emitEvent("tool_result", event);
|
|
78
|
+
const merged = mergeToolResult(event, ret);
|
|
79
|
+
assert.match(merged.content[0].text, /ran without verification/);
|
|
80
|
+
assert.deepEqual(merged.content.slice(1), event.content);
|
|
81
|
+
assert.deepEqual(merged, { ...event, content: [merged.content[0], ...event.content] });
|
|
82
|
+
assert.equal((await h.emitEvent("tool_result", event))[0], undefined);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
for (const toolName of ["bash", "write", "edit"]) {
|
|
87
|
+
test(`nested ${toolName} ship warning leaves the child clean and reaches its parent once`, async () => {
|
|
88
|
+
const h = harness();
|
|
89
|
+
await editSource(h);
|
|
90
|
+
await h.emitEvent("tool_call", { ...bashCall("parent/0", "git push origin main"), parentToolCallId: "parent" });
|
|
91
|
+
const child = {
|
|
92
|
+
...bashResult("parent/0"), toolName, parentToolCallId: "parent",
|
|
93
|
+
structuredContent: { output: "ok", exit_code: 0 },
|
|
94
|
+
};
|
|
95
|
+
const [childRet] = await h.emitEvent("tool_result", child);
|
|
96
|
+
assert.deepEqual(mergeToolResult(child, childRet), child);
|
|
97
|
+
assert.equal(childRet, undefined);
|
|
98
|
+
const parent = {
|
|
99
|
+
...bashResult("parent"), toolName: "codemode",
|
|
100
|
+
content: [{ type: "text", text: "script value: 0" }], structuredContent: { value: 0 },
|
|
101
|
+
};
|
|
102
|
+
const [parentRet] = await h.emitEvent("tool_result", parent);
|
|
103
|
+
const merged = mergeToolResult(parent, parentRet);
|
|
104
|
+
assert.match(merged.content[0].text, /ran without verification/);
|
|
105
|
+
assert.equal(merged.content[0].text.split("ran without verification").length - 1, 1);
|
|
106
|
+
assert.deepEqual(merged.content.slice(1), parent.content);
|
|
107
|
+
assert.deepEqual(merged, { ...parent, content: [merged.content[0], ...parent.content] });
|
|
108
|
+
assert.equal((await h.emitEvent("tool_result", parent))[0], undefined);
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
test("nested ship warnings aggregate in arrival order without crediting the parent as a test", async () => {
|
|
61
113
|
const h = harness();
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
114
|
+
for (const [index, command] of ["git push origin first", "git push origin second"].entries()) {
|
|
115
|
+
const toolCallId = `parent/${index}`;
|
|
116
|
+
await h.emitEvent("tool_call", { ...bashCall(toolCallId, command), parentToolCallId: "parent" });
|
|
117
|
+
const child = { ...bashResult(toolCallId), parentToolCallId: "parent", structuredContent: { exit_code: 0 } };
|
|
118
|
+
assert.equal((await h.emitEvent("tool_result", child))[0], undefined);
|
|
119
|
+
}
|
|
120
|
+
const parent = { ...bashResult("parent", true), toolName: "codemode", structuredContent: null };
|
|
121
|
+
const [ret] = await h.emitEvent("tool_result", parent);
|
|
122
|
+
const merged = mergeToolResult(parent, ret);
|
|
123
|
+
assert.match(merged.content[0].text, /origin first[\s\S]*origin second/);
|
|
124
|
+
assert.equal(merged.content[0].text.split("ran without verification").length - 1, 2);
|
|
125
|
+
assert.deepEqual(merged, { ...parent, content: [merged.content[0], ...parent.content] });
|
|
126
|
+
assert.equal((await h.emitEvent("tool_result", parent))[0], undefined);
|
|
127
|
+
|
|
128
|
+
await h.emitEvent("tool_call", { ...bashCall("test-parent/0", "npm test"), parentToolCallId: "test-parent" });
|
|
129
|
+
await h.emitEvent("tool_result", { ...bashResult("test-parent/0"), parentToolCallId: "test-parent" });
|
|
130
|
+
await h.emitEvent("tool_result", { ...bashResult("test-parent", true), toolName: "codemode" });
|
|
131
|
+
await h.emitEvent("tool_call", bashCall("verified", "git push"));
|
|
132
|
+
assert.equal((await h.emitEvent("tool_result", bashResult("verified")))[0], undefined);
|
|
66
133
|
});
|
|
67
134
|
|
|
68
135
|
test("gh pr create warns when unverified", async () => {
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
settingsErrorWarning,
|
|
37
37
|
} from "./lib/gauntlet-settings.ts";
|
|
38
38
|
import { loadGauntletSettings } from "./lib/gauntlet-settings-loader.ts";
|
|
39
|
+
import { createPendingWarnings } from "./lib/tool-result-warnings.ts";
|
|
39
40
|
import { matchShipStatement } from "./lib/telemetry-paths.ts";
|
|
40
41
|
import { checkoutOfSync, gitSync } from "./lib/checkout.ts";
|
|
41
42
|
import { parseLegs } from "./lib/plan-check.ts";
|
|
@@ -214,7 +215,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
214
215
|
});
|
|
215
216
|
};
|
|
216
217
|
const pendingTests = new Map<string, Set<string>>();
|
|
217
|
-
const
|
|
218
|
+
const pendingWarnings = createPendingWarnings();
|
|
218
219
|
|
|
219
220
|
const getCommand = (event: { input: unknown }): string => {
|
|
220
221
|
const input = event.input as { command?: unknown } | undefined;
|
|
@@ -236,11 +237,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
236
237
|
const command = getCommand(event);
|
|
237
238
|
if (!command) return undefined;
|
|
238
239
|
|
|
239
|
-
const addShipWarning = (id: string, text: string) => {
|
|
240
|
-
const prior = pendingShipWarnings.get(id);
|
|
241
|
-
pendingShipWarnings.set(id, prior ? prior + "\n\n" + text : text);
|
|
242
|
-
};
|
|
243
|
-
|
|
244
240
|
const targets = new Set<string>();
|
|
245
241
|
let settingsErrShown = false;
|
|
246
242
|
for (const statement of statements(command, ctx.cwd)) {
|
|
@@ -255,11 +251,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
255
251
|
const isTest = buildTestCmdRegex(testCommands).test(statement.testCommand) && !/^(?:echo|printf|rg|grep)\b/.test(statement.command);
|
|
256
252
|
if (isTest && statement.trusted) targets.add(statement.target);
|
|
257
253
|
if ((isTest || matchShipStatement(statement.command)) && errors.length && !settingsErrShown) {
|
|
258
|
-
|
|
254
|
+
pendingWarnings.add(event.toolCallId, settingsErrorWarning(errors));
|
|
259
255
|
settingsErrShown = true;
|
|
260
256
|
}
|
|
261
257
|
if (matchShipStatement(statement.command) && (!statement.trusted || !verified.has(statement.target))) {
|
|
262
|
-
|
|
258
|
+
pendingWarnings.add(event.toolCallId, formatWarning(command, testCommands, warningReference));
|
|
263
259
|
}
|
|
264
260
|
}
|
|
265
261
|
if (targets.size) pendingTests.set(event.toolCallId, targets);
|
|
@@ -268,18 +264,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
268
264
|
|
|
269
265
|
pi.on("tool_result", async (event, ctx) => {
|
|
270
266
|
rememberPlan(event, ctx.cwd);
|
|
271
|
-
if (event.toolName
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
const warning = pendingShipWarnings.get(event.toolCallId);
|
|
278
|
-
if (!warning) return undefined;
|
|
279
|
-
pendingShipWarnings.delete(event.toolCallId);
|
|
267
|
+
if (event.toolName === "bash") {
|
|
268
|
+
const targets = pendingTests.get(event.toolCallId);
|
|
269
|
+
pendingTests.delete(event.toolCallId);
|
|
270
|
+
if (!event.isError) for (const target of targets ?? []) verified.add(target);
|
|
271
|
+
}
|
|
280
272
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
273
|
+
const text = pendingWarnings.take(event.toolCallId);
|
|
274
|
+
if (text) return pendingWarnings.deliver(event, text);
|
|
275
|
+
return undefined;
|
|
284
276
|
});
|
|
285
277
|
}
|
package/package.json
CHANGED