pi-gauntlet 8.2.1 → 8.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +5 -5
- package/agents/loop-implementer.md +19 -4
- package/bin/gauntlet-performance.mjs +477 -64
- package/bin/gauntlet-performance.test.mjs +27 -27
- package/bin/gauntlet-spec-index.mjs +290 -47
- package/bin/gauntlet-spec-index.test.mjs +42 -18
- package/bin/gauntlet-telemetry-seal.mjs +220 -77
- package/bin/gauntlet-telemetry-seal.test.mjs +222 -457
- package/extensions/lib/gauntlet-settings.test.ts +12 -14
- package/extensions/lib/gauntlet-settings.ts +7 -16
- package/extensions/lib/plan-check.test.ts +10 -0
- package/extensions/lib/plan-check.ts +2 -2
- package/extensions/lib/telemetry-branch.test.ts +102 -0
- package/extensions/lib/telemetry-branch.ts +145 -0
- package/extensions/lib/telemetry-diff.test.ts +2 -2
- package/extensions/lib/telemetry-diff.ts +1 -1
- package/extensions/lib/telemetry-leg.ts +163 -0
- package/extensions/lib/telemetry-record.ts +2 -0
- package/extensions/lib/vcs.ts +6 -23
- package/extensions/telemetry-leg-failures.test.ts +138 -0
- package/extensions/telemetry-leg-fields.test.ts +130 -0
- package/extensions/telemetry-leg-no-write.test.ts +90 -0
- package/extensions/telemetry-leg-rounds.test.ts +157 -0
- package/extensions/telemetry-leg-trackers.test.ts +114 -0
- package/extensions/telemetry-leg.test.ts +151 -0
- package/extensions/telemetry-settlement-contract.test.ts +67 -0
- package/extensions/telemetry.test.ts +44 -27
- package/extensions/telemetry.ts +68 -48
- package/extensions/test-support/git-env.mjs +13 -0
- package/package.json +1 -1
- package/skills/brainstorming/SKILL.md +3 -1
- package/skills/check-delivery/SKILL.md +2 -0
- package/skills/finishing-a-development-branch/SKILL.md +13 -13
- package/skills/finishing-a-development-branch/reference/verification-set.md +2 -2
- package/skills/gatekeep-pr/reference/batch.md +3 -1
- package/skills/gatekeep-pr/reference/cleanup.md +1 -1
- package/skills/gatekeep-pr/reference/post-selection-loop.md +7 -1
- package/skills/gauntlet-resume/reference/reconstruction.md +6 -6
- package/skills/gauntlet-resume/reference/seed.md +1 -6
- package/skills/loop-implementation/SKILL.md +40 -21
- package/skills/loop-implementation/reference/action-spend.md +10 -0
- package/skills/loop-implementation/reference/coordinated-legs.md +5 -5
- package/skills/loop-implementation/reference/interrupted-recovery.md +10 -0
- package/skills/loop-implementation/reference/unreachable-returns.md +13 -0
- package/skills/shape-ticket/SKILL.md +1 -1
- package/skills/shape-ticket/reference/repo-audit.md +2 -2
- package/skills/verification-before-completion/reference/conformance-check.md +1 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { execFileSync, spawnSync } from "node:child_process";
|
|
3
|
+
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { after, test } from "node:test";
|
|
7
|
+
import { parse } from "yaml";
|
|
8
|
+
import { isolatedGitEnv } from "./test-support/git-env.mjs";
|
|
9
|
+
import { resolveTelemetry } from "./lib/gauntlet-settings.ts";
|
|
10
|
+
import type { ReadToolCallEvent } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
|
|
12
|
+
const gitEnv = isolatedGitEnv(mkdtempSync(join(tmpdir(), "telemetry-leg-env-")));
|
|
13
|
+
const originalGitEnv = Object.fromEntries(Object.keys(gitEnv).map(key => [key, process.env[key]]));
|
|
14
|
+
Object.assign(process.env, gitEnv);
|
|
15
|
+
// vcs.ts snapshots the environment at import time, including publisher subprocess settings.
|
|
16
|
+
const { default: registerTelemetry, realDeps } = await import("./telemetry.ts");
|
|
17
|
+
const { registerLegTelemetry } = await import("./lib/telemetry-leg.ts");
|
|
18
|
+
|
|
19
|
+
const ref = "refs/heads/gauntlet/telemetry";
|
|
20
|
+
const originalDepth = process.env.PI_SUBAGENT_DEPTH;
|
|
21
|
+
process.env.PI_SUBAGENT_DEPTH = "0";
|
|
22
|
+
after(() => {
|
|
23
|
+
for (const [key, value] of Object.entries(originalGitEnv)) {
|
|
24
|
+
if (value === undefined) delete process.env[key];
|
|
25
|
+
else process.env[key] = value;
|
|
26
|
+
}
|
|
27
|
+
if (originalDepth === undefined) delete process.env.PI_SUBAGENT_DEPTH;
|
|
28
|
+
else process.env.PI_SUBAGENT_DEPTH = originalDepth;
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
for (const skill of ["shape-ticket", "gatekeep-pr", "check-delivery"] as const) {
|
|
32
|
+
test(`${skill} read tool_call opens a leg before any tool_result`, async () => {
|
|
33
|
+
const handlers = new Map<string, (event: unknown, ctx: unknown) => unknown>();
|
|
34
|
+
const live = registerLegTelemetry({
|
|
35
|
+
on: (name: string, handler: (event: unknown, ctx: unknown) => unknown) => handlers.set(name, handler),
|
|
36
|
+
} as any, {
|
|
37
|
+
...realDeps,
|
|
38
|
+
now: () => "2026-11-06T12:34:56Z",
|
|
39
|
+
settings: () => ({ telemetry: resolveTelemetry({}), specDirs: [], planDirs: [], errors: [], versions: {} }),
|
|
40
|
+
}, () => false);
|
|
41
|
+
const ctx = { cwd: tmpdir(), sessionManager: { getSessionId: () => `session-${skill}` } };
|
|
42
|
+
const event: ReadToolCallEvent = {
|
|
43
|
+
type: "tool_call", toolCallId: "load-skill", toolName: "read",
|
|
44
|
+
input: { path: `/package/skills/${skill}/SKILL.md` },
|
|
45
|
+
};
|
|
46
|
+
assert.equal(live(), undefined);
|
|
47
|
+
await handlers.get("tool_call")!(event, ctx);
|
|
48
|
+
assert.ok(live(), "the read call must open the accumulator without waiting for a result");
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Real plumbing stays local: origin's portable identity is rewritten to a scratch bare repo.
|
|
53
|
+
function harness(skill: string) {
|
|
54
|
+
const scratch = mkdtempSync(join(tmpdir(), "telemetry-leg-"));
|
|
55
|
+
const root = join(scratch, "product");
|
|
56
|
+
const origin = join(scratch, "origin.git");
|
|
57
|
+
mkdirSync(root);
|
|
58
|
+
const git = (cwd: string, ...args: string[]) => execFileSync("git", ["-C", cwd, "-c", "commit.gpgsign=false", ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
|
|
59
|
+
git(scratch, "init", "--bare", origin);
|
|
60
|
+
git(root, "init", "-b", "feature/example");
|
|
61
|
+
git(root, "config", "user.name", "Leg Tester");
|
|
62
|
+
git(root, "config", "user.email", "leg@example.test");
|
|
63
|
+
git(root, "remote", "add", "origin", "https://example.test/team/product.git");
|
|
64
|
+
git(root, "config", `url.${origin}.insteadOf`, "https://example.test/team/product.git");
|
|
65
|
+
writeFileSync(join(root, "file.txt"), "base\n");
|
|
66
|
+
git(root, "add", "file.txt");
|
|
67
|
+
git(root, "commit", "-m", "initial");
|
|
68
|
+
writeFileSync(join(root, "file.txt"), "staged user change\n");
|
|
69
|
+
git(root, "add", "file.txt");
|
|
70
|
+
writeFileSync(join(root, "file.txt"), "unstaged user change\n");
|
|
71
|
+
const head = git(root, "rev-parse", "HEAD");
|
|
72
|
+
const index = readFileSync(join(root, ".git/index"));
|
|
73
|
+
const handlers = new Map<string, ((event: any, ctx: any) => unknown)[]>();
|
|
74
|
+
const messages: unknown[] = [];
|
|
75
|
+
const ctx = { cwd: root, sessionManager: { getBranch: () => [], getSessionId: () => `session-${skill}` } };
|
|
76
|
+
registerTelemetry({
|
|
77
|
+
on: (name: string, handler: (event: any, ctx: any) => unknown) => handlers.set(name, [...(handlers.get(name) ?? []), handler]),
|
|
78
|
+
sendMessage: (message: unknown) => messages.push(message),
|
|
79
|
+
} as any, {
|
|
80
|
+
...realDeps,
|
|
81
|
+
now: () => "2026-11-06T12:34:56Z",
|
|
82
|
+
settings: () => ({ telemetry: resolveTelemetry({}), specDirs: [], planDirs: [], errors: [], versions: { "pi-gauntlet": "8.2.1" } }),
|
|
83
|
+
});
|
|
84
|
+
const emit = async (name: string, event: unknown = {}) => {
|
|
85
|
+
for (const handler of handlers.get(name) ?? []) await handler(event, ctx);
|
|
86
|
+
};
|
|
87
|
+
const say = (text: string) => emit("message_end", { message: { role: "assistant", content: [{ type: "text", text }] } });
|
|
88
|
+
const hasRecord = () => spawnSync("git", ["-C", origin, "rev-parse", "--verify", ref], { stdio: "ignore" }).status === 0;
|
|
89
|
+
return { root, origin, git, head, index, emit, say, hasRecord, messages };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
for (const [skill, outcome, activation, empty] of [
|
|
93
|
+
["shape-ticket", "created", "input"],
|
|
94
|
+
["gatekeep-pr", "approved", "expanded"],
|
|
95
|
+
["check-delivery", "delivered", "read"],
|
|
96
|
+
["gatekeep-pr", "stopped", "input"],
|
|
97
|
+
["shape-ticket", "written", "input", true],
|
|
98
|
+
] as const) {
|
|
99
|
+
test(`${skill} ${outcome}${empty ? " with no items" : ""} seals one session leg only after the final settlement and follow-on work`, async () => {
|
|
100
|
+
const marker = outcome === "stopped" ? "stuck" : "done";
|
|
101
|
+
const h = harness(skill);
|
|
102
|
+
await h.emit("session_start");
|
|
103
|
+
// Terminal-looking text outside a selected skill is not a leg.
|
|
104
|
+
await h.say(`gauntlet: done ${outcome}\n#42: ${outcome}`);
|
|
105
|
+
await h.emit("agent_settled", { aborted: false });
|
|
106
|
+
assert.equal(h.hasRecord(), false);
|
|
107
|
+
if (activation === "input") await h.emit("input", { text: `/skill:${skill} create a ticket`, source: "interactive" });
|
|
108
|
+
if (activation === "expanded") await h.emit("before_agent_start", { messageType: "user", prompt: `<skill name="${skill}" location="/package/skills/${skill}/SKILL.md">\nInstructions\n</skill>\n\n#42` });
|
|
109
|
+
if (activation === "read") await h.emit("tool_call", { type: "tool_call", toolName: "read", toolCallId: "load-skill", input: { path: `/package/skills/${skill}/SKILL.md` } });
|
|
110
|
+
await h.say("gauntlet: gate approval\n1. Approve\n2. Decline");
|
|
111
|
+
await h.emit("agent_settled", { aborted: false });
|
|
112
|
+
assert.equal(h.hasRecord(), false, "human gate is not completion");
|
|
113
|
+
await h.say(`gauntlet: done ${outcome}\n#42: ${outcome}`);
|
|
114
|
+
await h.emit("agent_end");
|
|
115
|
+
assert.equal(h.hasRecord(), false, "agent_end can precede follow-on work");
|
|
116
|
+
await h.emit("tool_call", { toolName: "bash", toolCallId: "follow-on", input: { command: "write approved result" } });
|
|
117
|
+
await h.emit("tool_result", { toolName: "bash", toolCallId: "follow-on", input: {}, isError: false, content: [] });
|
|
118
|
+
await h.say("Follow-on assessment is still running.");
|
|
119
|
+
await h.emit("agent_settled", { aborted: false });
|
|
120
|
+
assert.equal(h.hasRecord(), false, "an earlier terminal message cannot settle follow-on work");
|
|
121
|
+
await h.say(`gauntlet: ${marker} ${outcome}\n#42: ${outcome}`);
|
|
122
|
+
await h.emit("agent_settled", { aborted: true });
|
|
123
|
+
assert.equal(h.hasRecord(), false, "aborting is not a terminal settlement");
|
|
124
|
+
await h.emit("message_end", { message: {
|
|
125
|
+
role: "assistant", content: [{ type: "text", text: `gauntlet: ${marker} ${outcome}${empty ? "\nworktree: done\nNo settled items." : `\n#42: ${outcome}`}` }],
|
|
126
|
+
usage: { input: 12, output: 4, cost: 0.125 },
|
|
127
|
+
} });
|
|
128
|
+
await h.emit("agent_settled", { aborted: false });
|
|
129
|
+
assert.equal(h.hasRecord(), true, "final settlement must push a leg record even without items");
|
|
130
|
+
const path = `team/product/${skill}/20261106T123456Z-session-${skill}.yaml`;
|
|
131
|
+
assert.equal(h.git(h.origin, "ls-tree", "-r", "--name-only", ref), path);
|
|
132
|
+
const record = parse(h.git(h.origin, "show", `${ref}:${path}`));
|
|
133
|
+
assert.equal(record.skill, skill);
|
|
134
|
+
assert.equal(record.session, `session-${skill}`);
|
|
135
|
+
assert.equal(record.repo, "team/product");
|
|
136
|
+
assert.equal(record.ts, "2026-11-06T12:34:56Z");
|
|
137
|
+
assert.deepEqual(record.items, empty ? [] : [{ kind: skill === "gatekeep-pr" ? "pr" : "ticket", key: "#42", outcome }]);
|
|
138
|
+
assert.equal(record.cost, 0.125);
|
|
139
|
+
assert.equal(record.tokens.input, 12);
|
|
140
|
+
const tip = h.git(h.origin, "rev-parse", ref);
|
|
141
|
+
await h.emit("agent_settled", { aborted: false });
|
|
142
|
+
await h.say(`gauntlet: done settled\n#43: ${outcome}`);
|
|
143
|
+
await h.emit("agent_settled", { aborted: false });
|
|
144
|
+
await h.emit("session_shutdown");
|
|
145
|
+
assert.equal(h.git(h.origin, "rev-parse", ref), tip, "duplicate lifecycle events do not append records");
|
|
146
|
+
assert.equal(h.git(h.root, "rev-parse", "HEAD"), h.head);
|
|
147
|
+
assert.deepEqual(readFileSync(join(h.root, ".git/index")), h.index);
|
|
148
|
+
assert.equal(readFileSync(join(h.root, "file.txt"), "utf8"), "unstaged user change\n");
|
|
149
|
+
assert.equal(h.git(h.root, "rev-parse", ref), tip);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { readFileSync, readdirSync } from "node:fs";
|
|
3
|
+
import test from "node:test";
|
|
4
|
+
import { settlementItems } from "./lib/telemetry-leg.ts";
|
|
5
|
+
import { MODELS } from "../eval/lib/models.mjs";
|
|
6
|
+
|
|
7
|
+
const read = (path: string) => readFileSync(new URL(`../${path}`, import.meta.url), "utf8");
|
|
8
|
+
const head = "abc123456789abcdef0123456789abcdef01234567";
|
|
9
|
+
const otherHead = "def123456789abcdef0123456789abcdef01234567";
|
|
10
|
+
|
|
11
|
+
for (const [skill, items] of [
|
|
12
|
+
["shape-ticket", [{ kind: "ticket", key: "#83", outcome: "written" }]],
|
|
13
|
+
["gatekeep-pr", [
|
|
14
|
+
{ kind: "pr", key: `#31@${head}`, ticket: "#12", outcome: "merged" },
|
|
15
|
+
{ kind: "pr", key: `#32@${otherHead}`, outcome: "stuck - CI pending past 15 minutes" },
|
|
16
|
+
]],
|
|
17
|
+
["check-delivery", [{ kind: "ticket", key: "#52", outcome: "delivered" }]],
|
|
18
|
+
] as const) {
|
|
19
|
+
test(`${skill} selected eval replies parse into settled item identities`, () => {
|
|
20
|
+
const result = JSON.parse(read(`eval/${skill}/results/per-ref-settlement/candidate.json`));
|
|
21
|
+
for (const model of MODELS.replay) {
|
|
22
|
+
const output = result.outputs[model];
|
|
23
|
+
assert.equal(output?.status, "ok", output?.error ?? model);
|
|
24
|
+
assert.match(output.text, /^gauntlet: (done|stuck)\b/);
|
|
25
|
+
assert.equal(output.text.match(/^gauntlet: (done|stuck)\b/gm)?.length, 1);
|
|
26
|
+
assert.deepEqual(settlementItems(skill, output.text), items, model);
|
|
27
|
+
assert.ok(Object.values(output.mechanical).every((value) => value === "holds"), model);
|
|
28
|
+
assert.ok(["improved", "unchanged"].includes(result.judge[model]?.verdict), model);
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
test("skill owners keep terminal rows after writes and cleanup", () => {
|
|
34
|
+
const shape = read("skills/shape-ticket/reference/repo-audit.md");
|
|
35
|
+
assert.match(shape, /actual ref returned by the write/);
|
|
36
|
+
assert.doesNotMatch(shape, /use the drafted title for create/);
|
|
37
|
+
assert.match(shape, /after every selected write/);
|
|
38
|
+
const delivery = read("skills/check-delivery/SKILL.md");
|
|
39
|
+
assert.match(delivery, /<ref>: <outcome>/);
|
|
40
|
+
assert.match(delivery, /already recorded/);
|
|
41
|
+
const gate = read("skills/gatekeep-pr/reference/post-selection-loop.md");
|
|
42
|
+
assert.ok(gate.includes("#<N>@<head sha>: <outcome>"));
|
|
43
|
+
assert.match(gate, /full assessed `headRefOid`/);
|
|
44
|
+
assert.match(gate, /cleanup pick is answered/);
|
|
45
|
+
assert.match(read("skills/gatekeep-pr/reference/batch.md"), /gauntlet: retired #<N> - <reason>/);
|
|
46
|
+
assert.match(read("skills/gatekeep-pr/reference/cleanup.md"), /### Settlement rows/);
|
|
47
|
+
const ci = read("scripts/ci.mjs");
|
|
48
|
+
assert.ok(ci.includes('includes("#<N>@<head sha>: <outcome>")'));
|
|
49
|
+
assert.ok(ci.includes("gatekeep-pr must not mention telemetry"));
|
|
50
|
+
for (const file of readdirSync(new URL("../skills/gatekeep-pr/", import.meta.url), { recursive: true })) {
|
|
51
|
+
if (file.endsWith(".md")) assert.doesNotMatch(read(`skills/gatekeep-pr/${file}`), /telemetry/i);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("configuration names the settlement grammar and per-skill tokens", () => {
|
|
56
|
+
const docs = read("doc/configuration.md");
|
|
57
|
+
assert.ok(docs.includes("<ref>: <outcome>[ (<ticket ref>)]"));
|
|
58
|
+
for (const tokens of [
|
|
59
|
+
["shape-ticket", "written", "closed", "close failed (<operation>: <reason>)", "close by hand", "dropped", "declined", "stopped", "no-op", "unfetched"],
|
|
60
|
+
["gatekeep-pr", "approved", "merged", "closed-pr", "final", "merge-queued", "stuck - <reason>"],
|
|
61
|
+
["check-delivery", "delivered", "manual", "declined", "already recorded", "skipped", "failed - <reason>"],
|
|
62
|
+
]) {
|
|
63
|
+
const row = docs.split("\n").find((line) => line.startsWith(`| ${tokens[0]} |`));
|
|
64
|
+
assert.ok(row, tokens[0]);
|
|
65
|
+
for (const token of tokens.slice(1)) assert.ok(row.includes(`\`${token}\``), token);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
type Handler = (event: any, ctx: any) => unknown;
|
|
22
22
|
|
|
23
|
-
function harness(o: { branch?: unknown[]; enabled?: boolean; gitFail?: (args: string[], cwd: string) => GitResult | undefined; jjWorkspace?: boolean; jjFail?: (args: string[], cwd: string) => GitResult | undefined; cwdSub?: string; sessionId?: string; model?: { provider: string; id: string }; thinkingLevel?: string; contextTokens?: number | null; telemetryWarning?: string
|
|
23
|
+
function harness(o: { branch?: unknown[]; enabled?: boolean; gitFail?: (args: string[], cwd: string) => GitResult | undefined; jjWorkspace?: boolean; jjFail?: (args: string[], cwd: string) => GitResult | undefined; cwdSub?: string; sessionId?: string; model?: { provider: string; id: string }; thinkingLevel?: string; contextTokens?: number | null; telemetryWarning?: string } = {}) {
|
|
24
24
|
const root = mkdtempSync(join(tmpdir(), "telemetry-test-"));
|
|
25
25
|
tempDirs.push(root);
|
|
26
26
|
mkdirSync(join(root, "doc/specs"), { recursive: true });
|
|
@@ -33,8 +33,11 @@
|
|
|
33
33
|
const jjCalls: string[][] = [];
|
|
34
34
|
const readFileCalls: string[] = [];
|
|
35
35
|
const specDirs = resolveFlowGuards({}).specDirs;
|
|
36
|
+
const fetchedRecords = new Map<string, string>();
|
|
36
37
|
let clock = Date.parse("2026-09-17T10:00:00Z");
|
|
37
38
|
const deps: Deps = {
|
|
39
|
+
telemetryRepo: () => "test/repo",
|
|
40
|
+
readTelemetryBranch: () => ({ records: [...fetchedRecords].map(([path, bytes]) => ({ path, bytes })) }),
|
|
38
41
|
fs: { ...realFs, readFile: (p) => { readFileCalls.push(p); return realFs.readFile(p); } },
|
|
39
42
|
now: () => new Date((clock += 1000)).toISOString().replace(/\.\d{3}Z$/, "Z"),
|
|
40
43
|
git: async (args, cwd) => {
|
|
@@ -58,7 +61,7 @@
|
|
|
58
61
|
if (args[0] === "ls-files") return { code: 0, stdout: "", stderr: "" };
|
|
59
62
|
return { code: 0, stdout: "", stderr: "" };
|
|
60
63
|
},
|
|
61
|
-
settings: () => ({ specDirs, planDirs: planDirsFor(specDirs), telemetry: { enabled: o.enabled ?? true,
|
|
64
|
+
settings: () => ({ specDirs, planDirs: planDirsFor(specDirs), telemetry: { enabled: o.enabled ?? true, buckets: DEFAULT_TELEMETRY_BUCKETS, warning: o.telemetryWarning }, errors: [], agentOverrides: { implementer: { model: "p/x" } }, versions: { pi: "0.85.1" } }),
|
|
62
65
|
// A plain jj workspace: `jj root` answers the temp root while git rev-parse fails.
|
|
63
66
|
jj: async (args, cwd) => {
|
|
64
67
|
jjCalls.push(args);
|
|
@@ -83,7 +86,7 @@
|
|
|
83
86
|
for (const h of handlers.get(name) ?? []) out.push(await h(event, ctx));
|
|
84
87
|
return out;
|
|
85
88
|
};
|
|
86
|
-
const recordPath = (spec = "doc/specs/a.md") => join(ctx.cwd, ".
|
|
89
|
+
const recordPath = (spec = "doc/specs/a.md") => join(ctx.cwd.replace(/\/\.worktrees\/.*$/, ""), o.jjWorkspace ? ".jj/gauntlet/telemetry" : ".git/gauntlet/telemetry", spec.replace(/\.md$/, ".yaml"));
|
|
87
90
|
const readRecord = (spec?: string) => parseRecord(readFileSync(recordPath(spec), "utf8"))!;
|
|
88
91
|
const gitWrites = () => gitCalls.filter((a) => a[0] === "add" || a[0] === "commit");
|
|
89
92
|
const excludeFile = () => join(root, ".git/info/exclude");
|
|
@@ -97,7 +100,7 @@
|
|
|
97
100
|
await emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: rel, content: body } });
|
|
98
101
|
return emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: rel, content: body }, content: [], isError: false, details: undefined });
|
|
99
102
|
};
|
|
100
|
-
return { root, handlers, emit, gitCalls, gitCwds, jjCalls, readFileCalls, gitWrites, excludeFile, bash, recordPath, readRecord, phaseResult, writeSpec, setBranch: (b: unknown[]) => (branch = b), ctx };
|
|
103
|
+
return { fetchedRecords, root, handlers, emit, gitCalls, gitCwds, jjCalls, readFileCalls, gitWrites, excludeFile, bash, recordPath, readRecord, phaseResult, writeSpec, setBranch: (b: unknown[]) => (branch = b), ctx };
|
|
101
104
|
}
|
|
102
105
|
|
|
103
106
|
const P = (over: Record<string, string> = {}) => Object.fromEntries(["brainstorm", "plan", "implement", "verify", "ship"].map((p) => [p, { status: over[p] ?? "pending" }]));
|
|
@@ -151,7 +154,7 @@
|
|
|
151
154
|
}
|
|
152
155
|
});
|
|
153
156
|
|
|
154
|
-
test("bind on first spec write flushes buffered phase event
|
|
157
|
+
test("bind on first spec write flushes buffered phase event and mints run_id without excludes, staging or commits", async () => {
|
|
155
158
|
const h = harness();
|
|
156
159
|
await h.emit("session_start", { type: "session_start", reason: "startup" });
|
|
157
160
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
@@ -175,19 +178,19 @@
|
|
|
175
178
|
const rec2 = h.readRecord();
|
|
176
179
|
assert.equal(rec2.approved_at, rec2.events.findLast((e) => e.kind === "phase")!.ts);
|
|
177
180
|
assert.equal(h.gitWrites().length, 0, "the recorder never runs git add or git commit");
|
|
178
|
-
assert.equal(
|
|
181
|
+
assert.equal(existsSync(h.excludeFile()), false);
|
|
179
182
|
await h.phaseResult("reset", P());
|
|
180
183
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
181
184
|
await h.writeSpec("doc/specs/a.md", "# Spec again\n");
|
|
182
|
-
assert.equal(
|
|
185
|
+
assert.equal(existsSync(h.excludeFile()), false, "rebind never edits excludes");
|
|
183
186
|
});
|
|
184
187
|
|
|
185
|
-
test("a docs/specs write binds under the
|
|
188
|
+
test("a docs/specs write binds under the common-dir in-progress store", async () => {
|
|
186
189
|
const h = harness();
|
|
187
190
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
188
191
|
assert.equal(existsSync(h.recordPath("docs/specs/x.md")), false);
|
|
189
192
|
await h.writeSpec("docs/specs/x.md");
|
|
190
|
-
assert.equal(h.recordPath("docs/specs/x.md"), join(h.root, ".
|
|
193
|
+
assert.equal(h.recordPath("docs/specs/x.md"), join(h.root, ".git/gauntlet/telemetry/docs/specs/x.yaml"));
|
|
191
194
|
assert.equal(h.readRecord("docs/specs/x.md").spec, "docs/specs/x.md");
|
|
192
195
|
await h.phaseResult("complete", P({ brainstorm: "complete" }));
|
|
193
196
|
const rec = h.readRecord("docs/specs/x.md");
|
|
@@ -359,7 +362,7 @@
|
|
|
359
362
|
const rec = h.readRecord("doc/specs/draft.md");
|
|
360
363
|
assert.equal(rec.spec, "doc/specs/draft.md");
|
|
361
364
|
assert.equal(readFileSync(h.recordPath("doc/specs/final.md"), "utf8"), "existing\n");
|
|
362
|
-
assert.deepEqual(rec.events.filter((e) => e.kind === "warning").map((e: any) => e.message), [
|
|
365
|
+
assert.deepEqual(rec.events.filter((e) => e.kind === "warning").map((e: any) => e.message), [`record move skipped: ${h.recordPath("doc/specs/final.md")} already exists`]);
|
|
363
366
|
});
|
|
364
367
|
|
|
365
368
|
test("rename refuses a tracked destination collision and keeps the draft binding", async () => {
|
|
@@ -374,9 +377,9 @@
|
|
|
374
377
|
assert.ok(rec.events.some((e) => e.kind === "warning" && (e as any).message.endsWith("already exists")));
|
|
375
378
|
});
|
|
376
379
|
|
|
377
|
-
test("telemetry resolver warning is emitted once
|
|
378
|
-
const message = "telemetry.
|
|
379
|
-
const h = harness({ telemetryWarning: message
|
|
380
|
+
test("telemetry resolver warning is emitted once", async () => {
|
|
381
|
+
const message = "telemetry.remote must be a non-empty URL; using origin";
|
|
382
|
+
const h = harness({ telemetryWarning: message });
|
|
380
383
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
381
384
|
await h.writeSpec("doc/specs/a.md");
|
|
382
385
|
await h.writeSpec("doc/specs/a.md", "# Again\n");
|
|
@@ -409,11 +412,13 @@
|
|
|
409
412
|
assert.deepEqual(nogit.readRecord().events.filter((e) => e.kind === "warning").map((e: any) => e.message), [`spec ${outsideSpec} is outside a git checkout; telemetry not recorded`]);
|
|
410
413
|
});
|
|
411
414
|
|
|
412
|
-
test("failed git-common-dir lookup
|
|
415
|
+
test("failed git-common-dir lookup never falls back to writing inside the checkout", async () => {
|
|
413
416
|
const h = harness({ gitFail: (args) => args.includes("--git-common-dir") && !args.includes("--show-toplevel") ? { code: 1, stdout: "", stderr: "unsupported flag\nmore detail" } : undefined });
|
|
414
417
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
415
418
|
await h.writeSpec("doc/specs/a.md");
|
|
416
|
-
assert.
|
|
419
|
+
assert.equal(existsSync(h.recordPath()), false);
|
|
420
|
+
assert.equal(existsSync(join(h.root, ".pi/gauntlet")), false);
|
|
421
|
+
assert.equal(existsSync(h.excludeFile()), false);
|
|
417
422
|
});
|
|
418
423
|
|
|
419
424
|
test("jj-bound checkout: record is written with no warning, no exclude line, and no git add/commit", async () => {
|
|
@@ -438,27 +443,27 @@
|
|
|
438
443
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
439
444
|
await h.writeSpec("doc/specs/a.md");
|
|
440
445
|
const wt = join(h.root, ".worktrees/x");
|
|
441
|
-
const sealedPath =
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
const before = readFileSync(sealedPath, "utf8");
|
|
446
|
+
const sealedPath = "test/repo/brainstorming/doc/specs/b.yaml";
|
|
447
|
+
const before = serializeRecord({ ...h.readRecord(), spec: "doc/specs/b.md", status: "shipped", shipped_at: "2026-09-17T18:00:00Z" });
|
|
448
|
+
h.fetchedRecords.set(sealedPath, before);
|
|
445
449
|
await h.emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: join(wt, "doc/specs/b.md"), content: "# B\n" }, content: [], isError: false, details: undefined });
|
|
446
450
|
await h.phaseResult("complete", P({ brainstorm: "complete" }));
|
|
447
451
|
assert.ok(h.readRecord().events.some((e) => e.kind === "phase" && e.action === "complete"));
|
|
448
|
-
assert.equal(
|
|
452
|
+
assert.equal(h.fetchedRecords.get(sealedPath), before);
|
|
449
453
|
});
|
|
450
454
|
|
|
451
|
-
test("worktree spec from a primary cwd
|
|
455
|
+
test("worktree spec from a primary cwd uses the common-dir store without changing excludes", async () => {
|
|
452
456
|
const h = harness();
|
|
453
457
|
await h.emit("session_start", { type: "session_start", reason: "startup" });
|
|
454
458
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
455
459
|
await h.writeSpec(".worktrees/x/doc/specs/a.md");
|
|
456
460
|
const wt = join(h.root, ".worktrees/x");
|
|
457
|
-
assert.ok(existsSync(
|
|
461
|
+
assert.ok(existsSync(h.recordPath()), "record lives under the shared git common dir");
|
|
462
|
+
assert.equal(existsSync(join(wt, ".pi/gauntlet")), false);
|
|
458
463
|
assert.equal(existsSync(join(h.root, ".pi/gauntlet")), false, "nothing written under the primary checkout");
|
|
459
464
|
assert.ok(h.gitCwds.every((c) => c === wt || c.startsWith(wt + "/")), "no git call runs in the primary");
|
|
460
465
|
assert.equal(h.gitWrites().length, 0);
|
|
461
|
-
assert.equal(
|
|
466
|
+
assert.equal(existsSync(h.excludeFile()), false);
|
|
462
467
|
});
|
|
463
468
|
|
|
464
469
|
test("a spec in another checkout is a fresh bind, never a git mv", async () => {
|
|
@@ -473,7 +478,7 @@
|
|
|
473
478
|
await h.emit("tool_result", { toolName: "edit", toolCallId: "e", input: { path: "doc/specs/a.md", oldText: "# Old\n", newText: "# Spec\n" }, content: [], isError: false, details: undefined });
|
|
474
479
|
}
|
|
475
480
|
assert.ok(existsSync(h.recordPath("doc/specs/a.md")), `primary record created for ${tool}`);
|
|
476
|
-
assert.
|
|
481
|
+
assert.equal(existsSync(join(h.root, ".worktrees/x/.pi/gauntlet")), false, `no worktree-local record for ${tool}`);
|
|
477
482
|
assert.equal(h.gitCalls.some((a) => a[0] === "mv"), false);
|
|
478
483
|
assert.equal(h.readRecord("doc/specs/a.md").events.some((e) => e.kind === "spec_renamed"), false);
|
|
479
484
|
}
|
|
@@ -973,9 +978,10 @@
|
|
|
973
978
|
const before = readFileSync(h.recordPath(), "utf8");
|
|
974
979
|
const h2 = harness();
|
|
975
980
|
h2.ctx.cwd = h.root;
|
|
981
|
+
h2.fetchedRecords.set("test/repo/brainstorming/doc/specs/a.yaml", before);
|
|
976
982
|
await h2.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
977
983
|
const [blocked] = await h2.emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: "doc/specs/a.md" } });
|
|
978
|
-
assert.deepEqual(blocked, { block: true, reason: guardReason("doc/specs/a.md", rec.shipped_at, "
|
|
984
|
+
assert.deepEqual(blocked, { block: true, reason: guardReason("doc/specs/a.md", rec.shipped_at, "gauntlet/telemetry") });
|
|
979
985
|
assert.equal(readFileSync(h.recordPath(), "utf8"), before);
|
|
980
986
|
assert.equal(h.readRecord().derived.spec_edits_after_ship, 0);
|
|
981
987
|
});
|
|
@@ -989,8 +995,10 @@
|
|
|
989
995
|
const before = readFileSync(first.recordPath("doc/specs/old.md"), "utf8");
|
|
990
996
|
const h = harness({ sessionId: "s2" });
|
|
991
997
|
h.ctx.cwd = first.root;
|
|
998
|
+
h.fetchedRecords.set("test/repo/brainstorming/doc/specs/old.yaml", before);
|
|
992
999
|
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
993
1000
|
await h.writeSpec("doc/specs/old.md");
|
|
1001
|
+
await h.emit("tool_result", { toolName: "plan_check", input: {}, content: [], isError: false, details: { status: "pass", specPath: join(first.root, "doc/specs/old.md") } });
|
|
994
1002
|
assert.equal(readFileSync(first.recordPath("doc/specs/old.md"), "utf8"), before);
|
|
995
1003
|
assert.equal(existsSync(h.excludeFile()), false);
|
|
996
1004
|
await h.writeSpec("doc/specs/new.md");
|
|
@@ -1090,7 +1098,7 @@
|
|
|
1090
1098
|
assert.equal(h.readRecord().status, "in_progress", "git keep path no longer stamps; the seal bin owns shipped_at");
|
|
1091
1099
|
});
|
|
1092
1100
|
|
|
1093
|
-
test("a
|
|
1101
|
+
test("a stamped pending record freezes the recorder on shutdown", async () => {
|
|
1094
1102
|
const h = await boundInShip();
|
|
1095
1103
|
const sealed = { ...h.readRecord(), status: "shipped" as const, shipped_at: "2026-09-17T18:00:00Z" };
|
|
1096
1104
|
writeFileSync(h.recordPath(), serializeRecord(sealed));
|
|
@@ -1100,6 +1108,14 @@
|
|
|
1100
1108
|
assert.equal(readFileSync(h.recordPath(), "utf8"), before);
|
|
1101
1109
|
});
|
|
1102
1110
|
|
|
1111
|
+
test("a branch-owned record freezes the recorder on shutdown", async () => {
|
|
1112
|
+
const h = await boundInShip();
|
|
1113
|
+
rmSync(h.recordPath());
|
|
1114
|
+
await h.emit("session_compact", { type: "session_compact", reason: "manual" });
|
|
1115
|
+
await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" });
|
|
1116
|
+
assert.equal(existsSync(h.recordPath()), false, "flush must not recreate a published record");
|
|
1117
|
+
});
|
|
1118
|
+
|
|
1103
1119
|
test("a frozen record cannot bind another spec through plan_check or another checkout write", async () => {
|
|
1104
1120
|
const h = await boundInShip();
|
|
1105
1121
|
writeFileSync(h.recordPath(), serializeRecord({ ...h.readRecord(), status: "shipped", shipped_at: "2026-09-17T18:00:00Z" }));
|
|
@@ -1294,9 +1310,10 @@
|
|
|
1294
1310
|
const before = readFileSync(h.recordPath(), "utf8");
|
|
1295
1311
|
const h2 = harness();
|
|
1296
1312
|
h2.ctx.cwd = h.root;
|
|
1313
|
+
h2.fetchedRecords.set("test/repo/brainstorming/doc/specs/a.yaml", before);
|
|
1297
1314
|
await h2.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
1298
1315
|
const [blocked] = await h2.emit("tool_call", { toolName: "write", toolCallId: "w1", input: { path: "doc/specs/a.md", content: "# again" } });
|
|
1299
|
-
assert.deepEqual(blocked, { block: true, reason: guardReason("doc/specs/a.md", h.readRecord().shipped_at!, "
|
|
1316
|
+
assert.deepEqual(blocked, { block: true, reason: guardReason("doc/specs/a.md", h.readRecord().shipped_at!, "gauntlet/telemetry") });
|
|
1300
1317
|
assert.equal(readFileSync(h.recordPath(), "utf8"), before, "shipped record byte-identical");
|
|
1301
1318
|
const [editPass] = await h2.emit("tool_call", { toolName: "edit", toolCallId: "e1", input: { path: "doc/specs/a.md", edits: [] } });
|
|
1302
1319
|
assert.equal(editPass, undefined);
|