pi-gauntlet 7.15.0 → 7.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +2 -1
- package/bin/gauntlet-performance.mjs +24 -3
- package/bin/gauntlet-performance.test.mjs +64 -13
- package/bin/gauntlet-telemetry-seal.mjs +1 -1
- package/extensions/lib/gauntlet-settings.test.ts +26 -0
- package/extensions/lib/gauntlet-settings.ts +20 -0
- package/extensions/lib/telemetry-record.test.ts +18 -0
- package/extensions/lib/telemetry-record.ts +2 -2
- package/extensions/phase-tracker.test.ts +150 -2
- package/extensions/phase-tracker.ts +66 -1
- package/extensions/telemetry.test.ts +19 -0
- package/extensions/telemetry.ts +5 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v7.16.0 - 2026-10-18
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- **Per-phase parent model switch, cache-warming warning, and per-call cache telemetry.** `piGauntlet.phaseModels` (`plan`, `implement`, `verify`, `ship` -> `provider/id[:level]`) switches the parent session's model and thinking level persistently when that phase starts; an unresolvable model, unknown `:level`, or missing auth prints one warning in the `phase_tracker` result and leaves the session unchanged. `start implement` warns when the active model declares no `promptCache` lifetime for the active retention tier, so pi's cache warmer never runs. Telemetry records one `[cache_read, cache_write, cache_write_1h]` triplet per parent call (`parent_calls`), and `gauntlet-performance` gains a `big` column with the count and share of >50k cache-write parent calls (#67).
|
|
8
|
+
|
|
3
9
|
## v7.15.0 - 2026-10-18
|
|
4
10
|
|
|
5
11
|
### Changed
|
package/README.md
CHANGED
|
@@ -150,7 +150,7 @@ Pin an exact release with `npm:pi-gauntlet@X.Y.Z`. See [doc/install-internals.md
|
|
|
150
150
|
|
|
151
151
|
## Performance digest
|
|
152
152
|
|
|
153
|
-
`gauntlet-performance` digests telemetry records without judging them: `node <pi-gauntlet-package>/bin/gauntlet-performance.mjs [--dir <repo root or telemetry dir>]... [--since <version>] [--json]` prints a `corpus:` line, one row per run (`run_id`, repo, spec, version, status - `shipped*` when the record has no ship phase - wall, per-phase minutes, tokens, cost, models, dispatches, `fix_round_grants`, reopens, conformance loops, reviewer findings, council dispatches), a `by version` block of p50/max over shipped, non-truncated runs, a `council` block grouped by member roster, and `skipped:` lines for files it could not use. Each council table shows member `total`, `unique`, `applied`, `uniq_appl`, `deferred`, `rejected` blocker/major/minor counts and `uniq_appl_nonminor/run`; `chair:` shows runs, average clusters and members reported, and retried runs. Roster flags (JSON `flags[].rule`: `low_unique_applied` - fewer than 0.2 unique-and-applied blocker/major findings per run; `high_rejection` - over 50% rejected while another member is below 25%) require five runs and print as `flag: <member> - <detail>`. Smaller groups print `not enough runs to assess (N of 5)`; absent council data prints `no council data in selected records`. JSON includes `council: [{ roster, runs, members, chairs, flags }]`. Default corpus is the current repo's telemetry dir; each `--dir` adds a repo root (its own `telemetry.dir` setting is honoured) or a bare telemetry dir. `--since` compares versions numerically. Exit 0 always except usage errors. `/skill:gauntlet-performance` is the reasoning layer on top.
|
|
153
|
+
`gauntlet-performance` digests telemetry records without judging them: `node <pi-gauntlet-package>/bin/gauntlet-performance.mjs [--dir <repo root or telemetry dir>]... [--since <version>] [--json]` prints a `corpus:` line, one row per run (`run_id`, repo, spec, version, status - `shipped*` when the record has no ship phase - wall, per-phase minutes, tokens, cost, models, dispatches, `fix_round_grants`, reopens, conformance loops, reviewer findings, council dispatches, and `big` - the count of parent calls whose cache write exceeded 50k tokens with their share of the run's parent cache-write tokens (`-` for records without `parent_calls`)), a `by version` block of p50/max over shipped, non-truncated runs, a `council` block grouped by member roster, and `skipped:` lines for files it could not use. Each council table shows member `total`, `unique`, `applied`, `uniq_appl`, `deferred`, `rejected` blocker/major/minor counts and `uniq_appl_nonminor/run`; `chair:` shows runs, average clusters and members reported, and retried runs. Roster flags (JSON `flags[].rule`: `low_unique_applied` - fewer than 0.2 unique-and-applied blocker/major findings per run; `high_rejection` - over 50% rejected while another member is below 25%) require five runs and print as `flag: <member> - <detail>`. Smaller groups print `not enough runs to assess (N of 5)`; absent council data prints `no council data in selected records`. JSON includes `council: [{ roster, runs, members, chairs, flags }]`. Default corpus is the current repo's telemetry dir; each `--dir` adds a repo root (its own `telemetry.dir` setting is honoured) or a bare telemetry dir. `--since` compares versions numerically. Exit 0 always except usage errors. `/skill:gauntlet-performance` is the reasoning layer on top.
|
|
154
154
|
|
|
155
155
|
For local development against a checkout instead of npm:
|
|
156
156
|
|
|
@@ -532,6 +532,7 @@ The conformance gate's model, the spec council's roster, and the phase-tracker's
|
|
|
532
532
|
| `closureReview.model` | phase-tracker | unset (inherits) | conformance reviewer model - [detail](./doc/configuration.md#conformance-gate-model) |
|
|
533
533
|
| `closureReview.maxFixRounds` | phase-tracker | `3` | fix rounds before escalation - [detail](./doc/configuration.md#conformance-gate-model) |
|
|
534
534
|
| `escalationLoop.implModel` | phase-tracker | unset (main loop) | implementer model on escalation - [detail](./doc/configuration.md#escalation-loop) |
|
|
535
|
+
| `phaseModels.plan` / `.implement` / `.verify` / `.ship` | phase-tracker | unset (no switch) | `provider/id[:level]` applied persistently at that phase's start - [detail](./doc/configuration.md#phase-models) |
|
|
535
536
|
| `flowGuards.enforce` | phase-tracker | `true` | phase gates on/off - [detail](./doc/configuration.md#phase-tracker) |
|
|
536
537
|
| `flowGuards.specDirs` | phase-tracker, plan-tracker; telemetry and spec-index consumers | `["doc/specs", "docs/specs"]` | where specs and sibling plans live - [detail](./doc/configuration.md#phase-tracker) |
|
|
537
538
|
| `verifyBeforeShip.testCommands` | verify-before-ship | the `DEFAULT_TEST_COMMANDS` patterns in `extensions/lib/gauntlet-settings.ts` (make, npm, pnpm, yarn, pytest, rspec, cargo, go) | patterns that count as a test run - [detail](./doc/configuration.md#verify-before-ship) |
|
|
@@ -51,6 +51,7 @@ function resolveTelemetry(g) {
|
|
|
51
51
|
// src/bins/performance/shared.mjs
|
|
52
52
|
var PHASES = ["brainstorm", "plan", "implement", "verify", "ship"];
|
|
53
53
|
var TOKEN_KEYS = ["input", "output", "cache_read", "cache_write"];
|
|
54
|
+
var BIG_CACHE_WRITE = 5e4;
|
|
54
55
|
var semver = (s) => {
|
|
55
56
|
const m = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.exec(typeof s === "string" ? s.trim() : "");
|
|
56
57
|
return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : void 0;
|
|
@@ -91,7 +92,7 @@ __export(runs_exports, {
|
|
|
91
92
|
renderText: () => renderText
|
|
92
93
|
});
|
|
93
94
|
var key = "runs";
|
|
94
|
-
var HEADER = ["run_id", "repo", "spec", "version", "status", "wall", "b/p/i/v/s min", "tokens", "cost", "models", "disp", "grants", "reopens", "loops", "findings", "council"];
|
|
95
|
+
var HEADER = ["run_id", "repo", "spec", "version", "status", "wall", "b/p/i/v/s min", "tokens", "cost", "models", "disp", "grants", "reopens", "loops", "findings", "council", "big"];
|
|
95
96
|
var runRow = (r) => [
|
|
96
97
|
r.run_id.slice(0, 8),
|
|
97
98
|
r.repo,
|
|
@@ -108,7 +109,8 @@ var runRow = (r) => [
|
|
|
108
109
|
dash(r.reopens),
|
|
109
110
|
dash(r.loops),
|
|
110
111
|
findingsCell(r.findings),
|
|
111
|
-
dash(r.council)
|
|
112
|
+
dash(r.council),
|
|
113
|
+
r.big_calls === null ? dash(null) : `${r.big_calls}/${r.big_write_share === null ? "-" : `${Math.round(r.big_write_share * 100)}%`}`
|
|
112
114
|
];
|
|
113
115
|
var aggregate = (entries) => entries.map((e) => e.row);
|
|
114
116
|
var renderText = (rows) => [`runs (${rows.length})`, table(HEADER, rows.map(runRow))].join("\n");
|
|
@@ -339,6 +341,22 @@ function loadRecord(file, label) {
|
|
|
339
341
|
const phases = obj(derived.phases) ?? {};
|
|
340
342
|
const ph = (p) => obj(phases[p]);
|
|
341
343
|
const tok = (p) => obj(ph(p)?.tokens);
|
|
344
|
+
const callsOf = (p) => {
|
|
345
|
+
const raw = ph(p)?.parent_calls;
|
|
346
|
+
if (!Array.isArray(raw)) return null;
|
|
347
|
+
return raw.filter((c) => Array.isArray(c) && c.length === 3 && c.every((v) => num(v) !== null));
|
|
348
|
+
};
|
|
349
|
+
const bigStats = (calls) => {
|
|
350
|
+
if (calls === null) return { big_calls: null, big_write_share: null };
|
|
351
|
+
const total = calls.reduce((a, c) => a + c[1], 0);
|
|
352
|
+
const bigs = calls.filter((c) => c[1] > BIG_CACHE_WRITE);
|
|
353
|
+
const big = bigs.reduce((a, c) => a + c[1], 0);
|
|
354
|
+
return { big_calls: bigs.length, big_write_share: total === 0 ? null : big / total };
|
|
355
|
+
};
|
|
356
|
+
const phaseKeys = [...PHASES, "unphased"];
|
|
357
|
+
const perPhase = Object.fromEntries(phaseKeys.map((p) => [p, callsOf(p)]).filter(([, c]) => c !== null).map(([p, c]) => [p, bigStats(c)]));
|
|
358
|
+
const allCalls = phaseKeys.map(callsOf).filter((c) => c !== null);
|
|
359
|
+
const runBig = allCalls.length ? bigStats(allCalls.flat()) : { big_calls: null, big_write_share: null };
|
|
342
360
|
const personas = obj(derived.personas) ?? {};
|
|
343
361
|
const reviews = obj(derived.reviews);
|
|
344
362
|
const gates = obj(derived.gates) ?? {};
|
|
@@ -383,7 +401,10 @@ function loadRecord(file, label) {
|
|
|
383
401
|
findings,
|
|
384
402
|
council: num(obj(personas["spec-council-member"])?.dispatches),
|
|
385
403
|
ship_option: str(gates.ship_option),
|
|
386
|
-
tests: str(obj(derived.tests)?.result)
|
|
404
|
+
tests: str(obj(derived.tests)?.result),
|
|
405
|
+
big_calls: runBig.big_calls,
|
|
406
|
+
big_write_share: runBig.big_write_share,
|
|
407
|
+
phases: Object.keys(perPhase).length ? perPhase : void 0
|
|
387
408
|
},
|
|
388
409
|
council: derived.council ?? null
|
|
389
410
|
};
|
|
@@ -123,6 +123,18 @@ const REVIEW_WITHOUT_FINDINGS = IN_PROGRESS
|
|
|
123
123
|
.replace("spec: doc/specs/c.md", "spec: doc/specs/h.md")
|
|
124
124
|
.replace(" conformance_loops: 0", " reviews: { code-reviewer: { dispatches: 1, nonzero_exit: 0 } }\n conformance_loops: 0");
|
|
125
125
|
|
|
126
|
+
const PARENT_CALLS = COMPLETE
|
|
127
|
+
.replace("run_id: aaaaaaaa-0000-0000-0000-000000000001", "run_id: iiiiiiii-0000-0000-0000-000000000009")
|
|
128
|
+
.replace("spec: doc/specs/a.md", "spec: doc/specs/i.md")
|
|
129
|
+
.replace("created_at: 2026-09-01T00:00:00Z", "created_at: 2026-09-06T00:00:00Z")
|
|
130
|
+
.replace("plan: " + phase(`model: m/alpha, ${tokens(50, 5, 1000, 100, 0.5)}`),
|
|
131
|
+
"plan: " + phase(`model: m/alpha, ${tokens(50, 5, 1000, 100, 0.5)}, parent_calls: [[1, 60000, 60000], [1, 100, 0]]`))
|
|
132
|
+
.replace("implement: " + phase(`model: m/beta, ${tokens(10, 1, 100, 10, 0.25)}`),
|
|
133
|
+
"implement: " + phase(`model: m/beta, ${tokens(10, 1, 100, 10, 0.25)}, parent_calls: [[1, 70000, 0], [1, 200, 0], [1, 300, 0]]`))
|
|
134
|
+
.replace(`unphased: { ${tokens(99999, 99999, 99999, 99999, 999)} }`,
|
|
135
|
+
`unphased: { ${tokens(99999, 99999, 99999, 99999, 999)}, parent_calls: [[1, 50000, 0]] }`)
|
|
136
|
+
.replace("cache_write: 100, cost: 0.5", "cache_write: 100, cache_write_1h: 60000, cost: 0.5");
|
|
137
|
+
|
|
126
138
|
const corpus = (root) => {
|
|
127
139
|
write(root, `${TDIR}/a.yaml`, COMPLETE);
|
|
128
140
|
write(root, `${TDIR}/b.yaml`, TRUNCATED);
|
|
@@ -130,6 +142,7 @@ const corpus = (root) => {
|
|
|
130
142
|
write(root, `${TDIR}/d.yaml`, COMPLETE_2);
|
|
131
143
|
write(root, `${TDIR}/e.yaml`, OLD_VERSION);
|
|
132
144
|
write(root, `${TDIR}/f.yaml`, BAD_GATES);
|
|
145
|
+
write(root, `${TDIR}/i.yaml`, PARENT_CALLS);
|
|
133
146
|
write(root, `${TDIR}/present-empty-reviews.yaml`, EMPTY_REVIEWS);
|
|
134
147
|
write(root, `${TDIR}/broken.yaml`, "schema: 1\nspec: [\n");
|
|
135
148
|
write(root, `${TDIR}/v2.yaml`, "schema: 2\nspec: doc/specs/z.md\nrun_id: z\n");
|
|
@@ -141,7 +154,7 @@ const byId = (runs, prefix) => runs.find((r) => r.run_id.startsWith(prefix));
|
|
|
141
154
|
test("per-run fields: typed reads, unphased excluded, truncated wall null, findings null vs 0/0/0", (t) => {
|
|
142
155
|
const root = corpus(gitRepo()); cleanup(t, root);
|
|
143
156
|
const d = json(root);
|
|
144
|
-
assert.deepEqual(d.corpus, { [root.split("/").pop()]:
|
|
157
|
+
assert.deepEqual(d.corpus, { [root.split("/").pop()]: 8 });
|
|
145
158
|
assert.equal(d.since, null);
|
|
146
159
|
const a = byId(d.runs, "aaaaaaaa");
|
|
147
160
|
assert.equal(a.repo, root.split("/").pop());
|
|
@@ -184,6 +197,44 @@ test("per-run fields: typed reads, unphased excluded, truncated wall null, findi
|
|
|
184
197
|
assert.deepEqual(emptyReviews.findings, { blocker: 0, major: 0, minor: 0 });
|
|
185
198
|
});
|
|
186
199
|
|
|
200
|
+
test("parent_calls: big counts and parent-only shares include unphased without double-counting tokens", (t) => {
|
|
201
|
+
const root = corpus(gitRepo()); cleanup(t, root);
|
|
202
|
+
const i = byId(json(root).runs, "iiiiiiii");
|
|
203
|
+
assert.equal(i.big_calls, 2);
|
|
204
|
+
assert.equal(i.big_write_share, (60000 + 70000) / (60000 + 100 + 70000 + 200 + 300 + 50000));
|
|
205
|
+
assert.deepEqual(i.phases.plan, { big_calls: 1, big_write_share: 60000 / 60100 });
|
|
206
|
+
assert.deepEqual(i.phases.implement, { big_calls: 1, big_write_share: 70000 / 70500 });
|
|
207
|
+
assert.deepEqual(i.phases.unphased, { big_calls: 0, big_write_share: 0 });
|
|
208
|
+
assert.equal(i.tokens, 100 + 10 + 5000 + 200 + 50 + 5 + 1000 + 100 + 10 + 1 + 100 + 10);
|
|
209
|
+
const row = invoke(root).stdout.split("\n").find((l) => l.startsWith("iiiiiiii"));
|
|
210
|
+
assert.match(row, /\b2\/72%/);
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test("parent_calls with zero cache writes: null shares and text 0/-", (t) => {
|
|
214
|
+
const root = gitRepo(); cleanup(t, root);
|
|
215
|
+
write(root, `${TDIR}/zero.yaml`, `schema: 1
|
|
216
|
+
spec: doc/specs/zero.md
|
|
217
|
+
run_id: zero0000
|
|
218
|
+
derived: { phases: { plan: { parent_calls: [[5, 0, 0]] } } }
|
|
219
|
+
`);
|
|
220
|
+
const run = json(root).runs[0];
|
|
221
|
+
assert.equal(run.big_calls, 0);
|
|
222
|
+
assert.equal(run.big_write_share, null);
|
|
223
|
+
assert.deepEqual(run.phases.plan, { big_calls: 0, big_write_share: null });
|
|
224
|
+
const row = invoke(root).lines.find((line) => line.startsWith("zero0000"));
|
|
225
|
+
assert.match(row, /\s0\/-$/);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
test("parent_calls absent: JSON nulls and text dash", (t) => {
|
|
229
|
+
const root = corpus(gitRepo()); cleanup(t, root);
|
|
230
|
+
const a = byId(json(root).runs, "aaaaaaaa");
|
|
231
|
+
assert.equal(a.big_calls, null);
|
|
232
|
+
assert.equal(a.big_write_share, null);
|
|
233
|
+
assert.equal(a.phases, undefined);
|
|
234
|
+
const row = invoke(root).stdout.split("\n").find((l) => l.startsWith("aaaaaaaa"));
|
|
235
|
+
assert.match(row, /\s-\s*$/);
|
|
236
|
+
});
|
|
237
|
+
|
|
187
238
|
test("review entries without findings contribute zero counters", (t) => {
|
|
188
239
|
const root = gitRepo(); cleanup(t, root);
|
|
189
240
|
write(root, `${TDIR}/review-without-findings.yaml`, REVIEW_WITHOUT_FINDINGS);
|
|
@@ -201,7 +252,7 @@ test("malformed findings counters remain null in JSON and render as dashes in te
|
|
|
201
252
|
const d = json(root);
|
|
202
253
|
assert.deepEqual(d.runs[0].findings, { blocker: null, major: 2, minor: 3 });
|
|
203
254
|
const row = invoke(root).lines.find((line) => line.startsWith("aaaaaaaa"));
|
|
204
|
-
assert.match(row, /\s-\/2\/3\s+4
|
|
255
|
+
assert.match(row, /\s-\/2\/3\s+4\s+-$/);
|
|
205
256
|
});
|
|
206
257
|
|
|
207
258
|
test("malformed reviewer entries and findings objects contribute null counters", (t) => {
|
|
@@ -237,14 +288,14 @@ test("aggregates: only shipped non-truncated rows feed p50/max; even-count p50 i
|
|
|
237
288
|
const d = json(root);
|
|
238
289
|
assert.deepEqual(d.by_version.map((g) => g.version), ["5.9.1", "5.10.0", "5.11.0", "unknown"]);
|
|
239
290
|
const v591 = d.by_version[0];
|
|
240
|
-
assert.equal(v591.n,
|
|
241
|
-
assert.equal(v591.shipped,
|
|
291
|
+
assert.equal(v591.n, 4);
|
|
292
|
+
assert.equal(v591.shipped, 3);
|
|
242
293
|
assert.equal(v591.truncated, 1);
|
|
243
|
-
assert.deepEqual(v591.wall_s, { p50:
|
|
244
|
-
assert.deepEqual(v591.grants, { p50:
|
|
294
|
+
assert.deepEqual(v591.wall_s, { p50: 3600, max: 7200 });
|
|
295
|
+
assert.deepEqual(v591.grants, { p50: 1, max: 3 });
|
|
245
296
|
assert.deepEqual(v591.dispatches, { p50: 7 });
|
|
246
297
|
assert.deepEqual(v591.findings, { blocker: { p50: 1 }, major: { p50: 2 }, minor: { p50: 3 } });
|
|
247
|
-
assert.deepEqual(v591.models, { "m/alpha":
|
|
298
|
+
assert.deepEqual(v591.models, { "m/alpha": 3, "m/beta": 3 });
|
|
248
299
|
const v5110 = d.by_version[2];
|
|
249
300
|
assert.equal(v5110.n, 3);
|
|
250
301
|
assert.equal(v5110.shipped, 0);
|
|
@@ -256,7 +307,7 @@ test("aggregates: only shipped non-truncated rows feed p50/max; even-count p50 i
|
|
|
256
307
|
test("row order is created_at then run_id; skipped lists unparseable, schema, not a record", (t) => {
|
|
257
308
|
const root = corpus(gitRepo()); cleanup(t, root);
|
|
258
309
|
const d = json(root);
|
|
259
|
-
assert.deepEqual(d.runs.map((r) => r.run_id[0]), ["a", "e", "b", "c", "f", "d", "g"]);
|
|
310
|
+
assert.deepEqual(d.runs.map((r) => r.run_id[0]), ["a", "e", "b", "c", "f", "d", "g", "i"]);
|
|
260
311
|
const reasons = Object.fromEntries(d.skipped.map((s) => [s.file.split("/").pop(), s.reason]));
|
|
261
312
|
assert.equal(reasons["broken.yaml"], "unparseable");
|
|
262
313
|
assert.equal(reasons["v2.yaml"], "schema 2");
|
|
@@ -267,17 +318,17 @@ test("text output: header, shipped* marker, - cells, by version block, skipped l
|
|
|
267
318
|
const root = corpus(gitRepo()); cleanup(t, root);
|
|
268
319
|
const r = invoke(root);
|
|
269
320
|
assert.equal(r.status, 0, r.stderr);
|
|
270
|
-
assert.equal(r.lines[0], `corpus: ${root.split("/").pop()}=
|
|
271
|
-
assert.equal(r.lines[1], "runs (
|
|
272
|
-
assert.match(r.lines[2], /^run_id\s+repo\s+spec\s+version\s+status\s+wall\s+b\/p\/i\/v\/s min\s+tokens\s+cost\s+models\s+disp\s+grants\s+reopens\s+loops\s+findings\s+council$/);
|
|
321
|
+
assert.equal(r.lines[0], `corpus: ${root.split("/").pop()}=8`);
|
|
322
|
+
assert.equal(r.lines[1], "runs (8)");
|
|
323
|
+
assert.match(r.lines[2], /^run_id\s+repo\s+spec\s+version\s+status\s+wall\s+b\/p\/i\/v\/s min\s+tokens\s+cost\s+models\s+disp\s+grants\s+reopens\s+loops\s+findings\s+council\s+big$/);
|
|
273
324
|
const bRow = r.lines.find((l) => l.startsWith("bbbbbbbb"));
|
|
274
325
|
assert.match(bRow, /\s5\.9\.1\s+shipped\*\s+-\s+10\/-\/-\/-\/-\s/);
|
|
275
326
|
assert.match(bRow, /\s-\s*$/);
|
|
276
327
|
const aRow = r.lines.find((l) => l.startsWith("aaaaaaaa"));
|
|
277
|
-
assert.match(aRow, /\sshipped\s+60m\s+10\/10\/10\/10\/10\s+6\.6k\s+2\.25\s+m\/alpha,m\/beta\s+7\s+1\s+0\s+2\s+1\/2\/3\s+4
|
|
328
|
+
assert.match(aRow, /\sshipped\s+60m\s+10\/10\/10\/10\/10\s+6\.6k\s+2\.25\s+m\/alpha,m\/beta\s+7\s+1\s+0\s+2\s+1\/2\/3\s+4\s+-$/);
|
|
278
329
|
assert.ok(r.lines.includes("by version"));
|
|
279
330
|
const v = r.lines.find((l) => l.startsWith("5.9.1"));
|
|
280
|
-
assert.match(v, /^5\.9\.1\s+
|
|
331
|
+
assert.match(v, /^5\.9\.1\s+4\s+3\s+1\s+60m\/120m\s/);
|
|
281
332
|
assert.ok(r.lines.some((l) => /^skipped: .*broken\.yaml: unparseable$/.test(l)));
|
|
282
333
|
});
|
|
283
334
|
|
|
@@ -294,7 +294,7 @@ function foldAccumulators(a, b) {
|
|
|
294
294
|
const out = emptyAccumulators();
|
|
295
295
|
for (const k of /* @__PURE__ */ new Set([...Object.keys(a.phases), ...Object.keys(b.phases)])) {
|
|
296
296
|
const x = a.phases[k] ?? {}, y = b.phases[k] ?? {};
|
|
297
|
-
out.phases[k] = compact({ tokens: x.tokens && y.tokens ? addTokens(x.tokens, y.tokens) : x.tokens ?? y.tokens, compactions: optSum(x.compactions, y.compactions), user_messages: optSum(x.user_messages, y.user_messages), peak_context: optMax(x.peak_context, y.peak_context) });
|
|
297
|
+
out.phases[k] = compact({ tokens: x.tokens && y.tokens ? addTokens(x.tokens, y.tokens) : x.tokens ?? y.tokens, compactions: optSum(x.compactions, y.compactions), user_messages: optSum(x.user_messages, y.user_messages), peak_context: optMax(x.peak_context, y.peak_context), parent_calls: x.parent_calls || y.parent_calls ? [...x.parent_calls ?? [], ...y.parent_calls ?? []] : void 0 });
|
|
298
298
|
}
|
|
299
299
|
for (const k of /* @__PURE__ */ new Set([...Object.keys(a.personas), ...Object.keys(b.personas)])) {
|
|
300
300
|
const x = a.personas[k], y = b.personas[k];
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
resolveSpecCouncil,
|
|
6
6
|
resolveClosureReview,
|
|
7
7
|
resolveEscalationLoop,
|
|
8
|
+
resolvePhaseModels,
|
|
8
9
|
mainLoopModel,
|
|
9
10
|
resolveFlowGuards,
|
|
10
11
|
planDirsFor,
|
|
@@ -57,6 +58,31 @@ test("escalationLoop: non-empty string wins, trimmed; undefined mainLoop passes
|
|
|
57
58
|
assert.equal(resolveEscalationLoop({}, undefined).implModel, undefined);
|
|
58
59
|
});
|
|
59
60
|
|
|
61
|
+
test("phaseModels: absent -> empty map, no warning", () => {
|
|
62
|
+
assert.deepEqual(resolvePhaseModels({}), { phaseModels: {}, warning: undefined });
|
|
63
|
+
assert.deepEqual(resolvePhaseModels({ phaseModels: {} }), { phaseModels: {}, warning: undefined });
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("phaseModels: non-empty strings kept trimmed; other present values dropped with a warning naming the phase", () => {
|
|
67
|
+
const r = resolvePhaseModels({ phaseModels: { plan: " p/m:low ", implement: "p/m", verify: 7, ship: "" } } as PiGauntlet);
|
|
68
|
+
assert.deepEqual(r.phaseModels, { plan: "p/m:low", implement: "p/m" });
|
|
69
|
+
assert.equal(
|
|
70
|
+
r.warning,
|
|
71
|
+
'piGauntlet.phaseModels.verify must be a "provider/id" or "provider/id:level" string; piGauntlet.phaseModels.ship must be a "provider/id" or "provider/id:level" string',
|
|
72
|
+
);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("phaseModels: unknown keys are ignored; brainstorm is not a phase key", () => {
|
|
76
|
+
const r = resolvePhaseModels({ phaseModels: { brainstorm: "p/m", implement: "p/m" } } as PiGauntlet);
|
|
77
|
+
assert.deepEqual(r.phaseModels, { implement: "p/m" });
|
|
78
|
+
assert.equal(r.warning, undefined);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test("mergeGauntlet: repo phaseModels replaces preset whole-object", () => {
|
|
82
|
+
const g = mergeGauntlet({ phaseModels: { plan: "a/b", implement: "a/b" } }, { phaseModels: { verify: "c/d" } });
|
|
83
|
+
assert.deepEqual(g.phaseModels, { verify: "c/d" });
|
|
84
|
+
});
|
|
85
|
+
|
|
60
86
|
test("mainLoopModel: always suffixed; unset -> off, max -> xhigh, recognised pass through", () => {
|
|
61
87
|
const m = { provider: "p", id: "id" };
|
|
62
88
|
assert.equal(mainLoopModel(m, undefined), "p/id:off");
|
|
@@ -11,6 +11,7 @@ export interface PiGauntlet {
|
|
|
11
11
|
flowGuards?: { enforce?: unknown; specDirs?: unknown };
|
|
12
12
|
verifyBeforeShip?: { testCommands?: unknown; warningReference?: unknown };
|
|
13
13
|
escalationLoop?: { implModel?: unknown };
|
|
14
|
+
phaseModels?: { plan?: unknown; implement?: unknown; verify?: unknown; ship?: unknown };
|
|
14
15
|
telemetry?: { enabled?: unknown; dir?: unknown; buckets?: unknown };
|
|
15
16
|
}
|
|
16
17
|
|
|
@@ -100,6 +101,25 @@ export function resolveEscalationLoop(g: PiGauntlet, mainLoop: string | undefine
|
|
|
100
101
|
return { implModel: nonEmptyString(raw) ? raw.trim() : mainLoop };
|
|
101
102
|
}
|
|
102
103
|
|
|
104
|
+
export type PhaseModelPhase = "plan" | "implement" | "verify" | "ship";
|
|
105
|
+
const PHASE_MODEL_PHASES: PhaseModelPhase[] = ["plan", "implement", "verify", "ship"];
|
|
106
|
+
export interface PhaseModelsResolved {
|
|
107
|
+
phaseModels: Partial<Record<PhaseModelPhase, string>>;
|
|
108
|
+
warning: string | undefined;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function resolvePhaseModels(g: PiGauntlet): PhaseModelsResolved {
|
|
112
|
+
const phaseModels: Partial<Record<PhaseModelPhase, string>> = {};
|
|
113
|
+
const warnings: string[] = [];
|
|
114
|
+
for (const phase of PHASE_MODEL_PHASES) {
|
|
115
|
+
const raw = g.phaseModels?.[phase];
|
|
116
|
+
if (raw === undefined) continue;
|
|
117
|
+
if (nonEmptyString(raw)) phaseModels[phase] = raw.trim();
|
|
118
|
+
else warnings.push(`piGauntlet.phaseModels.${phase} must be a "provider/id" or "provider/id:level" string`);
|
|
119
|
+
}
|
|
120
|
+
return { phaseModels, warning: joinWarn(warnings) };
|
|
121
|
+
}
|
|
122
|
+
|
|
103
123
|
const THINKING_SUFFIXES = new Set(["off", "minimal", "low", "medium", "high", "xhigh"]);
|
|
104
124
|
|
|
105
125
|
// Always emit a suffix so pi-cohort's applyThinkingSuffix never falls back to the
|
|
@@ -166,6 +166,24 @@ test("parseRecord drops malformed accumulator blocks and fills missing fields",
|
|
|
166
166
|
assert.equal(derive(parsed, "2026-09-17T10:01:00Z").amendments, 2);
|
|
167
167
|
});
|
|
168
168
|
|
|
169
|
+
test("foldAccumulators concatenates parent_calls in order and leaves the field absent when neither side has it", () => {
|
|
170
|
+
const a: Accumulators = { ...emptyAccumulators(), phases: { implement: { parent_calls: [[1, 2, 0]] } } };
|
|
171
|
+
const b: Accumulators = { ...emptyAccumulators(), phases: { implement: { parent_calls: [[3, 60000, 60000]] }, plan: { compactions: 1 } } };
|
|
172
|
+
const ab = foldAccumulators(a, b);
|
|
173
|
+
assert.deepEqual(ab.phases.implement, { parent_calls: [[1, 2, 0], [3, 60000, 60000]] });
|
|
174
|
+
assert.deepEqual(ab.phases.plan, { compactions: 1 });
|
|
175
|
+
assert.equal("parent_calls" in (ab.phases.plan ?? {}), false);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("derive exposes parent_calls per phase and an old record without the field round-trips unchanged", () => {
|
|
179
|
+
const rec = newRecord({ spec: "doc/specs/a.md", session: "s", now: "2026-01-01T00:00:00Z", runId: "r" });
|
|
180
|
+
rec.accumulators.s = { ...emptyAccumulators(), phases: { implement: { parent_calls: [[0, 70000, 0]] } } };
|
|
181
|
+
assert.deepEqual(derive(rec, "2026-01-01T00:01:00Z").phases.implement, { parent_calls: [[0, 70000, 0]] });
|
|
182
|
+
const old = newRecord({ spec: "doc/specs/b.md", session: "s", now: "2026-01-01T00:00:00Z", runId: "r2" });
|
|
183
|
+
old.accumulators.s = { ...emptyAccumulators(), phases: { implement: { compactions: 2 } } };
|
|
184
|
+
assert.deepEqual(derive(old, "2026-01-01T00:01:00Z").phases.implement, { compactions: 2 });
|
|
185
|
+
});
|
|
186
|
+
|
|
169
187
|
const COUNCIL: CouncilBlock = {
|
|
170
188
|
chair: { model: "p/chair:medium", dispatches: 1, clusters: 2, members_reported: 2 },
|
|
171
189
|
members: {
|
|
@@ -48,7 +48,7 @@ export function capEvents(events: TelemetryEvent[], cap = EVENT_CAP): { events:
|
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
export interface Tokens { input: number; output: number; cache_read: number; cache_write: number; cost: number }
|
|
51
|
-
export interface PhaseAcc { tokens?: Tokens; compactions?: number; user_messages?: number; peak_context?: number }
|
|
51
|
+
export interface PhaseAcc { tokens?: Tokens; compactions?: number; user_messages?: number; peak_context?: number; parent_calls?: [number, number, number][] }
|
|
52
52
|
export interface PersonaAcc { dispatches: number; models: string[]; tokens?: Tokens }
|
|
53
53
|
export interface ReviewAcc { dispatches: number; nonzero_exit: number; findings?: { blocker: number; major: number; minor: number } }
|
|
54
54
|
export interface Gates { spec_rounds: number; plan_rounds: number; fix_round_grants: number; task_reopens: number }
|
|
@@ -67,7 +67,7 @@ export function foldAccumulators(a: Accumulators, b: Accumulators): Accumulators
|
|
|
67
67
|
const out = emptyAccumulators();
|
|
68
68
|
for (const k of new Set([...Object.keys(a.phases), ...Object.keys(b.phases)]) as Set<PhaseKey>) {
|
|
69
69
|
const x = a.phases[k] ?? {}, y = b.phases[k] ?? {};
|
|
70
|
-
out.phases[k] = compact({ tokens: x.tokens && y.tokens ? addTokens(x.tokens, y.tokens) : x.tokens ?? y.tokens, compactions: optSum(x.compactions, y.compactions), user_messages: optSum(x.user_messages, y.user_messages), peak_context: optMax(x.peak_context, y.peak_context) });
|
|
70
|
+
out.phases[k] = compact({ tokens: x.tokens && y.tokens ? addTokens(x.tokens, y.tokens) : x.tokens ?? y.tokens, compactions: optSum(x.compactions, y.compactions), user_messages: optSum(x.user_messages, y.user_messages), peak_context: optMax(x.peak_context, y.peak_context), parent_calls: x.parent_calls || y.parent_calls ? [...(x.parent_calls ?? []), ...(y.parent_calls ?? [])] : undefined });
|
|
71
71
|
}
|
|
72
72
|
for (const k of new Set([...Object.keys(a.personas), ...Object.keys(b.personas)])) {
|
|
73
73
|
const x = a.personas[k], y = b.personas[k];
|
|
@@ -69,7 +69,9 @@ const resumedBranch = (rest: Partial<Record<Phase, Status>>) => [
|
|
|
69
69
|
phaseResult("complete", phases({ brainstorm: "skipped", ...rest })),
|
|
70
70
|
];
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
type TestModel = { provider: string; id: string; promptCache?: { short?: number; long?: number } };
|
|
73
|
+
|
|
74
|
+
function harness(options: { cwd?: string; branch?: unknown[]; idle?: boolean; beforeSettled?: (setIdle: (idle: boolean) => void) => void; sendThrows?: boolean; model?: TestModel; thinkingLevel?: string; registry?: Record<string, TestModel>; setModelResult?: boolean | Error } = {}) {
|
|
73
75
|
const handlers = new Map<string, ((event: unknown, ctx: unknown) => unknown)[]>();
|
|
74
76
|
const tools: { name: string; executionMode?: string; parameters?: any; execute: (...args: any[]) => unknown }[] = [];
|
|
75
77
|
const sent: { message: any; options: any }[] = [];
|
|
@@ -82,8 +84,21 @@ function harness(options: { cwd?: string; branch?: unknown[]; idle?: boolean; be
|
|
|
82
84
|
sessionManager: { getBranch: () => branch },
|
|
83
85
|
model: options.model,
|
|
84
86
|
thinkingLevel: options.thinkingLevel,
|
|
87
|
+
modelRegistry: { find: (provider: string, id: string) => options.registry?.[`${provider}/${id}`] },
|
|
85
88
|
};
|
|
86
89
|
const pi = {
|
|
90
|
+
calls: [] as string[],
|
|
91
|
+
setModelCalls: [] as unknown[],
|
|
92
|
+
thinkingCalls: [] as string[],
|
|
93
|
+
async setModel(model: TestModel) {
|
|
94
|
+
pi.calls.push("setModel");
|
|
95
|
+
pi.setModelCalls.push(model);
|
|
96
|
+
if (options.setModelResult instanceof Error) throw options.setModelResult;
|
|
97
|
+
const ok = options.setModelResult ?? true;
|
|
98
|
+
if (ok) ctx.model = model;
|
|
99
|
+
return ok;
|
|
100
|
+
},
|
|
101
|
+
setThinkingLevel(level: string) { pi.calls.push("setThinkingLevel"); pi.thinkingCalls.push(level); },
|
|
87
102
|
on(event: string, handler: (event: unknown, context: unknown) => unknown) {
|
|
88
103
|
const registered = handlers.get(event) ?? [];
|
|
89
104
|
registered.push(handler);
|
|
@@ -107,7 +122,7 @@ function harness(options: { cwd?: string; branch?: unknown[]; idle?: boolean; be
|
|
|
107
122
|
for (const handler of handlers.get(name) ?? []) results.push(await handler(event, ctx));
|
|
108
123
|
return results;
|
|
109
124
|
};
|
|
110
|
-
return { emit, emitEvent, sent, tools, ctx, setIdle: (next: boolean) => (idle = next), setBranch: (next: unknown[]) => (branch = next) };
|
|
125
|
+
return { pi, emit, emitEvent, sent, tools, ctx, setIdle: (next: boolean) => (idle = next), setBranch: (next: unknown[]) => (branch = next) };
|
|
111
126
|
}
|
|
112
127
|
|
|
113
128
|
const settle = async (h: ReturnType<typeof harness>) => {
|
|
@@ -115,6 +130,139 @@ const settle = async (h: ReturnType<typeof harness>) => {
|
|
|
115
130
|
await h.emit("agent_settled");
|
|
116
131
|
};
|
|
117
132
|
|
|
133
|
+
const startPhase = async (h: ReturnType<typeof harness>, phase: string) => {
|
|
134
|
+
const tool = h.tools.find((t) => t.name === "phase_tracker")!;
|
|
135
|
+
const r = (await tool.execute("id", { action: "start", phase }, undefined, undefined, h.ctx)) as { content: { text: string }[]; details: { phases: Record<string, { status: string }> } };
|
|
136
|
+
return { text: r.content.map((c) => c.text).join("\n"), status: r.details.phases[phase].status };
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const M = (id: string, promptCache?: { short?: number; long?: number }) => ({ provider: "p", id, promptCache });
|
|
140
|
+
|
|
141
|
+
const preparePhase = async (h: ReturnType<typeof harness>, phase = "plan") => {
|
|
142
|
+
await settle(h);
|
|
143
|
+
const tool = h.tools.find((t) => t.name === "phase_tracker")!;
|
|
144
|
+
for (const prior of phase === "implement" ? ["brainstorm", "plan"] : ["brainstorm"]) {
|
|
145
|
+
await startPhase(h, prior);
|
|
146
|
+
await tool.execute("id", { action: "complete", phase: prior }, undefined, undefined, h.ctx);
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
test("phaseModels: switch applies thinking and reports the old and new model", async () => {
|
|
151
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { phaseModels: { plan: "p/m:medium" } } }), model: M("old"), thinkingLevel: "high", registry: { "p/m": M("m") } });
|
|
152
|
+
await preparePhase(h);
|
|
153
|
+
const r = await startPhase(h, "plan");
|
|
154
|
+
assert.deepEqual(h.pi.setModelCalls, [M("m")]);
|
|
155
|
+
assert.deepEqual(h.pi.thinkingCalls, ["medium"]);
|
|
156
|
+
assert.deepEqual(h.pi.calls, ["setModel", "setThinkingLevel"]);
|
|
157
|
+
assert.match(r.text, /model: p\/old:high -> p\/m:medium/);
|
|
158
|
+
assert.equal(r.status, "in_progress");
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
for (const row of [
|
|
162
|
+
{ name: "unknown model", target: "p/nope", warning: /phaseModels\.plan: model p\/nope not found in the registry/, calls: 0 },
|
|
163
|
+
{ name: "unknown thinking suffix", target: "p/m:turbo", warning: /":turbo" is not a pi thinking level \(off\|minimal\|low\|medium\|high\|xhigh\|max\)/, calls: 0 },
|
|
164
|
+
{ name: "colon model id", target: "p/a:b", warning: undefined, calls: 1 },
|
|
165
|
+
{ name: "missing auth", target: "p/m", result: false, warning: /cannot switch to p\/m - no auth configured for p/, calls: 1 },
|
|
166
|
+
{ name: "auth exception", target: "p/m", result: new Error("No API key for p"), warning: /cannot switch to p\/m - No API key for p/, calls: 1 },
|
|
167
|
+
]) {
|
|
168
|
+
test(`phaseModels: ${row.name} still starts the phase`, async () => {
|
|
169
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { phaseModels: { plan: row.target } } }), model: M("old"), registry: { "p/m": M("m"), "p/a:b": M("a:b") }, setModelResult: row.result });
|
|
170
|
+
await preparePhase(h);
|
|
171
|
+
const r = await startPhase(h, "plan");
|
|
172
|
+
assert.equal(h.pi.setModelCalls.length, row.calls);
|
|
173
|
+
assert.deepEqual(h.pi.thinkingCalls, []);
|
|
174
|
+
if (row.warning) assert.match(r.text, row.warning);
|
|
175
|
+
else assert.doesNotMatch(r.text, /phaseModels\.plan:/);
|
|
176
|
+
assert.equal(r.status, "in_progress");
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
for (const row of [
|
|
181
|
+
{ target: "p/m:high", levels: ["high"], line: /thinking: medium -> high/ },
|
|
182
|
+
{ target: "p/m:medium", levels: [], line: undefined },
|
|
183
|
+
{ target: "p/m", levels: [], line: undefined },
|
|
184
|
+
]) {
|
|
185
|
+
test(`phaseModels: active target ${row.target} never reselects the model`, async () => {
|
|
186
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { phaseModels: { plan: row.target } } }), model: M("m"), thinkingLevel: "medium", registry: { "p/m": M("m") } });
|
|
187
|
+
await preparePhase(h);
|
|
188
|
+
const r = await startPhase(h, "plan");
|
|
189
|
+
assert.deepEqual(h.pi.setModelCalls, []);
|
|
190
|
+
assert.deepEqual(h.pi.thinkingCalls, row.levels);
|
|
191
|
+
if (row.line) assert.match(r.text, row.line);
|
|
192
|
+
else assert.doesNotMatch(r.text, /model:|thinking:/);
|
|
193
|
+
assert.equal(r.status, "in_progress");
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
test("phaseModels: child sessions neither switch nor warn", async () => {
|
|
198
|
+
const depth = process.env.PI_SUBAGENT_DEPTH;
|
|
199
|
+
process.env.PI_SUBAGENT_DEPTH = "1";
|
|
200
|
+
try {
|
|
201
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { phaseModels: { plan: "p/nope" } } }) });
|
|
202
|
+
await preparePhase(h);
|
|
203
|
+
const r = await startPhase(h, "plan");
|
|
204
|
+
assert.deepEqual(h.pi.setModelCalls, []);
|
|
205
|
+
assert.deepEqual(h.pi.thinkingCalls, []);
|
|
206
|
+
assert.doesNotMatch(r.text, /phaseModels/);
|
|
207
|
+
assert.equal(r.status, "in_progress");
|
|
208
|
+
} finally {
|
|
209
|
+
if (depth === undefined) delete process.env.PI_SUBAGENT_DEPTH;
|
|
210
|
+
else process.env.PI_SUBAGENT_DEPTH = depth;
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test("phaseModels: resolver warnings appear only for their phase", async () => {
|
|
215
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { phaseModels: { plan: 5 } } }) });
|
|
216
|
+
await settle(h);
|
|
217
|
+
assert.doesNotMatch((await startPhase(h, "brainstorm")).text, /phaseModels/);
|
|
218
|
+
await h.tools.find(t => t.name === "phase_tracker")!.execute("id", { action: "complete", phase: "brainstorm" }, undefined, undefined, h.ctx);
|
|
219
|
+
const r = await startPhase(h, "plan");
|
|
220
|
+
assert.match(r.text, /piGauntlet\.phaseModels\.plan must be a "provider\/id" or "provider\/id:level" string/);
|
|
221
|
+
assert.equal(r.status, "in_progress");
|
|
222
|
+
assert.deepEqual(h.pi.setModelCalls, []);
|
|
223
|
+
assert.deepEqual(h.pi.thinkingCalls, []);
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
for (const row of [
|
|
227
|
+
{ name: "missing short", phase: "implement", cache: undefined, retention: undefined, tier: "short" },
|
|
228
|
+
{ name: "declared short", phase: "implement", cache: { short: 300 }, retention: undefined, tier: undefined },
|
|
229
|
+
{ name: "missing long", phase: "implement", cache: { short: 300 }, retention: "long", tier: "long" },
|
|
230
|
+
{ name: "plan ignores cache", phase: "plan", cache: undefined, retention: undefined, tier: undefined },
|
|
231
|
+
]) {
|
|
232
|
+
test(`phaseModels: cache warning ${row.name}`, async () => {
|
|
233
|
+
const retention = process.env.PI_CACHE_RETENTION;
|
|
234
|
+
if (row.retention === undefined) delete process.env.PI_CACHE_RETENTION;
|
|
235
|
+
else process.env.PI_CACHE_RETENTION = row.retention;
|
|
236
|
+
try {
|
|
237
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { flowGuards: { enforce: false } } }), model: M("m", row.cache) });
|
|
238
|
+
await preparePhase(h, row.phase);
|
|
239
|
+
const r = await startPhase(h, row.phase);
|
|
240
|
+
if (row.tier) {
|
|
241
|
+
assert.ok(r.text.includes(`p/m declares no promptCache.${row.tier} lifetime, so pi's cache warmer skips it`));
|
|
242
|
+
assert.ok(r.text.includes("providers.p.modelOverrides.m.promptCache"));
|
|
243
|
+
assert.ok(r.text.includes("providers.p.models[].promptCache"));
|
|
244
|
+
} else assert.doesNotMatch(r.text, /cache warmer/);
|
|
245
|
+
assert.deepEqual(h.pi.setModelCalls, []);
|
|
246
|
+
assert.deepEqual(h.pi.thinkingCalls, []);
|
|
247
|
+
assert.equal(r.status, "in_progress");
|
|
248
|
+
} finally {
|
|
249
|
+
if (retention === undefined) delete process.env.PI_CACHE_RETENTION;
|
|
250
|
+
else process.env.PI_CACHE_RETENTION = retention;
|
|
251
|
+
}
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
test("phaseModels: cache warning checks the switched model", async () => {
|
|
256
|
+
const h = harness({ cwd: tempCwd({ piGauntlet: { flowGuards: { enforce: false }, phaseModels: { implement: "p/warm" } } }), model: M("old"), registry: { "p/warm": M("warm", { short: 300, long: 3600 }) } });
|
|
257
|
+
await preparePhase(h, "implement");
|
|
258
|
+
const r = await startPhase(h, "implement");
|
|
259
|
+
assert.equal(h.pi.setModelCalls.length, 1);
|
|
260
|
+
assert.deepEqual(h.pi.thinkingCalls, []);
|
|
261
|
+
assert.equal(h.ctx.model?.id, "warm");
|
|
262
|
+
assert.doesNotMatch(r.text, /cache warmer/);
|
|
263
|
+
assert.equal(r.status, "in_progress");
|
|
264
|
+
});
|
|
265
|
+
|
|
118
266
|
test("agent_settled nudges each exact edge with persisted details", async () => {
|
|
119
267
|
for (const [state, edge, skill] of [
|
|
120
268
|
[phases({ brainstorm: "complete", plan: "complete" }), "plan-implement", "/skill:subagent-driven-development"],
|
|
@@ -21,6 +21,8 @@ import {
|
|
|
21
21
|
resolveClosureReview,
|
|
22
22
|
resolveEscalationLoop,
|
|
23
23
|
resolveFlowGuards,
|
|
24
|
+
resolvePhaseModels,
|
|
25
|
+
type PhaseModelPhase,
|
|
24
26
|
resolveSpecCouncil,
|
|
25
27
|
settingsErrorWarning,
|
|
26
28
|
} from "./lib/gauntlet-settings.ts";
|
|
@@ -373,6 +375,50 @@ function formatStatus(phases: PhaseMap): string {
|
|
|
373
375
|
return lines.join("\n");
|
|
374
376
|
}
|
|
375
377
|
|
|
378
|
+
const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]);
|
|
379
|
+
const PHASE_MODEL_PHASES = new Set<string>(["plan", "implement", "verify", "ship"]);
|
|
380
|
+
|
|
381
|
+
async function applyPhaseModel(
|
|
382
|
+
pi: Pick<ExtensionAPI, "setModel" | "setThinkingLevel">,
|
|
383
|
+
ctx: Pick<ExtensionContext, "model" | "thinkingLevel" | "modelRegistry">,
|
|
384
|
+
phase: PhaseModelPhase,
|
|
385
|
+
target: string,
|
|
386
|
+
): Promise<string[]> {
|
|
387
|
+
const warn = (msg: string) => [`\u26a0\ufe0f phaseModels.${phase}: ${msg}`];
|
|
388
|
+
const slash = target.indexOf("/");
|
|
389
|
+
if (slash <= 0 || slash === target.length - 1) return warn(`model ${target} not found in the registry`);
|
|
390
|
+
const provider = target.slice(0, slash);
|
|
391
|
+
const rest = target.slice(slash + 1);
|
|
392
|
+
const colon = rest.lastIndexOf(":");
|
|
393
|
+
const suffix = colon >= 0 ? rest.slice(colon + 1) : undefined;
|
|
394
|
+
const level = suffix !== undefined && THINKING_LEVELS.has(suffix) ? suffix : undefined;
|
|
395
|
+
const id = level === undefined ? rest : rest.slice(0, colon);
|
|
396
|
+
const model = ctx.modelRegistry.find(provider, id);
|
|
397
|
+
if (!model) {
|
|
398
|
+
if (suffix !== undefined && level === undefined && ctx.modelRegistry.find(provider, rest.slice(0, colon))) {
|
|
399
|
+
return warn(`":${suffix}" is not a pi thinking level (off|minimal|low|medium|high|xhigh|max)`);
|
|
400
|
+
}
|
|
401
|
+
return warn(`model ${provider}/${id} not found in the registry`);
|
|
402
|
+
}
|
|
403
|
+
const cur = ctx.model;
|
|
404
|
+
const curLevel = ctx.thinkingLevel;
|
|
405
|
+
const fmt = (m: { provider: string; id: string } | undefined, l: string | undefined) => (m ? `${m.provider}/${m.id}${l ? `:${l}` : ""}` : "none");
|
|
406
|
+
if (cur && cur.provider === model.provider && cur.id === model.id) {
|
|
407
|
+
if (level === undefined || level === curLevel) return [];
|
|
408
|
+
pi.setThinkingLevel(level as Parameters<ExtensionAPI["setThinkingLevel"]>[0]);
|
|
409
|
+
return [`thinking: ${curLevel ?? "off"} -> ${level}`];
|
|
410
|
+
}
|
|
411
|
+
let ok = false;
|
|
412
|
+
try {
|
|
413
|
+
ok = await pi.setModel(model);
|
|
414
|
+
} catch (e) {
|
|
415
|
+
return warn(`cannot switch to ${provider}/${id} - ${String((e as Error)?.message ?? e).split("\n")[0]}`);
|
|
416
|
+
}
|
|
417
|
+
if (!ok) return warn(`cannot switch to ${provider}/${id} - no auth configured for ${provider}`);
|
|
418
|
+
if (level !== undefined) pi.setThinkingLevel(level as Parameters<ExtensionAPI["setThinkingLevel"]>[0]);
|
|
419
|
+
return [`model: ${fmt(cur, curLevel)} -> ${fmt(model, level)}`];
|
|
420
|
+
}
|
|
421
|
+
|
|
376
422
|
export default function (pi: ExtensionAPI) {
|
|
377
423
|
let phases: PhaseMap = emptyPhases();
|
|
378
424
|
let conformanceDispatched = false;
|
|
@@ -978,6 +1024,25 @@ export default function (pi: ExtensionAPI) {
|
|
|
978
1024
|
}
|
|
979
1025
|
}
|
|
980
1026
|
}
|
|
1027
|
+
const startLines: string[] = [];
|
|
1028
|
+
if (!isSubagentChild && PHASE_MODEL_PHASES.has(params.phase)) {
|
|
1029
|
+
const phase = params.phase as PhaseModelPhase;
|
|
1030
|
+
const resolved = resolvePhaseModels(loadGauntletSettings(ctx.cwd).gauntlet);
|
|
1031
|
+
for (const w of resolved.warning?.split("; ") ?? []) if (w.includes(`phaseModels.${phase} `)) startLines.push(`\u26a0\ufe0f ${w}`);
|
|
1032
|
+
const target = resolved.phaseModels[phase];
|
|
1033
|
+
if (target) startLines.push(...(await applyPhaseModel(pi, ctx, phase, target)));
|
|
1034
|
+
if (phase === "implement") {
|
|
1035
|
+
const tier = process.env.PI_CACHE_RETENTION === "long" ? "long" : "short";
|
|
1036
|
+
const m = ctx.model;
|
|
1037
|
+
if (m && m.promptCache?.[tier] === undefined) {
|
|
1038
|
+
startLines.push(
|
|
1039
|
+
`\u26a0\ufe0f ${m.provider}/${m.id} declares no promptCache.${tier} lifetime, so pi's cache warmer skips it. ` +
|
|
1040
|
+
`Declare it in <agent-dir>/models.json (default ~/.pi/agent): providers.${m.provider}.modelOverrides.${m.id}.promptCache for a built-in model, ` +
|
|
1041
|
+
`or providers.${m.provider}.models[].promptCache for a custom one; the lifetime is required by the warmer but not sufficient - see doc/configuration.md#phase-models.`,
|
|
1042
|
+
);
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
981
1046
|
phases = { ...phases, [params.phase]: transitionPhaseState("in_progress") as PhaseState };
|
|
982
1047
|
// A rewind must not inherit the prior verify's conformance latch.
|
|
983
1048
|
if (params.phase === "implement") {
|
|
@@ -989,7 +1054,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
989
1054
|
firedGuards.clear();
|
|
990
1055
|
updateWidget(ctx);
|
|
991
1056
|
return {
|
|
992
|
-
content: [{ type: "text", text: `Phase "${params.phase}" → in_progress
|
|
1057
|
+
content: [{ type: "text", text: [`Phase "${params.phase}" → in_progress`, ...startLines, formatStatus(phases)].join("\n") }],
|
|
993
1058
|
details: { action: "start", phases: { ...phases } } as PhaseTrackerDetails,
|
|
994
1059
|
};
|
|
995
1060
|
}
|
|
@@ -493,6 +493,25 @@
|
|
|
493
493
|
assert.equal(rec.derived.amendments, 7);
|
|
494
494
|
});
|
|
495
495
|
|
|
496
|
+
test("message_end appends one cache triplet per parent call; child usage is excluded from parent_calls but still summed into tokens", async () => {
|
|
497
|
+
const h = harness();
|
|
498
|
+
await h.phaseResult("start", P({ brainstorm: "in_progress" }));
|
|
499
|
+
await h.writeSpec("doc/specs/a.md");
|
|
500
|
+
await h.phaseResult("complete", P({ brainstorm: "complete" }));
|
|
501
|
+
await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" }));
|
|
502
|
+
await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" }));
|
|
503
|
+
await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "in_progress" }));
|
|
504
|
+
const big = { input: 1, output: 1, cacheRead: 10, cacheWrite: 60000, cacheWrite1h: 60000, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } };
|
|
505
|
+
const small = { input: 1, output: 1, cacheRead: 20, cacheWrite: 5, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } };
|
|
506
|
+
await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: big } });
|
|
507
|
+
await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: small } });
|
|
508
|
+
await h.emit("tool_result", subagentResult([{ agent: "implementer", exitCode: 0, model: "p/i", usage: { input: 7, output: 1, cacheRead: 0, cacheWrite: 90000, totalTokens: 8, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } }]));
|
|
509
|
+
await h.emit("turn_end", { type: "turn_end", turnIndex: 1, message: {}, toolResults: [] });
|
|
510
|
+
const impl = h.readRecord("doc/specs/a.md").derived.phases.implement!;
|
|
511
|
+
assert.deepEqual(impl.parent_calls, [[10, 60000, 60000], [20, 5, 0]]);
|
|
512
|
+
assert.equal(impl.tokens!.cache_write, 60000 + 5 + 90000);
|
|
513
|
+
});
|
|
514
|
+
|
|
496
515
|
const usage = (input: number, output = 0, cost = 0) => ({ input, output, cacheRead: 0, cacheWrite: 0, totalTokens: input + output, cost: { input: cost, output: 0, cacheRead: 0, cacheWrite: 0, total: cost } });
|
|
497
516
|
|
|
498
517
|
test("pre-binding observations merge into the bound record in event order", async () => {
|
package/extensions/telemetry.ts
CHANGED
|
@@ -772,6 +772,11 @@
|
|
|
772
772
|
const msg = event.message as { role?: string; usage?: unknown; content?: unknown };
|
|
773
773
|
if (msg.role !== "assistant") return;
|
|
774
774
|
addPhaseTokens(usageToTokens(msg.usage));
|
|
775
|
+
const u = msg.usage as { cacheRead?: unknown; cacheWrite?: unknown; cacheWrite1h?: unknown } | undefined;
|
|
776
|
+
if (u && typeof u === "object") {
|
|
777
|
+
const n = (v: unknown): number => (typeof v === "number" && Number.isFinite(v) ? v : 0);
|
|
778
|
+
(phaseAcc().parent_calls ??= []).push([n(u.cacheRead), n(u.cacheWrite), n(u.cacheWrite1h)]);
|
|
779
|
+
}
|
|
775
780
|
if (phaseNow() === "brainstorm") onCouncilAudit(textOf(msg.content));
|
|
776
781
|
});
|
|
777
782
|
|
package/package.json
CHANGED