prose-qa 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -28
- package/dist/agent/provider-options.d.ts.map +1 -1
- package/dist/agent/provider-options.js +13 -19
- package/dist/agent/provider-options.js.map +1 -1
- package/dist/agent/provider-options.test.js +12 -2
- package/dist/agent/provider-options.test.js.map +1 -1
- package/dist/agent/runner.d.ts.map +1 -1
- package/dist/agent/runner.js +14 -5
- package/dist/agent/runner.js.map +1 -1
- package/dist/agent/verdict.d.ts +5 -2
- package/dist/agent/verdict.d.ts.map +1 -1
- package/dist/agent/verdict.js +33 -1
- package/dist/agent/verdict.js.map +1 -1
- package/dist/agent/verdict.test.js +73 -1
- package/dist/agent/verdict.test.js.map +1 -1
- package/dist/reporter/index.d.ts.map +1 -1
- package/dist/reporter/index.js +4 -2
- package/dist/reporter/index.js.map +1 -1
- package/dist/reporter/index.test.js +32 -1
- package/dist/reporter/index.test.js.map +1 -1
- package/dist/reporter/stats.d.ts +10 -0
- package/dist/reporter/stats.d.ts.map +1 -0
- package/dist/reporter/stats.js +85 -0
- package/dist/reporter/stats.js.map +1 -0
- package/dist/reporter/stats.test.d.ts +2 -0
- package/dist/reporter/stats.test.d.ts.map +1 -0
- package/dist/reporter/stats.test.js +157 -0
- package/dist/reporter/stats.test.js.map +1 -0
- package/dist/types/config.d.ts +1 -1
- package/dist/types/verdict.d.ts +11 -0
- package/dist/types/verdict.d.ts.map +1 -1
- package/dist/types/verdict.js +6 -0
- package/dist/types/verdict.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import { describe, it } from "node:test";
|
|
3
|
-
import { appendFinalTextToTranscript, appendStepToTranscript, computeTranscriptStats, enrichVerdictWithStats, formatStepForTranscript, stripLastAssistantTurn, } from "./verdict.js";
|
|
3
|
+
import { addLanguageModelUsage, appendFinalTextToTranscript, appendStepToTranscript, computeTranscriptStats, emptyTokenUsage, enrichVerdictWithStats, formatStepForTranscript, stripLastAssistantTurn, } from "./verdict.js";
|
|
4
4
|
describe("stripLastAssistantTurn", () => {
|
|
5
5
|
it("removes the last assistant message", () => {
|
|
6
6
|
const messages = [
|
|
@@ -95,6 +95,44 @@ describe("appendFinalTextToTranscript", () => {
|
|
|
95
95
|
assert.equal(transcript.entries.length, 1);
|
|
96
96
|
});
|
|
97
97
|
});
|
|
98
|
+
describe("addLanguageModelUsage", () => {
|
|
99
|
+
it("accumulates input and output tokens across LLM calls", () => {
|
|
100
|
+
const first = {
|
|
101
|
+
inputTokens: 100,
|
|
102
|
+
outputTokens: 50,
|
|
103
|
+
inputTokenDetails: {
|
|
104
|
+
noCacheTokens: 100,
|
|
105
|
+
cacheReadTokens: undefined,
|
|
106
|
+
cacheWriteTokens: undefined,
|
|
107
|
+
},
|
|
108
|
+
outputTokenDetails: {
|
|
109
|
+
textTokens: 50,
|
|
110
|
+
reasoningTokens: undefined,
|
|
111
|
+
},
|
|
112
|
+
totalTokens: 150,
|
|
113
|
+
};
|
|
114
|
+
const second = {
|
|
115
|
+
inputTokens: 200,
|
|
116
|
+
outputTokens: 75,
|
|
117
|
+
inputTokenDetails: {
|
|
118
|
+
noCacheTokens: 150,
|
|
119
|
+
cacheReadTokens: 50,
|
|
120
|
+
cacheWriteTokens: undefined,
|
|
121
|
+
},
|
|
122
|
+
outputTokenDetails: {
|
|
123
|
+
textTokens: 75,
|
|
124
|
+
reasoningTokens: undefined,
|
|
125
|
+
},
|
|
126
|
+
totalTokens: 275,
|
|
127
|
+
};
|
|
128
|
+
const total = addLanguageModelUsage(addLanguageModelUsage(emptyTokenUsage(), first), second);
|
|
129
|
+
assert.deepEqual(total, {
|
|
130
|
+
input: 300,
|
|
131
|
+
output: 125,
|
|
132
|
+
cached: 50,
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
});
|
|
98
136
|
describe("computeTranscriptStats", () => {
|
|
99
137
|
it("counts LLM turns, user turns, tool calls, and durations from the transcript", () => {
|
|
100
138
|
const transcript = {
|
|
@@ -140,6 +178,17 @@ describe("computeTranscriptStats", () => {
|
|
|
140
178
|
bashDurationMs: 200,
|
|
141
179
|
});
|
|
142
180
|
});
|
|
181
|
+
it("includes token usage when provided", () => {
|
|
182
|
+
const transcript = { entries: [] };
|
|
183
|
+
const stats = computeTranscriptStats(transcript, {
|
|
184
|
+
tokens: { input: 12_000, output: 800, cached: 4_000 },
|
|
185
|
+
});
|
|
186
|
+
assert.deepEqual(stats.tokens, {
|
|
187
|
+
input: 12_000,
|
|
188
|
+
output: 800,
|
|
189
|
+
cached: 4_000,
|
|
190
|
+
});
|
|
191
|
+
});
|
|
143
192
|
it("includes healing metadata when provided", () => {
|
|
144
193
|
const transcript = { entries: [] };
|
|
145
194
|
const stats = computeTranscriptStats(transcript, {
|
|
@@ -178,6 +227,29 @@ describe("enrichVerdictWithStats", () => {
|
|
|
178
227
|
assert.equal(enriched?.stats?.llmTurns, 1);
|
|
179
228
|
assert.equal(enriched?.stats?.durationMs, 500);
|
|
180
229
|
});
|
|
230
|
+
it("preserves existing token stats when options omit tokens", () => {
|
|
231
|
+
const transcript = { entries: [] };
|
|
232
|
+
const enriched = enrichVerdictWithStats({
|
|
233
|
+
status: "pass",
|
|
234
|
+
checkpoints: [],
|
|
235
|
+
summary: "ok",
|
|
236
|
+
stats: {
|
|
237
|
+
durationMs: 500,
|
|
238
|
+
llmTurns: 1,
|
|
239
|
+
userTurns: 0,
|
|
240
|
+
toolCalls: 0,
|
|
241
|
+
failedToolCalls: 0,
|
|
242
|
+
llmDurationMs: 100,
|
|
243
|
+
bashDurationMs: 0,
|
|
244
|
+
tokens: { input: 900, output: 120, cached: 300 },
|
|
245
|
+
},
|
|
246
|
+
}, transcript, { durationMs: 500 });
|
|
247
|
+
assert.deepEqual(enriched?.stats?.tokens, {
|
|
248
|
+
input: 900,
|
|
249
|
+
output: 120,
|
|
250
|
+
cached: 300,
|
|
251
|
+
});
|
|
252
|
+
});
|
|
181
253
|
});
|
|
182
254
|
describe("appendStepToTranscript", () => {
|
|
183
255
|
it("appends one assistant message per step", () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.test.js","sourceRoot":"","sources":["../../src/agent/verdict.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"verdict.test.js","sourceRoot":"","sources":["../../src/agent/verdict.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAIzC,OAAO,EACL,qBAAqB,EACrB,2BAA2B,EAC3B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,sBAAsB,EACtB,uBAAuB,EACvB,sBAAsB,GACvB,MAAM,cAAc,CAAC;AAEtB,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,QAAQ,GAAmB;YAC/B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;YAC/B,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,EAAE;SACjD,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE;YACjD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,QAAQ,GAAmB;YAC/B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;YAC/B;gBACE,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,WAAW;wBACjB,UAAU,EAAE,GAAG;wBACf,QAAQ,EAAE,MAAM;wBAChB,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;qBAC3B;iBACF;aACF;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,aAAa;wBACnB,UAAU,EAAE,GAAG;wBACf,QAAQ,EAAE,MAAM;wBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE;qBACjD;iBACF;aACF;SACF,CAAC;QACF,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,QAAQ,CAAC,EAAE;YACjD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,2EAA2E,EAAE,GAAG,EAAE;QACnF,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,EAAE;YACrE,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,SAAS,CACd,uBAAuB,CAAC;YACtB,IAAI,EAAE,kBAAkB;YACxB,SAAS,EAAE,EAAE;SACd,CAAC,EACF,EAAE,OAAO,EAAE,kBAAkB,EAAE,CAChC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;QAC5E,MAAM,CAAC,SAAS,CACd,uBAAuB,CAAC;YACtB,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAE,CAAC;SAC9E,CAAC,EACF;YACE,OAAO,EAAE,gDAAgD;SAC1D,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,SAAS,CACd,uBAAuB,CAAC;YACtB,IAAI,EAAE,oBAAoB;YAC1B,aAAa,EAAE,wBAAwB;YACvC,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,2BAA2B,EAAE,EAAE,CAAC;SACnF,CAAC,EACF;YACE,OAAO,EAAE;gBACP,oBAAoB;gBACpB,qDAAqD;aACtD,CAAC,IAAI,CAAC,MAAM,CAAC;YACd,QAAQ,EAAE,wBAAwB;SACnC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;IAC3C,EAAE,CAAC,6EAA6E,EAAE,GAAG,EAAE;QACrF,MAAM,UAAU,GAAG;YACjB,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,SAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,8BAA8B,EAAE,EAAE,EAAE,0BAA0B,EAAE;aACzH;SACF,CAAC;QAEF,2BAA2B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAClD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,CACV,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAC/E,QAAQ,CACT,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QACpF,MAAM,UAAU,GAAG;YACjB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAkB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,0BAA0B,EAAE,CAAC;SAC7G,CAAC;QAEF,2BAA2B,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAuB;YAChC,WAAW,EAAE,GAAG;YAChB,YAAY,EAAE,EAAE;YAChB,iBAAiB,EAAE;gBACjB,aAAa,EAAE,GAAG;gBAClB,eAAe,EAAE,SAAS;gBAC1B,gBAAgB,EAAE,SAAS;aAC5B;YACD,kBAAkB,EAAE;gBAClB,UAAU,EAAE,EAAE;gBACd,eAAe,EAAE,SAAS;aAC3B;YACD,WAAW,EAAE,GAAG;SACjB,CAAC;QACF,MAAM,MAAM,GAAuB;YACjC,WAAW,EAAE,GAAG;YAChB,YAAY,EAAE,EAAE;YAChB,iBAAiB,EAAE;gBACjB,aAAa,EAAE,GAAG;gBAClB,eAAe,EAAE,EAAE;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,kBAAkB,EAAE;gBAClB,UAAU,EAAE,EAAE;gBACd,eAAe,EAAE,SAAS;aAC3B;YACD,WAAW,EAAE,GAAG;SACjB,CAAC;QAEF,MAAM,KAAK,GAAG,qBAAqB,CACjC,qBAAqB,CAAC,eAAe,EAAE,EAAE,KAAK,CAAC,EAC/C,MAAM,CACP,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;YACtB,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,GAAG;YACX,MAAM,EAAE,EAAE;SACX,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,6EAA6E,EAAE,GAAG,EAAE;QACrF,MAAM,UAAU,GAAoB;YAClC,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,mBAAmB;oBAC5B,EAAE,EAAE,0BAA0B;iBAC/B;gBACD;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,qDAAqD;oBAC9D,EAAE,EAAE,0BAA0B;oBAC9B,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,2BAA2B;oBACpC,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,GAAG;oBACf,EAAE,EAAE,0BAA0B;iBAC/B;gBACD;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,qDAAqD;oBAC9D,EAAE,EAAE,0BAA0B;oBAC9B,UAAU,EAAE,IAAI;iBACjB;aACF;SACF,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;YAC3E,UAAU,EAAE,MAAM;YAClB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,eAAe,EAAE,CAAC;YAClB,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,GAAG;SACpB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,sBAAsB,CAAC,UAAU,EAAE;YAC/C,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE;SACtD,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE;YAC7B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,GAAG;YACX,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,sBAAsB,CAAC,UAAU,EAAE;YAC/C,OAAO,EAAE;gBACP,IAAI,EAAE,IAAI;gBACV,aAAa,EAAE,CAAC;gBAChB,eAAe,EAAE,CAAC;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE;YAC9B,IAAI,EAAE,IAAI;YACV,aAAa,EAAE,CAAC;YAChB,eAAe,EAAE,CAAC;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,UAAU,GAAoB;YAClC,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,MAAM;oBACf,EAAE,EAAE,0BAA0B;oBAC9B,UAAU,EAAE,GAAG;iBAChB;aACF;SACF,CAAC;QAEF,MAAM,QAAQ,GAAG,sBAAsB,CACrC;YACE,MAAM,EAAE,MAAM;YACd,WAAW,EAAE,EAAE;YACf,OAAO,EAAE,IAAI;SACd,EACD,UAAU,EACV,EAAE,UAAU,EAAE,GAAG,EAAE,CACpB,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAEpD,MAAM,QAAQ,GAAG,sBAAsB,CACrC;YACE,MAAM,EAAE,MAAM;YACd,WAAW,EAAE,EAAE;YACf,OAAO,EAAE,IAAI;YACb,KAAK,EAAE;gBACL,UAAU,EAAE,GAAG;gBACf,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,eAAe,EAAE,CAAC;gBAClB,aAAa,EAAE,GAAG;gBAClB,cAAc,EAAE,CAAC;gBACjB,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE;aACjD;SACF,EACD,UAAU,EACV,EAAE,UAAU,EAAE,GAAG,EAAE,CACpB,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE;YACxC,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,GAAG;YACX,MAAM,EAAE,GAAG;SACZ,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAEpD,MAAM,CAAC,KAAK,CACV,sBAAsB,CAAC,UAAU,EAAE;YACjC,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAE,CAAC;SAC9E,CAAC,EACF,IAAI,CACL,CAAC;QACF,MAAM,CAAC,KAAK,CACV,sBAAsB,CAAC,UAAU,EAAE;YACjC,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,EAAE;SACd,CAAC,EACF,IAAI,CACL,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QAClD,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAEhD,sBAAsB,CACpB,UAAU,EACV,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,EAAE,EACpC,EAAE,EACF,SAAS,EACT,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CACzB,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,0BAA0B,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,UAAU,GAAoB,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAEpD,sBAAsB,CACpB,UAAU,EACV;YACE,IAAI,EAAE,kBAAkB;YACxB,aAAa,EAAE,8BAA8B;YAC7C,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAE,CAAC;SAC9E,EACD;YACE;gBACE,OAAO,EAAE,sBAAsB;gBAC/B,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,EAAE;aACf;SACF,CACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACvC,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,8BAA8B,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAI/D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,SAAS,EACjB,QAAQ,CAAC,EAAE,WAAW,GACrB,IAAI,CAcN;AAED,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,cAAc,CAAC,YAAY,CAAC,EACxC,QAAQ,CAAC,EAAE,WAAW,GACrB,IAAI,CAMN;AAED,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,cAAc,EACtB,QAAQ,CAAC,EAAE,WAAW,GACrB,IAAI,CAkBN;AAuBD,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,MAAM,CAgB3E;AA+ED,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,IAAI,EACf,OAAO,EAAE,cAAc,EAAE,GACxB,SAAS,CAmBX"}
|
package/dist/reporter/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { enrichVerdictWithStats, getTranscriptBashEntries } from "../agent/verdict.js";
|
|
4
|
+
import { buildRunStats, scenarioSlug } from "./stats.js";
|
|
4
5
|
export function createRunId() {
|
|
5
6
|
return new Date().toISOString().replace(/[:.]/g, "-");
|
|
6
7
|
}
|
|
@@ -10,8 +11,7 @@ export function ensureRunDir(cwd, runId) {
|
|
|
10
11
|
return dir;
|
|
11
12
|
}
|
|
12
13
|
export function scenarioArtifactDir(runDir, scenarioName) {
|
|
13
|
-
const
|
|
14
|
-
const dir = path.join(runDir, safe);
|
|
14
|
+
const dir = path.join(runDir, scenarioSlug(scenarioName));
|
|
15
15
|
mkdirSync(dir, { recursive: true });
|
|
16
16
|
return dir;
|
|
17
17
|
}
|
|
@@ -22,6 +22,7 @@ export function writeReport(runDir, report, redactor) {
|
|
|
22
22
|
const safeReport = { ...report, results: safeResults };
|
|
23
23
|
writeFileSync(path.join(runDir, "report.json"), `${JSON.stringify(safeReport, null, 2)}\n`);
|
|
24
24
|
writeFileSync(path.join(runDir, "report.html"), renderHtml(safeReport));
|
|
25
|
+
writeFileSync(path.join(runDir, "stats.json"), `${JSON.stringify(buildRunStats(safeResults), null, 2)}\n`);
|
|
25
26
|
}
|
|
26
27
|
export function writeTranscript(artifactDir, transcript, redactor) {
|
|
27
28
|
const safe = redactor ? redactor.redactTranscript(transcript) : transcript;
|
|
@@ -34,6 +35,7 @@ export function writeScenarioTranscript(artifactDir, result, redactor) {
|
|
|
34
35
|
const verdict = enrichVerdictWithStats(safe.verdict, safe.transcript, {
|
|
35
36
|
durationMs: safe.durationMs,
|
|
36
37
|
healing: safe.healing,
|
|
38
|
+
tokens: safe.verdict.stats?.tokens,
|
|
37
39
|
}) ?? safe.verdict;
|
|
38
40
|
writeFileSync(path.join(artifactDir, "verdict.json"), `${JSON.stringify(verdict, null, 2)}\n`);
|
|
39
41
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,UAAU,WAAW;IACzB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,KAAa;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAClD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAc,EAAE,YAAoB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;IAC1D,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,MAAc,EACd,MAAiB,EACjB,QAAsB;IAEtB,MAAM,WAAW,GAAG,QAAQ;QAC1B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;IACnB,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IACvD,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAChC,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAC3C,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;IACxE,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAC/B,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAC3D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,UAAwC,EACxC,QAAsB;IAEtB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAC3E,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EACzC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,WAAmB,EACnB,MAAsB,EACtB,QAAsB;IAEtB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EACzC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAChD,CAAC;IACF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,OAAO,GACX,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YACpD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM;SACnC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC;QACrB,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,EACtC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACxC,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAgC;IAEhC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACrC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,uBAAuB,CAAC,OAA6B;IAC5D,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;IAC/D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;IAC/D,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,UAAU,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU;IAClC,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,OAAyB;IACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,KAAK,GAAG,OAAO;SAClB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,WAAW,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClD,OAAO,cAAc,WAAW,yBAAyB,CAAC,CAAC,MAAM,WAAW,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,2BAA2B,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC;IAC5K,CAAC,CAAC;SACD,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClB,OAAO;;;MAGH,KAAK;;aAEE,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,MAAiB;IACnC,MAAM,WAAW,GAAG,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,WAAW,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,WAAW,GACf,CAAC,CAAC,OAAO,EAAE,WAAW;aACnB,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CACrG;aACA,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,wBAAwB,CAAC,CAAC,CAAC,UAAU,CAAC;aAChD,KAAK,CAAC,CAAC,EAAE,CAAC;aACV,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,gBAAgB,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,eAAe,CAClH;aACA,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,EAAE,IAAI;YAClC,CAAC,CAAC,0CAA0C;YAC5C,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,aAAa,GAAG,CAAC,CAAC,OAAO;YAC7B,CAAC,CAAC,oCAAoC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,aAAa,uBAAuB,CAAC,CAAC,OAAO,CAAC,eAAe,MAAM;YAChL,CAAC,CAAC,EAAE,CAAC;QAEP,OAAO,4BAA4B,WAAW;cACtC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC,MAAM,UAAU,YAAY;aAC7E,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;UACnC,aAAa;UACb,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,oBAAoB,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;UAC5D,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;UAC1D,WAAW,CAAC,CAAC,CAAC,OAAO,WAAW,OAAO,CAAC,CAAC,CAAC,EAAE;UAC5C,IAAI;iBACG,CAAC;IACd,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;;sBAIa,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;;;;eAmB/B,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;OAChC,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC;IAC1C,WAAW;IACX,IAAI;;QAEA,CAAC;AACT,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,CAAC;SACL,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,KAAa,EACb,SAAe,EACf,OAAyB;IAEzB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAErE,OAAO;QACL,KAAK;QACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM;YACN,MAAM;YACN,OAAO;SACR;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { describe, it } from "node:test";
|
|
6
6
|
import { createEnvRedactor } from "../redact/env-secrets.js";
|
|
7
|
-
import { buildReport, renderScenarioSummaryHtml, writeScenarioTranscript, writeTranscript, } from "./index.js";
|
|
7
|
+
import { buildReport, renderScenarioSummaryHtml, writeReport, writeScenarioTranscript, writeTranscript, } from "./index.js";
|
|
8
8
|
function stubResult(scenario, status) {
|
|
9
9
|
return {
|
|
10
10
|
scenario,
|
|
@@ -33,6 +33,22 @@ describe("renderScenarioSummaryHtml", () => {
|
|
|
33
33
|
assert.equal(renderScenarioSummaryHtml([]), "");
|
|
34
34
|
});
|
|
35
35
|
});
|
|
36
|
+
describe("writeReport", () => {
|
|
37
|
+
it("writes stats.json aggregated across scenarios", () => {
|
|
38
|
+
const runDir = mkdtempSync(path.join(tmpdir(), "pqa-run-stats-"));
|
|
39
|
+
const report = buildReport("test-run", new Date(), [
|
|
40
|
+
stubResult("Alpha Smoke", "pass"),
|
|
41
|
+
stubResult("Beta Flow", "fail"),
|
|
42
|
+
]);
|
|
43
|
+
writeReport(runDir, report);
|
|
44
|
+
const stats = JSON.parse(readFileSync(path.join(runDir, "stats.json"), "utf8"));
|
|
45
|
+
assert.deepEqual(Object.keys(stats.scenarios).sort(), [
|
|
46
|
+
"alpha-smoke",
|
|
47
|
+
"beta-flow",
|
|
48
|
+
]);
|
|
49
|
+
assert.equal(stats.global.durationMs, 25_000);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
36
52
|
describe("writeScenarioTranscript", () => {
|
|
37
53
|
it("writes verdict.json with harness-computed stats", () => {
|
|
38
54
|
const artifactDir = mkdtempSync(path.join(tmpdir(), "pqa-verdict-"));
|
|
@@ -51,6 +67,16 @@ describe("writeScenarioTranscript", () => {
|
|
|
51
67
|
},
|
|
52
68
|
],
|
|
53
69
|
summary: "All checkpoints passed",
|
|
70
|
+
stats: {
|
|
71
|
+
durationMs: 12_750,
|
|
72
|
+
llmTurns: 1,
|
|
73
|
+
userTurns: 1,
|
|
74
|
+
toolCalls: 1,
|
|
75
|
+
failedToolCalls: 0,
|
|
76
|
+
llmDurationMs: 5000,
|
|
77
|
+
bashDurationMs: 200,
|
|
78
|
+
tokens: { input: 42_000, output: 1_200, cached: 10_000 },
|
|
79
|
+
},
|
|
54
80
|
},
|
|
55
81
|
transcript: {
|
|
56
82
|
entries: [
|
|
@@ -89,6 +115,11 @@ describe("writeScenarioTranscript", () => {
|
|
|
89
115
|
assert.equal(written.stats.llmTurns, 1);
|
|
90
116
|
assert.equal(written.stats.userTurns, 1);
|
|
91
117
|
assert.equal(written.stats.toolCalls, 1);
|
|
118
|
+
assert.deepEqual(written.stats.tokens, {
|
|
119
|
+
input: 42_000,
|
|
120
|
+
output: 1_200,
|
|
121
|
+
cached: 10_000,
|
|
122
|
+
});
|
|
92
123
|
});
|
|
93
124
|
});
|
|
94
125
|
describe("writeTranscript", () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/reporter/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EACL,WAAW,EACX,yBAAyB,EACzB,uBAAuB,EACvB,eAAe,GAChB,MAAM,YAAY,CAAC;AAGpB,SAAS,UAAU,CACjB,QAAgB,EAChB,MAAgC;IAEhC,OAAO;QACL,QAAQ;QACR,QAAQ,EAAE,aAAa,QAAQ,KAAK;QACpC,MAAM;QACN,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE,IAAI;QACb,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;KAC5B,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE;YACjD,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC;YACjC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QACrE,uBAAuB,CAAC,WAAW,EAAE;YACnC,QAAQ,EAAE,aAAa;YACvB,QAAQ,EAAE,0BAA0B;YACpC,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,MAAM;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM;gBACd,WAAW,EAAE;oBACX;wBACE,SAAS,EAAE,kBAAkB;wBAC7B,IAAI,EAAE,IAAI;wBACV,MAAM,EAAE,iBAAiB;qBAC1B;iBACF;gBACD,OAAO,EAAE,wBAAwB;
|
|
1
|
+
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/reporter/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EACL,WAAW,EACX,yBAAyB,EACzB,WAAW,EACX,uBAAuB,EACvB,eAAe,GAChB,MAAM,YAAY,CAAC;AAGpB,SAAS,UAAU,CACjB,QAAgB,EAChB,MAAgC;IAEhC,OAAO;QACL,QAAQ;QACR,QAAQ,EAAE,aAAa,QAAQ,KAAK;QACpC,MAAM;QACN,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE,IAAI;QACb,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;KAC5B,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE;YACjD,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC;YACjC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE;YACjD,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC;YACjC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QACH,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAItD,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE;YACpD,aAAa;YACb,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QACrE,uBAAuB,CAAC,WAAW,EAAE;YACnC,QAAQ,EAAE,aAAa;YACvB,QAAQ,EAAE,0BAA0B;YACpC,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,MAAM;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM;gBACd,WAAW,EAAE;oBACX;wBACE,SAAS,EAAE,kBAAkB;wBAC7B,IAAI,EAAE,IAAI;wBACV,MAAM,EAAE,iBAAiB;qBAC1B;iBACF;gBACD,OAAO,EAAE,wBAAwB;gBACjC,KAAK,EAAE;oBACL,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,CAAC;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,IAAI;oBACnB,cAAc,EAAE,GAAG;oBACnB,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACzD;aACF;YACD,UAAU,EAAE;gBACV,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,IAAI;wBACb,EAAE,EAAE,0BAA0B;qBAC/B;oBACD;wBACE,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,qDAAqD;wBAC9D,EAAE,EAAE,0BAA0B;wBAC9B,UAAU,EAAE,IAAI;qBACjB;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,2BAA2B;wBACpC,MAAM,EAAE,IAAI;wBACZ,MAAM,EAAE,EAAE;wBACV,QAAQ,EAAE,CAAC;wBACX,UAAU,EAAE,GAAG;wBACf,EAAE,EAAE,0BAA0B;qBAC/B;iBACF;aACF;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,KAAK;gBACX,aAAa,EAAE,CAAC;gBAChB,eAAe,EAAE,CAAC;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAS7D,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE;YACrC,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE;YAChE,SAAS;SACV,CAAC,CAAC;QACH,eAAe,CACb,WAAW,EACX;YACE,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,0BAA0B,EAAE,EAAE,EAAE,0BAA0B,EAAE;gBAC3G;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,yCAAyC;oBAClD,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,CAAC;oBACb,EAAE,EAAE,0BAA0B;iBAC/B;aACF;SACF,EACD,QAAQ,CACT,CAAC;QACF,MAAM,OAAO,GAAG,YAAY,CAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EACzC,MAAM,CACP,CAAC;QACF,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ScenarioResult, VerdictStats } from "../types/verdict.js";
|
|
2
|
+
export interface RunStats {
|
|
3
|
+
scenarios: Record<string, VerdictStats>;
|
|
4
|
+
global: VerdictStats;
|
|
5
|
+
}
|
|
6
|
+
export declare function scenarioSlug(scenarioName: string): string;
|
|
7
|
+
export declare function resolveScenarioStats(result: ScenarioResult): VerdictStats;
|
|
8
|
+
export declare function aggregateVerdictStats(statsList: VerdictStats[]): VerdictStats;
|
|
9
|
+
export declare function buildRunStats(results: ScenarioResult[]): RunStats;
|
|
10
|
+
//# sourceMappingURL=stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/reporter/stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EAEd,YAAY,EACb,MAAM,qBAAqB,CAAC;AAM7B,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxC,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,YAAY,CAczE;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,YAAY,CA6D7E;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,QAAQ,CAcjE"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { computeTranscriptStats, enrichVerdictWithStats, } from "../agent/verdict.js";
|
|
2
|
+
export function scenarioSlug(scenarioName) {
|
|
3
|
+
return scenarioName.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
|
|
4
|
+
}
|
|
5
|
+
export function resolveScenarioStats(result) {
|
|
6
|
+
if (result.verdict) {
|
|
7
|
+
const enriched = enrichVerdictWithStats(result.verdict, result.transcript, {
|
|
8
|
+
durationMs: result.durationMs,
|
|
9
|
+
healing: result.healing,
|
|
10
|
+
tokens: result.verdict.stats?.tokens,
|
|
11
|
+
});
|
|
12
|
+
return enriched.stats;
|
|
13
|
+
}
|
|
14
|
+
return computeTranscriptStats(result.transcript, {
|
|
15
|
+
durationMs: result.durationMs,
|
|
16
|
+
healing: result.healing,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
export function aggregateVerdictStats(statsList) {
|
|
20
|
+
const aggregated = {
|
|
21
|
+
durationMs: 0,
|
|
22
|
+
llmTurns: 0,
|
|
23
|
+
userTurns: 0,
|
|
24
|
+
toolCalls: 0,
|
|
25
|
+
failedToolCalls: 0,
|
|
26
|
+
llmDurationMs: 0,
|
|
27
|
+
bashDurationMs: 0,
|
|
28
|
+
};
|
|
29
|
+
let tokens;
|
|
30
|
+
let healingUsed = false;
|
|
31
|
+
let recoveryTurns = 0;
|
|
32
|
+
let scenarioRetries = 0;
|
|
33
|
+
let hasHealing = false;
|
|
34
|
+
for (const stats of statsList) {
|
|
35
|
+
aggregated.durationMs += stats.durationMs;
|
|
36
|
+
aggregated.llmTurns += stats.llmTurns;
|
|
37
|
+
aggregated.userTurns += stats.userTurns;
|
|
38
|
+
aggregated.toolCalls += stats.toolCalls;
|
|
39
|
+
aggregated.failedToolCalls += stats.failedToolCalls;
|
|
40
|
+
aggregated.llmDurationMs += stats.llmDurationMs;
|
|
41
|
+
aggregated.bashDurationMs += stats.bashDurationMs;
|
|
42
|
+
if (stats.tokens) {
|
|
43
|
+
tokens ??= { input: 0, output: 0, cached: 0 };
|
|
44
|
+
tokens.input += stats.tokens.input;
|
|
45
|
+
tokens.output += stats.tokens.output;
|
|
46
|
+
tokens.cached = (tokens.cached ?? 0) + (stats.tokens.cached ?? 0);
|
|
47
|
+
}
|
|
48
|
+
if (stats.healing) {
|
|
49
|
+
hasHealing = true;
|
|
50
|
+
healingUsed = healingUsed || stats.healing.used;
|
|
51
|
+
recoveryTurns += stats.healing.recoveryTurns;
|
|
52
|
+
scenarioRetries += stats.healing.scenarioRetries;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (tokens &&
|
|
56
|
+
(tokens.input > 0 || tokens.output > 0 || (tokens.cached ?? 0) > 0)) {
|
|
57
|
+
aggregated.tokens = {
|
|
58
|
+
input: tokens.input,
|
|
59
|
+
output: tokens.output,
|
|
60
|
+
...((tokens.cached ?? 0) > 0 ? { cached: tokens.cached } : {}),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (hasHealing) {
|
|
64
|
+
aggregated.healing = {
|
|
65
|
+
used: healingUsed,
|
|
66
|
+
recoveryTurns,
|
|
67
|
+
scenarioRetries,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return aggregated;
|
|
71
|
+
}
|
|
72
|
+
export function buildRunStats(results) {
|
|
73
|
+
const scenarios = {};
|
|
74
|
+
const allStats = [];
|
|
75
|
+
for (const result of results) {
|
|
76
|
+
const stats = resolveScenarioStats(result);
|
|
77
|
+
scenarios[scenarioSlug(result.scenario)] = stats;
|
|
78
|
+
allStats.push(stats);
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
scenarios,
|
|
82
|
+
global: aggregateVerdictStats(allStats),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/reporter/stats.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,qBAAqB,CAAC;AAO7B,MAAM,UAAU,YAAY,CAAC,YAAoB;IAC/C,OAAO,YAAY,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAsB;IACzD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;YACzE,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM;SACrC,CAAC,CAAC;QACH,OAAO,QAAS,CAAC,KAAM,CAAC;IAC1B,CAAC;IAED,OAAO,sBAAsB,CAAC,MAAM,CAAC,UAAU,EAAE;QAC/C,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,SAAyB;IAC7D,MAAM,UAAU,GAAiB;QAC/B,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;IAEF,IAAI,MAAmC,CAAC;IACxC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,UAAU,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC1C,UAAU,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC;QACtC,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;QACxC,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;QACxC,UAAU,CAAC,eAAe,IAAI,KAAK,CAAC,eAAe,CAAC;QACpD,UAAU,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;QAChD,UAAU,CAAC,cAAc,IAAI,KAAK,CAAC,cAAc,CAAC;QAElD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YAC9C,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,MAAM,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,UAAU,GAAG,IAAI,CAAC;YAClB,WAAW,GAAG,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAChD,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;YAC7C,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IACE,MAAM;QACN,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EACnE,CAAC;QACD,UAAU,CAAC,MAAM,GAAG;YAClB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,OAAO,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,aAAa;YACb,eAAe;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAyB;IACrD,MAAM,SAAS,GAAiC,EAAE,CAAC;IACnD,MAAM,QAAQ,GAAmB,EAAE,CAAC;IAEpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC;QACjD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,qBAAqB,CAAC,QAAQ,CAAC;KACxC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.d.ts","sourceRoot":"","sources":["../../src/reporter/stats.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { aggregateVerdictStats, buildRunStats, resolveScenarioStats, scenarioSlug, } from "./stats.js";
|
|
4
|
+
function stubStats(overrides = {}) {
|
|
5
|
+
return {
|
|
6
|
+
durationMs: 1000,
|
|
7
|
+
llmTurns: 2,
|
|
8
|
+
userTurns: 1,
|
|
9
|
+
toolCalls: 3,
|
|
10
|
+
failedToolCalls: 0,
|
|
11
|
+
llmDurationMs: 800,
|
|
12
|
+
bashDurationMs: 200,
|
|
13
|
+
...overrides,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function stubResult(scenario, overrides = {}) {
|
|
17
|
+
return {
|
|
18
|
+
scenario,
|
|
19
|
+
filePath: `scenarios/${scenario}.md`,
|
|
20
|
+
status: "pass",
|
|
21
|
+
durationMs: 12_750,
|
|
22
|
+
verdict: {
|
|
23
|
+
status: "pass",
|
|
24
|
+
checkpoints: [],
|
|
25
|
+
summary: "ok",
|
|
26
|
+
stats: {
|
|
27
|
+
durationMs: 12_750,
|
|
28
|
+
llmTurns: 1,
|
|
29
|
+
userTurns: 1,
|
|
30
|
+
toolCalls: 1,
|
|
31
|
+
failedToolCalls: 0,
|
|
32
|
+
llmDurationMs: 5000,
|
|
33
|
+
bashDurationMs: 200,
|
|
34
|
+
tokens: { input: 100, output: 10, cached: 5 },
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
transcript: {
|
|
38
|
+
entries: [
|
|
39
|
+
{
|
|
40
|
+
type: "message",
|
|
41
|
+
role: "user",
|
|
42
|
+
content: "go",
|
|
43
|
+
at: "2026-01-01T00:00:00.000Z",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
type: "message",
|
|
47
|
+
role: "assistant",
|
|
48
|
+
content: "done",
|
|
49
|
+
at: "2026-01-01T00:00:05.000Z",
|
|
50
|
+
durationMs: 5000,
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
type: "bash",
|
|
54
|
+
command: "agent-browser snapshot -i",
|
|
55
|
+
stdout: "ok",
|
|
56
|
+
stderr: "",
|
|
57
|
+
exitCode: 0,
|
|
58
|
+
durationMs: 200,
|
|
59
|
+
at: "2026-01-01T00:00:05.000Z",
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
},
|
|
63
|
+
...overrides,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
describe("scenarioSlug", () => {
|
|
67
|
+
it("normalizes scenario names like artifact directories", () => {
|
|
68
|
+
assert.equal(scenarioSlug("VS QA Use"), "vs-qa-use");
|
|
69
|
+
assert.equal(scenarioSlug("hello_world"), "hello-world");
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
describe("resolveScenarioStats", () => {
|
|
73
|
+
it("returns enriched verdict stats when a verdict exists", () => {
|
|
74
|
+
const stats = resolveScenarioStats(stubResult("demo"));
|
|
75
|
+
assert.equal(stats.durationMs, 12_750);
|
|
76
|
+
assert.equal(stats.llmTurns, 1);
|
|
77
|
+
assert.deepEqual(stats.tokens, { input: 100, output: 10, cached: 5 });
|
|
78
|
+
});
|
|
79
|
+
it("computes transcript stats when there is no verdict", () => {
|
|
80
|
+
const stats = resolveScenarioStats(stubResult("demo", {
|
|
81
|
+
verdict: null,
|
|
82
|
+
durationMs: 5200,
|
|
83
|
+
}));
|
|
84
|
+
assert.equal(stats.durationMs, 5200);
|
|
85
|
+
assert.equal(stats.llmTurns, 1);
|
|
86
|
+
assert.equal(stats.toolCalls, 1);
|
|
87
|
+
assert.equal(stats.bashDurationMs, 200);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
describe("aggregateVerdictStats", () => {
|
|
91
|
+
it("sums numeric counters and token usage", () => {
|
|
92
|
+
const global = aggregateVerdictStats([
|
|
93
|
+
stubStats({
|
|
94
|
+
durationMs: 1000,
|
|
95
|
+
llmTurns: 2,
|
|
96
|
+
tokens: { input: 100, output: 10, cached: 5 },
|
|
97
|
+
healing: { used: false, recoveryTurns: 1, scenarioRetries: 0 },
|
|
98
|
+
}),
|
|
99
|
+
stubStats({
|
|
100
|
+
durationMs: 2000,
|
|
101
|
+
llmTurns: 3,
|
|
102
|
+
tokens: { input: 50, output: 20 },
|
|
103
|
+
healing: { used: true, recoveryTurns: 2, scenarioRetries: 1 },
|
|
104
|
+
}),
|
|
105
|
+
]);
|
|
106
|
+
assert.equal(global.durationMs, 3000);
|
|
107
|
+
assert.equal(global.llmTurns, 5);
|
|
108
|
+
assert.deepEqual(global.tokens, { input: 150, output: 30, cached: 5 });
|
|
109
|
+
assert.deepEqual(global.healing, {
|
|
110
|
+
used: true,
|
|
111
|
+
recoveryTurns: 3,
|
|
112
|
+
scenarioRetries: 1,
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
describe("buildRunStats", () => {
|
|
117
|
+
it("builds per-scenario stats and a global aggregate", () => {
|
|
118
|
+
const stats = buildRunStats([
|
|
119
|
+
stubResult("Alpha Smoke", {
|
|
120
|
+
durationMs: 1000,
|
|
121
|
+
verdict: {
|
|
122
|
+
status: "pass",
|
|
123
|
+
checkpoints: [],
|
|
124
|
+
summary: "ok",
|
|
125
|
+
stats: stubStats({
|
|
126
|
+
durationMs: 1000,
|
|
127
|
+
llmTurns: 1,
|
|
128
|
+
tokens: { input: 10, output: 1 },
|
|
129
|
+
}),
|
|
130
|
+
},
|
|
131
|
+
}),
|
|
132
|
+
stubResult("Beta Flow", {
|
|
133
|
+
durationMs: 2000,
|
|
134
|
+
verdict: {
|
|
135
|
+
status: "fail",
|
|
136
|
+
checkpoints: [],
|
|
137
|
+
summary: "nope",
|
|
138
|
+
stats: stubStats({
|
|
139
|
+
durationMs: 2000,
|
|
140
|
+
llmTurns: 2,
|
|
141
|
+
tokens: { input: 20, output: 2 },
|
|
142
|
+
}),
|
|
143
|
+
},
|
|
144
|
+
}),
|
|
145
|
+
]);
|
|
146
|
+
assert.deepEqual(Object.keys(stats.scenarios).sort(), [
|
|
147
|
+
"alpha-smoke",
|
|
148
|
+
"beta-flow",
|
|
149
|
+
]);
|
|
150
|
+
assert.equal(stats.scenarios["alpha-smoke"].durationMs, 1000);
|
|
151
|
+
assert.equal(stats.scenarios["beta-flow"].llmTurns, 1);
|
|
152
|
+
assert.equal(stats.global.durationMs, 3000);
|
|
153
|
+
assert.equal(stats.global.llmTurns, 2);
|
|
154
|
+
assert.deepEqual(stats.global.tokens, { input: 30, output: 3 });
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
//# sourceMappingURL=stats.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.js","sourceRoot":"","sources":["../../src/reporter/stats.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,oBAAoB,EACpB,YAAY,GACb,MAAM,YAAY,CAAC;AAEpB,SAAS,SAAS,CAAC,YAAmC,EAAE;IACtD,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,GAAG;QAClB,cAAc,EAAE,GAAG;QACnB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CACjB,QAAgB,EAChB,YAAqC,EAAE;IAEvC,OAAO;QACL,QAAQ;QACR,QAAQ,EAAE,aAAa,QAAQ,KAAK;QACpC,MAAM,EAAE,MAAM;QACd,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE;YACP,MAAM,EAAE,MAAM;YACd,WAAW,EAAE,EAAE;YACf,OAAO,EAAE,IAAI;YACb,KAAK,EAAE;gBACL,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,eAAe,EAAE,CAAC;gBAClB,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,GAAG;gBACnB,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;aAC9C;SACF;QACD,UAAU,EAAE;YACV,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,IAAI;oBACb,EAAE,EAAE,0BAA0B;iBAC/B;gBACD;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,MAAM;oBACf,EAAE,EAAE,0BAA0B;oBAC9B,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,2BAA2B;oBACpC,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,GAAG;oBACf,EAAE,EAAE,0BAA0B;iBAC/B;aACF;SACF;QACD,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,KAAK,GAAG,oBAAoB,CAChC,UAAU,CAAC,MAAM,EAAE;YACjB,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI;SACjB,CAAC,CACH,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,qBAAqB,CAAC;YACnC,SAAS,CAAC;gBACR,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;gBAC7C,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE;aAC/D,CAAC;YACF,SAAS,CAAC;gBACR,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;gBACjC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE;aAC9D,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE;YAC/B,IAAI,EAAE,IAAI;YACV,aAAa,EAAE,CAAC;YAChB,eAAe,EAAE,CAAC;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,KAAK,GAAG,aAAa,CAAC;YAC1B,UAAU,CAAC,aAAa,EAAE;gBACxB,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE;oBACP,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,EAAE;oBACf,OAAO,EAAE,IAAI;oBACb,KAAK,EAAE,SAAS,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;qBACjC,CAAC;iBACH;aACF,CAAC;YACF,UAAU,CAAC,WAAW,EAAE;gBACtB,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE;oBACP,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,EAAE;oBACf,OAAO,EAAE,MAAM;oBACf,KAAK,EAAE,SAAS,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;qBACjC,CAAC;iBACH;aACF,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE;YACpD,aAAa;YACb,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAE,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/types/config.d.ts
CHANGED
|
@@ -56,7 +56,7 @@ export interface PqaConfig {
|
|
|
56
56
|
model?: string;
|
|
57
57
|
/**
|
|
58
58
|
* Extended thinking / reasoning (opt-in).
|
|
59
|
-
* `budgetTokens`: Anthropic, Fireworks
|
|
59
|
+
* `budgetTokens`: Anthropic, Fireworks & OpenRouter reasoning budget (ignored for Google).
|
|
60
60
|
* `reasoningEffort`: OpenAI reasoning effort; Anthropic effort; Google thinking level (mapped); OpenRouter reasoning effort.
|
|
61
61
|
* Ollama: `think` only (other fields ignored).
|
|
62
62
|
*/
|
package/dist/types/verdict.d.ts
CHANGED
|
@@ -18,6 +18,11 @@ export declare const VerdictSchema: z.ZodObject<{
|
|
|
18
18
|
}, z.core.$strip>>;
|
|
19
19
|
summary: z.ZodString;
|
|
20
20
|
}, z.core.$strip>;
|
|
21
|
+
export declare const TokenUsageStatsSchema: z.ZodObject<{
|
|
22
|
+
input: z.ZodNumber;
|
|
23
|
+
output: z.ZodNumber;
|
|
24
|
+
cached: z.ZodOptional<z.ZodNumber>;
|
|
25
|
+
}, z.core.$strip>;
|
|
21
26
|
export declare const VerdictStatsSchema: z.ZodObject<{
|
|
22
27
|
durationMs: z.ZodNumber;
|
|
23
28
|
llmTurns: z.ZodNumber;
|
|
@@ -26,6 +31,11 @@ export declare const VerdictStatsSchema: z.ZodObject<{
|
|
|
26
31
|
failedToolCalls: z.ZodNumber;
|
|
27
32
|
llmDurationMs: z.ZodNumber;
|
|
28
33
|
bashDurationMs: z.ZodNumber;
|
|
34
|
+
tokens: z.ZodOptional<z.ZodObject<{
|
|
35
|
+
input: z.ZodNumber;
|
|
36
|
+
output: z.ZodNumber;
|
|
37
|
+
cached: z.ZodOptional<z.ZodNumber>;
|
|
38
|
+
}, z.core.$strip>>;
|
|
29
39
|
healing: z.ZodOptional<z.ZodObject<{
|
|
30
40
|
used: z.ZodBoolean;
|
|
31
41
|
recoveryTurns: z.ZodNumber;
|
|
@@ -35,6 +45,7 @@ export declare const VerdictStatsSchema: z.ZodObject<{
|
|
|
35
45
|
export type CheckpointResult = z.infer<typeof CheckpointResultSchema>;
|
|
36
46
|
/** Parsed agent verdict (LLM output). */
|
|
37
47
|
export type ParsedVerdict = z.infer<typeof VerdictSchema>;
|
|
48
|
+
export type TokenUsageStats = z.infer<typeof TokenUsageStatsSchema>;
|
|
38
49
|
export type VerdictStats = z.infer<typeof VerdictStatsSchema>;
|
|
39
50
|
/** Harness-enriched verdict written to verdict.json and report.json. */
|
|
40
51
|
export type Verdict = ParsedVerdict & {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/types/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;;iBAKjC,CAAC;AAEH,eAAO,MAAM,aAAa;;;;;;;;;;;;iBAIxB,CAAC;AAEH,eAAO,MAAM,kBAAkB
|
|
1
|
+
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../src/types/verdict.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,sBAAsB;;;;;iBAKjC,CAAC;AAEH,eAAO,MAAM,aAAa;;;;;;;;;;;;iBAIxB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;iBAgB7B,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,yCAAyC;AACzC,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,CAAC;AAC1D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,wEAAwE;AACxE,MAAM,MAAM,OAAO,GAAG,aAAa,GAAG;IAAE,KAAK,CAAC,EAAE,YAAY,CAAA;CAAE,CAAC;AAE/D,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,gBAAgB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEjF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,EAAE,EAAE,MAAM,CAAC;IACX,6EAA6E;IAC7E,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,mBAAoB,SAAQ,SAAS;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,MAAM,eAAe,GAAG,sBAAsB,GAAG,mBAAmB,CAAC;AAE3E,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,WAAW,CAAC;CACvB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH"}
|