prose-qa 0.2.0 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -28
- package/dist/agent/bash.d.ts +1 -1
- package/dist/agent/bash.d.ts.map +1 -1
- package/dist/agent/bash.js +23 -4
- package/dist/agent/bash.js.map +1 -1
- package/dist/agent/bash.test.js +27 -3
- package/dist/agent/bash.test.js.map +1 -1
- package/dist/agent/prompt.d.ts.map +1 -1
- package/dist/agent/prompt.js +9 -1
- package/dist/agent/prompt.js.map +1 -1
- package/dist/agent/provider-options.d.ts.map +1 -1
- package/dist/agent/provider-options.js +13 -19
- package/dist/agent/provider-options.js.map +1 -1
- package/dist/agent/provider-options.test.js +12 -2
- package/dist/agent/provider-options.test.js.map +1 -1
- package/dist/agent/runner.d.ts.map +1 -1
- package/dist/agent/runner.js +93 -16
- package/dist/agent/runner.js.map +1 -1
- package/dist/agent/runner.test.d.ts +2 -0
- package/dist/agent/runner.test.d.ts.map +1 -0
- package/dist/agent/runner.test.js +83 -0
- package/dist/agent/runner.test.js.map +1 -0
- package/dist/agent/transcript-persist.d.ts +9 -0
- package/dist/agent/transcript-persist.d.ts.map +1 -0
- package/dist/agent/transcript-persist.js +6 -0
- package/dist/agent/transcript-persist.js.map +1 -0
- package/dist/agent/transcript-persist.test.d.ts +2 -0
- package/dist/agent/transcript-persist.test.d.ts.map +1 -0
- package/dist/agent/transcript-persist.test.js +56 -0
- package/dist/agent/transcript-persist.test.js.map +1 -0
- package/dist/agent/verdict.d.ts +12 -3
- package/dist/agent/verdict.d.ts.map +1 -1
- package/dist/agent/verdict.js +81 -0
- package/dist/agent/verdict.js.map +1 -1
- package/dist/agent/verdict.test.js +157 -1
- package/dist/agent/verdict.test.js.map +1 -1
- package/dist/cli/help.d.ts.map +1 -1
- package/dist/cli/help.js +53 -17
- package/dist/cli/help.js.map +1 -1
- package/dist/cli/help.test.js +1 -0
- package/dist/cli/help.test.js.map +1 -1
- package/dist/cli/index.js +18 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/install-browser.d.ts +3 -0
- package/dist/cli/install-browser.d.ts.map +1 -0
- package/dist/cli/install-browser.js +42 -0
- package/dist/cli/install-browser.js.map +1 -0
- package/dist/cli/install-browser.test.d.ts +2 -0
- package/dist/cli/install-browser.test.d.ts.map +1 -0
- package/dist/cli/install-browser.test.js +19 -0
- package/dist/cli/install-browser.test.js.map +1 -0
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +20 -15
- package/dist/cli/run.js.map +1 -1
- package/dist/config/lightpanda.d.ts +8 -0
- package/dist/config/lightpanda.d.ts.map +1 -1
- package/dist/config/lightpanda.js +41 -4
- package/dist/config/lightpanda.js.map +1 -1
- package/dist/config/lightpanda.test.js +30 -1
- package/dist/config/lightpanda.test.js.map +1 -1
- package/dist/config/load.d.ts.map +1 -1
- package/dist/config/load.js +14 -6
- package/dist/config/load.js.map +1 -1
- package/dist/config/load.test.js +29 -0
- package/dist/config/load.test.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +2 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/redact/env-secrets.d.ts.map +1 -1
- package/dist/redact/env-secrets.js +1 -0
- package/dist/redact/env-secrets.js.map +1 -1
- package/dist/reporter/index.d.ts.map +1 -1
- package/dist/reporter/index.js +10 -4
- package/dist/reporter/index.js.map +1 -1
- package/dist/reporter/index.test.js +90 -1
- package/dist/reporter/index.test.js.map +1 -1
- package/dist/reporter/stats.d.ts +10 -0
- package/dist/reporter/stats.d.ts.map +1 -0
- package/dist/reporter/stats.js +85 -0
- package/dist/reporter/stats.js.map +1 -0
- package/dist/reporter/stats.test.d.ts +2 -0
- package/dist/reporter/stats.test.d.ts.map +1 -0
- package/dist/reporter/stats.test.js +157 -0
- package/dist/reporter/stats.test.js.map +1 -0
- package/dist/skills/loader.d.ts +8 -1
- package/dist/skills/loader.d.ts.map +1 -1
- package/dist/skills/loader.js +31 -5
- package/dist/skills/loader.js.map +1 -1
- package/dist/skills/loader.test.d.ts +2 -0
- package/dist/skills/loader.test.d.ts.map +1 -0
- package/dist/skills/loader.test.js +40 -0
- package/dist/skills/loader.test.js.map +1 -0
- package/dist/skills/on-demand.d.ts +57 -0
- package/dist/skills/on-demand.d.ts.map +1 -0
- package/dist/skills/on-demand.js +225 -0
- package/dist/skills/on-demand.js.map +1 -0
- package/dist/skills/on-demand.test.d.ts +2 -0
- package/dist/skills/on-demand.test.d.ts.map +1 -0
- package/dist/skills/on-demand.test.js +111 -0
- package/dist/skills/on-demand.test.js.map +1 -0
- package/dist/types/config.d.ts +15 -4
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/verdict.d.ts +32 -1
- package/dist/types/verdict.d.ts.map +1 -1
- package/dist/types/verdict.js +22 -0
- package/dist/types/verdict.js.map +1 -1
- package/dist/version.d.ts +3 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +6 -0
- package/dist/version.js.map +1 -0
- package/package.json +6 -4
- package/pqa.config.ts +8 -3
- package/prompt/SYSTEM.md +2 -1
- package/scripts/install-lightpanda.mjs +161 -0
- package/scripts/sync-skills.mjs +110 -0
- package/skills/agent-browser/SKILL.md +0 -1960
- package/skills/agent-browser/bundled/agentcore/SKILL.md +115 -0
- package/skills/agent-browser/bundled/dogfood/SKILL.md +220 -0
- package/skills/agent-browser/bundled/dogfood/references/issue-taxonomy.md +109 -0
- package/skills/agent-browser/bundled/dogfood/templates/dogfood-report-template.md +53 -0
- package/skills/agent-browser/bundled/electron/SKILL.md +236 -0
- package/skills/agent-browser/bundled/slack/SKILL.md +285 -0
- package/skills/agent-browser/bundled/slack/references/slack-tasks.md +348 -0
- package/skills/agent-browser/bundled/slack/templates/slack-report-template.md +163 -0
- package/skills/agent-browser/bundled/vercel-sandbox/SKILL.md +280 -0
- package/skills/agent-browser/manifest.json +42 -0
- package/skills/agent-browser/references/authentication.md +303 -0
- package/skills/agent-browser/references/commands.md +398 -0
- package/skills/agent-browser/references/profiling.md +120 -0
- package/skills/agent-browser/references/proxy-support.md +194 -0
- package/skills/agent-browser/references/session-management.md +193 -0
- package/skills/agent-browser/references/snapshot-refs.md +219 -0
- package/skills/agent-browser/references/trust-boundaries.md +89 -0
- package/skills/agent-browser/references/video-recording.md +175 -0
- package/skills/agent-browser/templates/authenticated-session.sh +105 -0
- package/skills/agent-browser/templates/capture-workflow.sh +69 -0
- package/skills/agent-browser/templates/form-automation.sh +62 -0
package/dist/mcp/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,6BAA6B,EAC7B,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACL,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,6BAA6B,EAC7B,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,4FAA4F,CAC7F;IACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IACjE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;IACrE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAClE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IAC1E,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,uCAAuC,CAAC;CACrD,CAAC,CAAC;AAEH,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CAAC,2DAA2D,CAAC;CACzE,CAAC,CAAC;AAEH,SAAS,aAAa,CACpB,IAAwC;IAExC,OAAO;QACL,UAAU,EAAE,IAAI,CAAC,MAAM;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;QAC7B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC;QAC1B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,SAAS,EAAE,YAAY;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,EAC9C;QACE,YAAY,EAAE;YACZ,8EAA8E;YAC9E,QAAQ,6BAA6B,oCAAoC;YACzE,0EAA0E;YAC1E,sEAAsE;SACvE,CAAC,IAAI,CAAC,GAAG,CAAC;KACZ,CACF,CAAC;IAEF,MAAM,CAAC,gBAAgB,CACrB,2BAA2B,EAC3B,6BAA6B,EAC7B;QACE,KAAK,EAAE,2BAA2B;QAClC,WAAW,EACT,uFAAuF;QACzF,QAAQ,EAAE,eAAe;KAC1B,EACD,KAAK,IAAI,EAAE,CAAC,CAAC;QACX,QAAQ,EAAE;YACR;gBACE,GAAG,EAAE,6BAA6B;gBAClC,QAAQ,EAAE,eAAe;gBACzB,IAAI,EAAE,0BAA0B,CAAC,GAAG,CAAC;aACtC;SACF;KACF,CAAC,CACH,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,+BAA+B,EAC/B;QACE,KAAK,EAAE,+BAA+B;QACtC,WAAW,EACT,iFAAiF;QACnF,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC,EACD,KAAK,IAAI,EAAE,CAAC,CAAC;QACX,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,0BAA0B,CAAC,GAAG,CAAC;aACtC;SACF;KACF,CAAC,CACH,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,mBAAmB,EACnB;QACE,KAAK,EAAE,4BAA4B;QACnC,WAAW,EACT,0GAA0G;QAC5G,WAAW,EAAE,uBAAuB;QACpC,WAAW,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE;KACpC,EACD,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,6BAA6B,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC3D,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;gBACxD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAC5B,MAAM,OAAO,GAAG;YACd,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;YAC/B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE;YACrC,GAAG,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG;YAC7B,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC,IAAI;YAC/B,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,OAAO,IAAI,KAAK;YAC9C,eAAe,EAAE,QAAQ,CAAC,WAAW,CAAC,MAAM;YAC5C,OAAO,EAAE,uBAAuB,CAAC,QAAQ,CAAC;SAC3C,CAAC;QACF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;iBACvC;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,cAAc,EACd;QACE,KAAK,EAAE,uBAAuB;QAC9B,WAAW,EACT,kIAAkI;QACpI,WAAW,EAAE,kBAAkB;QAC/B,WAAW,EAAE;YACX,KAAK,EAAE,cAAc;YACrB,eAAe,EAAE,KAAK;SACvB;KACF,EACD,KAAK,EAAE,IAAI,EAAE,EAAE;QACb,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,CAAC,EAAE,QAAQ,EAAE,GAAG,uBAAuB,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,qBAAqB,OAAO,EAAE,EAAE;iBAChE;gBACD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,MAAM,qBAAqB,CAC1C,QAAQ,EACR,MAAM,EACN,aAAa,CAAC,IAAI,CAAC,CACpB,CAAC;QAEF,MAAM,MAAM,GACV,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9D,MAAM,OAAO,GAAG;YACd,MAAM;YACN,QAAQ;YACR,MAAM;YACN,YAAY,EAAE,QAAQ;SACvB,CAAC;QAEF,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;iBACvC;aACF;YACD,OAAO,EAAE,QAAQ,KAAK,CAAC;SACxB,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,cAAc,CACnB,qBAAqB,EACrB;QACE,KAAK,EAAE,4BAA4B;QACnC,WAAW,EACT,2FAA2F;QAC7F,UAAU,EAAE;YACV,IAAI,EAAE,CAAC;iBACJ,MAAM,EAAE;iBACR,QAAQ,EAAE;iBACV,QAAQ,CAAC,qDAAqD,CAAC;SACnE;KACF,EACD,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACjB,MAAM,KAAK,GAAG,0BAA0B,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,kCAAkC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,OAAO;YACL,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAe;oBACrB,OAAO,EAAE;wBACP,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,GAAG,QAAQ,uHAAuH,KAAK,EAAE;qBAChJ;iBACF;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IACjD,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEhC,OAAO,CAAC,KAAK,CACX,sGAAsG,CACvG,CAAC;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC;QAC1C,SAAS,CAAC,OAAO,GAAG,GAAG,EAAE;YACvB,eAAe,EAAE,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env-secrets.d.ts","sourceRoot":"","sources":["../../src/redact/env-secrets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,SAAS,EACT,cAAc,EACd,OAAO,EACR,MAAM,qBAAqB,CAAC;AAS7B,MAAM,WAAW,kBAAkB;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,eAAe,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,CAAC;IAC7C,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,eAAe,CAAC;IAC/D,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC;IACvD,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAAC;CAC9D;AA0BD,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,GAAE,kBAAuB,GAC/B,WAAW,
|
|
1
|
+
{"version":3,"file":"env-secrets.d.ts","sourceRoot":"","sources":["../../src/redact/env-secrets.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,eAAe,EACf,SAAS,EACT,cAAc,EACd,OAAO,EACR,MAAM,qBAAqB,CAAC;AAS7B,MAAM,WAAW,kBAAkB;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,eAAe,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,CAAC;IAC7C,gBAAgB,CAAC,UAAU,EAAE,eAAe,GAAG,eAAe,CAAC;IAC/D,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC;IACvD,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,CAAC;CAC9D;AA0BD,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,GAAE,kBAAuB,GAC/B,WAAW,CAsEb;AAED,wBAAgB,kBAAkB,IAAI,WAAW,CAEhD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env-secrets.js","sourceRoot":"","sources":["../../src/redact/env-secrets.ts"],"names":[],"mappings":"AAOA,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAmBpC,SAAS,iBAAiB,CACxB,GAAsB,EACtB,cAAwB,EACxB,eAAuB;IAEvB,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAErD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe;YAAE,SAAS;QACvD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,GAAG,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,kBAAkB,CAAC,OAAO,EAAE,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvD,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM;QAC/B,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAC7C,CAAC;AACN,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,GAAsB,EACtB,cAAwB,EACxB,UAA8B,EAAE;IAEhC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;IAC7E,MAAM,YAAY,GAAG,iBAAiB,CAAC,GAAG,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;IAE7E,SAAS,MAAM,CAAC,IAAY;QAC1B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,IAAI,GAAG,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,YAAY,EAAE,CAAC;YAClD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,SAAS,eAAe,CAAC,KAAgB;QACvC,OAAO;YACL,GAAG,KAAK;YACR,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAC9B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAC5B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;SAC7B,CAAC;IACJ,CAAC;IAED,SAAS,gBAAgB,CAAC,UAA2B;QACnD,OAAO;YACL,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACxC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC7B,OAAO;wBACL,IAAI,EAAE,SAAkB;wBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;wBAC9B,EAAE,EAAE,KAAK,CAAC,EAAE;wBACZ,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC/D,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC5E,CAAC;gBACJ,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,MAAe,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YAC5E,CAAC,CAAC;SACH,CAAC;IACJ,CAAC;IAED,SAAS,aAAa,CAAC,OAAuB;QAC5C,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,OAAO;YACL,GAAG,OAAO;YACV,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;YAChC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,GAAG,CAAC;gBACJ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBACxB,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC;aAClC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"env-secrets.js","sourceRoot":"","sources":["../../src/redact/env-secrets.ts"],"names":[],"mappings":"AAOA,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAmBpC,SAAS,iBAAiB,CACxB,GAAsB,EACtB,cAAwB,EACxB,eAAuB;IAEvB,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAErD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,eAAe;YAAE,SAAS;QACvD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACnC,kBAAkB,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,GAAG,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,kBAAkB,CAAC,OAAO,EAAE,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;SACvD,IAAI,CACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM;QAC/B,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAC7C,CAAC;AACN,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,GAAsB,EACtB,cAAwB,EACxB,UAA8B,EAAE;IAEhC,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,yBAAyB,CAAC;IAC7E,MAAM,YAAY,GAAG,iBAAiB,CAAC,GAAG,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;IAE7E,SAAS,MAAM,CAAC,IAAY;QAC1B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,IAAI,GAAG,GAAG,IAAI,CAAC;QACf,KAAK,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,YAAY,EAAE,CAAC;YAClD,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,SAAS,eAAe,CAAC,KAAgB;QACvC,OAAO;YACL,GAAG,KAAK;YACR,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAC9B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAC5B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;SAC7B,CAAC;IACJ,CAAC;IAED,SAAS,gBAAgB,CAAC,UAA2B;QACnD,OAAO;YACL,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACxC,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC7B,OAAO;wBACL,IAAI,EAAE,SAAkB;wBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;wBAC9B,EAAE,EAAE,KAAK,CAAC,EAAE;wBACZ,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC/D,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;qBAC5E,CAAC;gBACJ,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,MAAe,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;YAC5E,CAAC,CAAC;SACH,CAAC;IACJ,CAAC;IAED,SAAS,aAAa,CAAC,OAAuB;QAC5C,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,OAAO;YACL,GAAG,OAAO;YACV,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;YAChC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3C,GAAG,CAAC;gBACJ,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;gBACxB,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC;aAClC,CAAC,CAAC;YACH,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnD,CAAC;IACJ,CAAC;IAED,SAAS,oBAAoB,CAAC,MAAsB;QAClD,OAAO;YACL,GAAG,MAAM;YACT,UAAU,EAAE,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC;YAC/C,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC;YACtC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;SACvD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM;QACN,eAAe;QACf,gBAAgB;QAChB,aAAa;QACb,oBAAoB;KACrB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB;IAChC,OAAO,iBAAiB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAI/D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAIhF;AAED,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,SAAS,EACjB,QAAQ,CAAC,EAAE,WAAW,GACrB,IAAI,CAcN;AAED,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,cAAc,CAAC,YAAY,CAAC,EACxC,QAAQ,CAAC,EAAE,WAAW,GACrB,IAAI,CAMN;AAED,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,cAAc,EACtB,QAAQ,CAAC,EAAE,WAAW,GACrB,IAAI,CAkBN;AAuBD,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,MAAM,CAgB3E;AA+ED,wBAAgB,WAAW,CACzB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,IAAI,EACf,OAAO,EAAE,cAAc,EAAE,GACxB,SAAS,CAmBX"}
|
package/dist/reporter/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { getTranscriptBashEntries } from "../agent/verdict.js";
|
|
3
|
+
import { enrichVerdictWithStats, getTranscriptBashEntries } from "../agent/verdict.js";
|
|
4
|
+
import { buildRunStats, scenarioSlug } from "./stats.js";
|
|
4
5
|
export function createRunId() {
|
|
5
6
|
return new Date().toISOString().replace(/[:.]/g, "-");
|
|
6
7
|
}
|
|
@@ -10,8 +11,7 @@ export function ensureRunDir(cwd, runId) {
|
|
|
10
11
|
return dir;
|
|
11
12
|
}
|
|
12
13
|
export function scenarioArtifactDir(runDir, scenarioName) {
|
|
13
|
-
const
|
|
14
|
-
const dir = path.join(runDir, safe);
|
|
14
|
+
const dir = path.join(runDir, scenarioSlug(scenarioName));
|
|
15
15
|
mkdirSync(dir, { recursive: true });
|
|
16
16
|
return dir;
|
|
17
17
|
}
|
|
@@ -22,6 +22,7 @@ export function writeReport(runDir, report, redactor) {
|
|
|
22
22
|
const safeReport = { ...report, results: safeResults };
|
|
23
23
|
writeFileSync(path.join(runDir, "report.json"), `${JSON.stringify(safeReport, null, 2)}\n`);
|
|
24
24
|
writeFileSync(path.join(runDir, "report.html"), renderHtml(safeReport));
|
|
25
|
+
writeFileSync(path.join(runDir, "stats.json"), `${JSON.stringify(buildRunStats(safeResults), null, 2)}\n`);
|
|
25
26
|
}
|
|
26
27
|
export function writeTranscript(artifactDir, transcript, redactor) {
|
|
27
28
|
const safe = redactor ? redactor.redactTranscript(transcript) : transcript;
|
|
@@ -31,7 +32,12 @@ export function writeScenarioTranscript(artifactDir, result, redactor) {
|
|
|
31
32
|
const safe = redactor ? redactor.redactScenarioResult(result) : result;
|
|
32
33
|
writeFileSync(path.join(artifactDir, "transcript.json"), `${JSON.stringify(safe.transcript, null, 2)}\n`);
|
|
33
34
|
if (safe.verdict) {
|
|
34
|
-
|
|
35
|
+
const verdict = enrichVerdictWithStats(safe.verdict, safe.transcript, {
|
|
36
|
+
durationMs: safe.durationMs,
|
|
37
|
+
healing: safe.healing,
|
|
38
|
+
tokens: safe.verdict.stats?.tokens,
|
|
39
|
+
}) ?? safe.verdict;
|
|
40
|
+
writeFileSync(path.join(artifactDir, "verdict.json"), `${JSON.stringify(verdict, null, 2)}\n`);
|
|
35
41
|
}
|
|
36
42
|
}
|
|
37
43
|
function scenarioStatusClass(status) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporter/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEzD,MAAM,UAAU,WAAW;IACzB,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,KAAa;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAClD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAc,EAAE,YAAoB;IACtE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC,CAAC;IAC1D,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,MAAc,EACd,MAAiB,EACjB,QAAsB;IAEtB,MAAM,WAAW,GAAG,QAAQ;QAC1B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;IACnB,MAAM,UAAU,GAAG,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IACvD,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAChC,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAC3C,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;IACxE,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAC/B,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAC3D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,UAAwC,EACxC,QAAsB;IAEtB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IAC3E,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EACzC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,WAAmB,EACnB,MAAsB,EACtB,QAAsB;IAEtB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EACzC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAChD,CAAC;IACF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,OAAO,GACX,sBAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YACpD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM;SACnC,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC;QACrB,aAAa,CACX,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,EACtC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CACxC,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAgC;IAEhC,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACrC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,uBAAuB,CAAC,OAA6B;IAC5D,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC;IAC5D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;IAC/D,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,SAAS,CAAC,CAAC;IAC/D,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,UAAU,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,gBAAgB,CAAC,EAAU;IAClC,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,OAAyB;IACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,KAAK,GAAG,OAAO;SAClB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,WAAW,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClD,OAAO,cAAc,WAAW,yBAAyB,CAAC,CAAC,MAAM,WAAW,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,2BAA2B,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC;IAC5K,CAAC,CAAC;SACD,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClB,OAAO;;;MAGH,KAAK;;aAEE,CAAC;AACd,CAAC;AAED,SAAS,UAAU,CAAC,MAAiB;IACnC,MAAM,WAAW,GAAG,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,WAAW,GAAG,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,WAAW,GACf,CAAC,CAAC,OAAO,EAAE,WAAW;aACnB,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CACrG;aACA,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,wBAAwB,CAAC,CAAC,CAAC,UAAU,CAAC;aAChD,KAAK,CAAC,CAAC,EAAE,CAAC;aACV,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,gBAAgB,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,eAAe,CAClH;aACA,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,MAAM,YAAY,GAAG,CAAC,CAAC,OAAO,EAAE,IAAI;YAClC,CAAC,CAAC,0CAA0C;YAC5C,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,aAAa,GAAG,CAAC,CAAC,OAAO;YAC7B,CAAC,CAAC,oCAAoC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,aAAa,uBAAuB,CAAC,CAAC,OAAO,CAAC,eAAe,MAAM;YAChL,CAAC,CAAC,EAAE,CAAC;QAEP,OAAO,4BAA4B,WAAW;cACtC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC,MAAM,UAAU,YAAY;aAC7E,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;UACnC,aAAa;UACb,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,oBAAoB,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;UAC5D,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;UAC1D,WAAW,CAAC,CAAC,CAAC,OAAO,WAAW,OAAO,CAAC,CAAC,CAAC,EAAE;UAC5C,IAAI;iBACG,CAAC;IACd,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO;;;;sBAIa,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;;;;eAmB/B,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;OAChC,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC;IAC1C,WAAW;IACX,IAAI;;QAEA,CAAC;AACT,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,CAAC;SACL,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,KAAa,EACb,SAAe,EACf,OAAyB;IAEzB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IAErE,OAAO;QACL,KAAK;QACL,SAAS,EAAE,SAAS,CAAC,WAAW,EAAE;QAClC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM;YACN,MAAM;YACN,MAAM;YACN,OAAO;SACR;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { describe, it } from "node:test";
|
|
6
6
|
import { createEnvRedactor } from "../redact/env-secrets.js";
|
|
7
|
-
import { buildReport, renderScenarioSummaryHtml, writeTranscript } from "./index.js";
|
|
7
|
+
import { buildReport, renderScenarioSummaryHtml, writeReport, writeScenarioTranscript, writeTranscript, } from "./index.js";
|
|
8
8
|
function stubResult(scenario, status) {
|
|
9
9
|
return {
|
|
10
10
|
scenario,
|
|
@@ -33,6 +33,95 @@ describe("renderScenarioSummaryHtml", () => {
|
|
|
33
33
|
assert.equal(renderScenarioSummaryHtml([]), "");
|
|
34
34
|
});
|
|
35
35
|
});
|
|
36
|
+
describe("writeReport", () => {
|
|
37
|
+
it("writes stats.json aggregated across scenarios", () => {
|
|
38
|
+
const runDir = mkdtempSync(path.join(tmpdir(), "pqa-run-stats-"));
|
|
39
|
+
const report = buildReport("test-run", new Date(), [
|
|
40
|
+
stubResult("Alpha Smoke", "pass"),
|
|
41
|
+
stubResult("Beta Flow", "fail"),
|
|
42
|
+
]);
|
|
43
|
+
writeReport(runDir, report);
|
|
44
|
+
const stats = JSON.parse(readFileSync(path.join(runDir, "stats.json"), "utf8"));
|
|
45
|
+
assert.deepEqual(Object.keys(stats.scenarios).sort(), [
|
|
46
|
+
"alpha-smoke",
|
|
47
|
+
"beta-flow",
|
|
48
|
+
]);
|
|
49
|
+
assert.equal(stats.global.durationMs, 25_000);
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
describe("writeScenarioTranscript", () => {
|
|
53
|
+
it("writes verdict.json with harness-computed stats", () => {
|
|
54
|
+
const artifactDir = mkdtempSync(path.join(tmpdir(), "pqa-verdict-"));
|
|
55
|
+
writeScenarioTranscript(artifactDir, {
|
|
56
|
+
scenario: "hello-world",
|
|
57
|
+
filePath: "scenarios/hello-world.md",
|
|
58
|
+
status: "pass",
|
|
59
|
+
durationMs: 12_750,
|
|
60
|
+
verdict: {
|
|
61
|
+
status: "pass",
|
|
62
|
+
checkpoints: [
|
|
63
|
+
{
|
|
64
|
+
assertion: "page shows Hello",
|
|
65
|
+
pass: true,
|
|
66
|
+
reason: "heading visible",
|
|
67
|
+
},
|
|
68
|
+
],
|
|
69
|
+
summary: "All checkpoints passed",
|
|
70
|
+
stats: {
|
|
71
|
+
durationMs: 12_750,
|
|
72
|
+
llmTurns: 1,
|
|
73
|
+
userTurns: 1,
|
|
74
|
+
toolCalls: 1,
|
|
75
|
+
failedToolCalls: 0,
|
|
76
|
+
llmDurationMs: 5000,
|
|
77
|
+
bashDurationMs: 200,
|
|
78
|
+
tokens: { input: 42_000, output: 1_200, cached: 10_000 },
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
transcript: {
|
|
82
|
+
entries: [
|
|
83
|
+
{
|
|
84
|
+
type: "message",
|
|
85
|
+
role: "user",
|
|
86
|
+
content: "go",
|
|
87
|
+
at: "2026-01-01T00:00:00.000Z",
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
type: "message",
|
|
91
|
+
role: "assistant",
|
|
92
|
+
content: '[tool bash] {"command":"agent-browser snapshot -i"}',
|
|
93
|
+
at: "2026-01-01T00:00:05.000Z",
|
|
94
|
+
durationMs: 5000,
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
type: "bash",
|
|
98
|
+
command: "agent-browser snapshot -i",
|
|
99
|
+
stdout: "ok",
|
|
100
|
+
stderr: "",
|
|
101
|
+
exitCode: 0,
|
|
102
|
+
durationMs: 200,
|
|
103
|
+
at: "2026-01-01T00:00:05.000Z",
|
|
104
|
+
},
|
|
105
|
+
],
|
|
106
|
+
},
|
|
107
|
+
healing: {
|
|
108
|
+
used: false,
|
|
109
|
+
recoveryTurns: 0,
|
|
110
|
+
scenarioRetries: 0,
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
const written = JSON.parse(readFileSync(path.join(artifactDir, "verdict.json"), "utf8"));
|
|
114
|
+
assert.equal(written.stats.durationMs, 12_750);
|
|
115
|
+
assert.equal(written.stats.llmTurns, 1);
|
|
116
|
+
assert.equal(written.stats.userTurns, 1);
|
|
117
|
+
assert.equal(written.stats.toolCalls, 1);
|
|
118
|
+
assert.deepEqual(written.stats.tokens, {
|
|
119
|
+
input: 42_000,
|
|
120
|
+
output: 1_200,
|
|
121
|
+
cached: 10_000,
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
});
|
|
36
125
|
describe("writeTranscript", () => {
|
|
37
126
|
it("redacts sensitive env values when a redactor is provided", () => {
|
|
38
127
|
const artifactDir = mkdtempSync(path.join(tmpdir(), "pqa-transcript-"));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/reporter/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,
|
|
1
|
+
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../src/reporter/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EACL,WAAW,EACX,yBAAyB,EACzB,WAAW,EACX,uBAAuB,EACvB,eAAe,GAChB,MAAM,YAAY,CAAC;AAGpB,SAAS,UAAU,CACjB,QAAgB,EAChB,MAAgC;IAEhC,OAAO;QACL,QAAQ;QACR,QAAQ,EAAE,aAAa,QAAQ,KAAK;QACpC,MAAM;QACN,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE,IAAI;QACb,UAAU,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;KAC5B,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE;YACjD,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC;YACjC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC7B,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,WAAW,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE;YACjD,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC;YACjC,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC;QACH,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CACtB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAItD,CAAC;QAEF,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE;YACpD,aAAa;YACb,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;QACrE,uBAAuB,CAAC,WAAW,EAAE;YACnC,QAAQ,EAAE,aAAa;YACvB,QAAQ,EAAE,0BAA0B;YACpC,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,MAAM;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM;gBACd,WAAW,EAAE;oBACX;wBACE,SAAS,EAAE,kBAAkB;wBAC7B,IAAI,EAAE,IAAI;wBACV,MAAM,EAAE,iBAAiB;qBAC1B;iBACF;gBACD,OAAO,EAAE,wBAAwB;gBACjC,KAAK,EAAE;oBACL,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,CAAC;oBACZ,SAAS,EAAE,CAAC;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,IAAI;oBACnB,cAAc,EAAE,GAAG;oBACnB,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;iBACzD;aACF;YACD,UAAU,EAAE;gBACV,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,IAAI;wBACb,EAAE,EAAE,0BAA0B;qBAC/B;oBACD;wBACE,IAAI,EAAE,SAAS;wBACf,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,qDAAqD;wBAC9D,EAAE,EAAE,0BAA0B;wBAC9B,UAAU,EAAE,IAAI;qBACjB;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,2BAA2B;wBACpC,MAAM,EAAE,IAAI;wBACZ,MAAM,EAAE,EAAE;wBACV,QAAQ,EAAE,CAAC;wBACX,UAAU,EAAE,GAAG;wBACf,EAAE,EAAE,0BAA0B;qBAC/B;iBACF;aACF;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,KAAK;gBACX,aAAa,EAAE,CAAC;gBAChB,eAAe,EAAE,CAAC;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CACxB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAS7D,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE;YACrC,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,iBAAiB,CAAC,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE;YAChE,SAAS;SACV,CAAC,CAAC;QACH,eAAe,CACb,WAAW,EACX;YACE,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,0BAA0B,EAAE,EAAE,EAAE,0BAA0B,EAAE;gBAC3G;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,yCAAyC;oBAClD,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,CAAC;oBACb,EAAE,EAAE,0BAA0B;iBAC/B;aACF;SACF,EACD,QAAQ,CACT,CAAC;QACF,MAAM,OAAO,GAAG,YAAY,CAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EACzC,MAAM,CACP,CAAC;QACF,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ScenarioResult, VerdictStats } from "../types/verdict.js";
|
|
2
|
+
export interface RunStats {
|
|
3
|
+
scenarios: Record<string, VerdictStats>;
|
|
4
|
+
global: VerdictStats;
|
|
5
|
+
}
|
|
6
|
+
export declare function scenarioSlug(scenarioName: string): string;
|
|
7
|
+
export declare function resolveScenarioStats(result: ScenarioResult): VerdictStats;
|
|
8
|
+
export declare function aggregateVerdictStats(statsList: VerdictStats[]): VerdictStats;
|
|
9
|
+
export declare function buildRunStats(results: ScenarioResult[]): RunStats;
|
|
10
|
+
//# sourceMappingURL=stats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../src/reporter/stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EAEd,YAAY,EACb,MAAM,qBAAqB,CAAC;AAM7B,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACxC,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,cAAc,GAAG,YAAY,CAczE;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,YAAY,CA6D7E;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,cAAc,EAAE,GAAG,QAAQ,CAcjE"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { computeTranscriptStats, enrichVerdictWithStats, } from "../agent/verdict.js";
|
|
2
|
+
export function scenarioSlug(scenarioName) {
|
|
3
|
+
return scenarioName.replace(/[^a-z0-9-]/gi, "-").toLowerCase();
|
|
4
|
+
}
|
|
5
|
+
export function resolveScenarioStats(result) {
|
|
6
|
+
if (result.verdict) {
|
|
7
|
+
const enriched = enrichVerdictWithStats(result.verdict, result.transcript, {
|
|
8
|
+
durationMs: result.durationMs,
|
|
9
|
+
healing: result.healing,
|
|
10
|
+
tokens: result.verdict.stats?.tokens,
|
|
11
|
+
});
|
|
12
|
+
return enriched.stats;
|
|
13
|
+
}
|
|
14
|
+
return computeTranscriptStats(result.transcript, {
|
|
15
|
+
durationMs: result.durationMs,
|
|
16
|
+
healing: result.healing,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
export function aggregateVerdictStats(statsList) {
|
|
20
|
+
const aggregated = {
|
|
21
|
+
durationMs: 0,
|
|
22
|
+
llmTurns: 0,
|
|
23
|
+
userTurns: 0,
|
|
24
|
+
toolCalls: 0,
|
|
25
|
+
failedToolCalls: 0,
|
|
26
|
+
llmDurationMs: 0,
|
|
27
|
+
bashDurationMs: 0,
|
|
28
|
+
};
|
|
29
|
+
let tokens;
|
|
30
|
+
let healingUsed = false;
|
|
31
|
+
let recoveryTurns = 0;
|
|
32
|
+
let scenarioRetries = 0;
|
|
33
|
+
let hasHealing = false;
|
|
34
|
+
for (const stats of statsList) {
|
|
35
|
+
aggregated.durationMs += stats.durationMs;
|
|
36
|
+
aggregated.llmTurns += stats.llmTurns;
|
|
37
|
+
aggregated.userTurns += stats.userTurns;
|
|
38
|
+
aggregated.toolCalls += stats.toolCalls;
|
|
39
|
+
aggregated.failedToolCalls += stats.failedToolCalls;
|
|
40
|
+
aggregated.llmDurationMs += stats.llmDurationMs;
|
|
41
|
+
aggregated.bashDurationMs += stats.bashDurationMs;
|
|
42
|
+
if (stats.tokens) {
|
|
43
|
+
tokens ??= { input: 0, output: 0, cached: 0 };
|
|
44
|
+
tokens.input += stats.tokens.input;
|
|
45
|
+
tokens.output += stats.tokens.output;
|
|
46
|
+
tokens.cached = (tokens.cached ?? 0) + (stats.tokens.cached ?? 0);
|
|
47
|
+
}
|
|
48
|
+
if (stats.healing) {
|
|
49
|
+
hasHealing = true;
|
|
50
|
+
healingUsed = healingUsed || stats.healing.used;
|
|
51
|
+
recoveryTurns += stats.healing.recoveryTurns;
|
|
52
|
+
scenarioRetries += stats.healing.scenarioRetries;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (tokens &&
|
|
56
|
+
(tokens.input > 0 || tokens.output > 0 || (tokens.cached ?? 0) > 0)) {
|
|
57
|
+
aggregated.tokens = {
|
|
58
|
+
input: tokens.input,
|
|
59
|
+
output: tokens.output,
|
|
60
|
+
...((tokens.cached ?? 0) > 0 ? { cached: tokens.cached } : {}),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (hasHealing) {
|
|
64
|
+
aggregated.healing = {
|
|
65
|
+
used: healingUsed,
|
|
66
|
+
recoveryTurns,
|
|
67
|
+
scenarioRetries,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return aggregated;
|
|
71
|
+
}
|
|
72
|
+
export function buildRunStats(results) {
|
|
73
|
+
const scenarios = {};
|
|
74
|
+
const allStats = [];
|
|
75
|
+
for (const result of results) {
|
|
76
|
+
const stats = resolveScenarioStats(result);
|
|
77
|
+
scenarios[scenarioSlug(result.scenario)] = stats;
|
|
78
|
+
allStats.push(stats);
|
|
79
|
+
}
|
|
80
|
+
return {
|
|
81
|
+
scenarios,
|
|
82
|
+
global: aggregateVerdictStats(allStats),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/reporter/stats.ts"],"names":[],"mappings":"AAKA,OAAO,EACL,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,qBAAqB,CAAC;AAO7B,MAAM,UAAU,YAAY,CAAC,YAAoB;IAC/C,OAAO,YAAY,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAsB;IACzD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;YACzE,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM;SACrC,CAAC,CAAC;QACH,OAAO,QAAS,CAAC,KAAM,CAAC;IAC1B,CAAC;IAED,OAAO,sBAAsB,CAAC,MAAM,CAAC,UAAU,EAAE;QAC/C,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,OAAO,EAAE,MAAM,CAAC,OAAO;KACxB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,SAAyB;IAC7D,MAAM,UAAU,GAAiB;QAC/B,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;IAEF,IAAI,MAAmC,CAAC;IACxC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,UAAU,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;QAC1C,UAAU,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC;QACtC,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;QACxC,UAAU,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;QACxC,UAAU,CAAC,eAAe,IAAI,KAAK,CAAC,eAAe,CAAC;QACpD,UAAU,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;QAChD,UAAU,CAAC,cAAc,IAAI,KAAK,CAAC,cAAc,CAAC;QAElD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YAC9C,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;YACnC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,MAAM,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,UAAU,GAAG,IAAI,CAAC;YAClB,WAAW,GAAG,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAChD,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;YAC7C,eAAe,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IACE,MAAM;QACN,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EACnE,CAAC;QACD,UAAU,CAAC,MAAM,GAAG;YAClB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;IACJ,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,OAAO,GAAG;YACnB,IAAI,EAAE,WAAW;YACjB,aAAa;YACb,eAAe;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAyB;IACrD,MAAM,SAAS,GAAiC,EAAE,CAAC;IACnD,MAAM,QAAQ,GAAmB,EAAE,CAAC;IAEpC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CAAC;QACjD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,qBAAqB,CAAC,QAAQ,CAAC;KACxC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.d.ts","sourceRoot":"","sources":["../../src/reporter/stats.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { describe, it } from "node:test";
|
|
3
|
+
import { aggregateVerdictStats, buildRunStats, resolveScenarioStats, scenarioSlug, } from "./stats.js";
|
|
4
|
+
function stubStats(overrides = {}) {
|
|
5
|
+
return {
|
|
6
|
+
durationMs: 1000,
|
|
7
|
+
llmTurns: 2,
|
|
8
|
+
userTurns: 1,
|
|
9
|
+
toolCalls: 3,
|
|
10
|
+
failedToolCalls: 0,
|
|
11
|
+
llmDurationMs: 800,
|
|
12
|
+
bashDurationMs: 200,
|
|
13
|
+
...overrides,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function stubResult(scenario, overrides = {}) {
|
|
17
|
+
return {
|
|
18
|
+
scenario,
|
|
19
|
+
filePath: `scenarios/${scenario}.md`,
|
|
20
|
+
status: "pass",
|
|
21
|
+
durationMs: 12_750,
|
|
22
|
+
verdict: {
|
|
23
|
+
status: "pass",
|
|
24
|
+
checkpoints: [],
|
|
25
|
+
summary: "ok",
|
|
26
|
+
stats: {
|
|
27
|
+
durationMs: 12_750,
|
|
28
|
+
llmTurns: 1,
|
|
29
|
+
userTurns: 1,
|
|
30
|
+
toolCalls: 1,
|
|
31
|
+
failedToolCalls: 0,
|
|
32
|
+
llmDurationMs: 5000,
|
|
33
|
+
bashDurationMs: 200,
|
|
34
|
+
tokens: { input: 100, output: 10, cached: 5 },
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
transcript: {
|
|
38
|
+
entries: [
|
|
39
|
+
{
|
|
40
|
+
type: "message",
|
|
41
|
+
role: "user",
|
|
42
|
+
content: "go",
|
|
43
|
+
at: "2026-01-01T00:00:00.000Z",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
type: "message",
|
|
47
|
+
role: "assistant",
|
|
48
|
+
content: "done",
|
|
49
|
+
at: "2026-01-01T00:00:05.000Z",
|
|
50
|
+
durationMs: 5000,
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
type: "bash",
|
|
54
|
+
command: "agent-browser snapshot -i",
|
|
55
|
+
stdout: "ok",
|
|
56
|
+
stderr: "",
|
|
57
|
+
exitCode: 0,
|
|
58
|
+
durationMs: 200,
|
|
59
|
+
at: "2026-01-01T00:00:05.000Z",
|
|
60
|
+
},
|
|
61
|
+
],
|
|
62
|
+
},
|
|
63
|
+
...overrides,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
describe("scenarioSlug", () => {
|
|
67
|
+
it("normalizes scenario names like artifact directories", () => {
|
|
68
|
+
assert.equal(scenarioSlug("VS QA Use"), "vs-qa-use");
|
|
69
|
+
assert.equal(scenarioSlug("hello_world"), "hello-world");
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
describe("resolveScenarioStats", () => {
|
|
73
|
+
it("returns enriched verdict stats when a verdict exists", () => {
|
|
74
|
+
const stats = resolveScenarioStats(stubResult("demo"));
|
|
75
|
+
assert.equal(stats.durationMs, 12_750);
|
|
76
|
+
assert.equal(stats.llmTurns, 1);
|
|
77
|
+
assert.deepEqual(stats.tokens, { input: 100, output: 10, cached: 5 });
|
|
78
|
+
});
|
|
79
|
+
it("computes transcript stats when there is no verdict", () => {
|
|
80
|
+
const stats = resolveScenarioStats(stubResult("demo", {
|
|
81
|
+
verdict: null,
|
|
82
|
+
durationMs: 5200,
|
|
83
|
+
}));
|
|
84
|
+
assert.equal(stats.durationMs, 5200);
|
|
85
|
+
assert.equal(stats.llmTurns, 1);
|
|
86
|
+
assert.equal(stats.toolCalls, 1);
|
|
87
|
+
assert.equal(stats.bashDurationMs, 200);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
describe("aggregateVerdictStats", () => {
|
|
91
|
+
it("sums numeric counters and token usage", () => {
|
|
92
|
+
const global = aggregateVerdictStats([
|
|
93
|
+
stubStats({
|
|
94
|
+
durationMs: 1000,
|
|
95
|
+
llmTurns: 2,
|
|
96
|
+
tokens: { input: 100, output: 10, cached: 5 },
|
|
97
|
+
healing: { used: false, recoveryTurns: 1, scenarioRetries: 0 },
|
|
98
|
+
}),
|
|
99
|
+
stubStats({
|
|
100
|
+
durationMs: 2000,
|
|
101
|
+
llmTurns: 3,
|
|
102
|
+
tokens: { input: 50, output: 20 },
|
|
103
|
+
healing: { used: true, recoveryTurns: 2, scenarioRetries: 1 },
|
|
104
|
+
}),
|
|
105
|
+
]);
|
|
106
|
+
assert.equal(global.durationMs, 3000);
|
|
107
|
+
assert.equal(global.llmTurns, 5);
|
|
108
|
+
assert.deepEqual(global.tokens, { input: 150, output: 30, cached: 5 });
|
|
109
|
+
assert.deepEqual(global.healing, {
|
|
110
|
+
used: true,
|
|
111
|
+
recoveryTurns: 3,
|
|
112
|
+
scenarioRetries: 1,
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
describe("buildRunStats", () => {
|
|
117
|
+
it("builds per-scenario stats and a global aggregate", () => {
|
|
118
|
+
const stats = buildRunStats([
|
|
119
|
+
stubResult("Alpha Smoke", {
|
|
120
|
+
durationMs: 1000,
|
|
121
|
+
verdict: {
|
|
122
|
+
status: "pass",
|
|
123
|
+
checkpoints: [],
|
|
124
|
+
summary: "ok",
|
|
125
|
+
stats: stubStats({
|
|
126
|
+
durationMs: 1000,
|
|
127
|
+
llmTurns: 1,
|
|
128
|
+
tokens: { input: 10, output: 1 },
|
|
129
|
+
}),
|
|
130
|
+
},
|
|
131
|
+
}),
|
|
132
|
+
stubResult("Beta Flow", {
|
|
133
|
+
durationMs: 2000,
|
|
134
|
+
verdict: {
|
|
135
|
+
status: "fail",
|
|
136
|
+
checkpoints: [],
|
|
137
|
+
summary: "nope",
|
|
138
|
+
stats: stubStats({
|
|
139
|
+
durationMs: 2000,
|
|
140
|
+
llmTurns: 2,
|
|
141
|
+
tokens: { input: 20, output: 2 },
|
|
142
|
+
}),
|
|
143
|
+
},
|
|
144
|
+
}),
|
|
145
|
+
]);
|
|
146
|
+
assert.deepEqual(Object.keys(stats.scenarios).sort(), [
|
|
147
|
+
"alpha-smoke",
|
|
148
|
+
"beta-flow",
|
|
149
|
+
]);
|
|
150
|
+
assert.equal(stats.scenarios["alpha-smoke"].durationMs, 1000);
|
|
151
|
+
assert.equal(stats.scenarios["beta-flow"].llmTurns, 1);
|
|
152
|
+
assert.equal(stats.global.durationMs, 3000);
|
|
153
|
+
assert.equal(stats.global.llmTurns, 2);
|
|
154
|
+
assert.deepEqual(stats.global.tokens, { input: 30, output: 3 });
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
//# sourceMappingURL=stats.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.js","sourceRoot":"","sources":["../../src/reporter/stats.test.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAEzC,OAAO,EACL,qBAAqB,EACrB,aAAa,EACb,oBAAoB,EACpB,YAAY,GACb,MAAM,YAAY,CAAC;AAEpB,SAAS,SAAS,CAAC,YAAmC,EAAE;IACtD,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,CAAC;QACX,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,eAAe,EAAE,CAAC;QAClB,aAAa,EAAE,GAAG;QAClB,cAAc,EAAE,GAAG;QACnB,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CACjB,QAAgB,EAChB,YAAqC,EAAE;IAEvC,OAAO;QACL,QAAQ;QACR,QAAQ,EAAE,aAAa,QAAQ,KAAK;QACpC,MAAM,EAAE,MAAM;QACd,UAAU,EAAE,MAAM;QAClB,OAAO,EAAE;YACP,MAAM,EAAE,MAAM;YACd,WAAW,EAAE,EAAE;YACf,OAAO,EAAE,IAAI;YACb,KAAK,EAAE;gBACL,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,eAAe,EAAE,CAAC;gBAClB,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,GAAG;gBACnB,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;aAC9C;SACF;QACD,UAAU,EAAE;YACV,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,IAAI;oBACb,EAAE,EAAE,0BAA0B;iBAC/B;gBACD;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,MAAM;oBACf,EAAE,EAAE,0BAA0B;oBAC9B,UAAU,EAAE,IAAI;iBACjB;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,2BAA2B;oBACpC,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,CAAC;oBACX,UAAU,EAAE,GAAG;oBACf,EAAE,EAAE,0BAA0B;iBAC/B;aACF;SACF;QACD,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,KAAK,GAAG,oBAAoB,CAChC,UAAU,CAAC,MAAM,EAAE;YACjB,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI;SACjB,CAAC,CACH,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,MAAM,GAAG,qBAAqB,CAAC;YACnC,SAAS,CAAC;gBACR,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;gBAC7C,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE;aAC/D,CAAC;YACF,SAAS,CAAC;gBACR,UAAU,EAAE,IAAI;gBAChB,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;gBACjC,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE;aAC9D,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACjC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE;YAC/B,IAAI,EAAE,IAAI;YACV,aAAa,EAAE,CAAC;YAChB,eAAe,EAAE,CAAC;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,KAAK,GAAG,aAAa,CAAC;YAC1B,UAAU,CAAC,aAAa,EAAE;gBACxB,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE;oBACP,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,EAAE;oBACf,OAAO,EAAE,IAAI;oBACb,KAAK,EAAE,SAAS,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;qBACjC,CAAC;iBACH;aACF,CAAC;YACF,UAAU,CAAC,WAAW,EAAE;gBACtB,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE;oBACP,MAAM,EAAE,MAAM;oBACd,WAAW,EAAE,EAAE;oBACf,OAAO,EAAE,MAAM;oBACf,KAAK,EAAE,SAAS,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE,CAAC;wBACX,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE;qBACjC,CAAC;iBACH;aACF,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE;YACpD,aAAa;YACb,WAAW;SACZ,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAE,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/skills/loader.d.ts
CHANGED
|
@@ -1,10 +1,17 @@
|
|
|
1
1
|
import { type Skill, type SkillCatalogEntry } from "../types/skill.js";
|
|
2
|
+
/** Vendored agent-browser skill always injected into the system prompt. */
|
|
3
|
+
export declare const CORE_SKILL_NAME = "core";
|
|
4
|
+
export declare function loadBundledCoreSkill(cwd: string): Skill;
|
|
2
5
|
export declare function discoverSkills(dirs: string[], cwd: string): Skill[];
|
|
6
|
+
/** Skill names always preloaded: bundled core + config skills.preloads. */
|
|
7
|
+
export declare function resolveBaseSkillNames(preloads?: string[]): string[];
|
|
3
8
|
export declare function catalog(skills: Skill[]): SkillCatalogEntry[];
|
|
4
9
|
export declare function getSkill(skills: Skill[], name: string): Skill | undefined;
|
|
5
10
|
export declare function requireSkills(skills: Skill[], names: string[]): Skill[];
|
|
6
11
|
export declare function mergeSkillNames(...groups: string[][]): string[];
|
|
7
12
|
export declare function resolveSkills(catalog: Skill[], ...nameGroups: string[][]): Skill[];
|
|
8
|
-
export declare function buildSkillPrompt(skills: Skill[]
|
|
13
|
+
export declare function buildSkillPrompt(skills: Skill[], options?: {
|
|
14
|
+
onDemandCatalog?: string;
|
|
15
|
+
}): string;
|
|
9
16
|
export declare function verifyBundledSkill(cwd: string): void;
|
|
10
17
|
//# sourceMappingURL=loader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/skills/loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,KAAK,EACV,KAAK,iBAAiB,EACvB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/skills/loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,KAAK,EACV,KAAK,iBAAiB,EACvB,MAAM,mBAAmB,CAAC;AAM3B,2EAA2E;AAC3E,eAAO,MAAM,eAAe,SAAS,CAAC;AAsCtC,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CASvD;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,KAAK,EAAE,CAsBnE;AAED,2EAA2E;AAC3E,wBAAgB,qBAAqB,CAAC,QAAQ,GAAE,MAAM,EAAO,GAAG,MAAM,EAAE,CAEvE;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,iBAAiB,EAAE,CAE5D;AAED,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,SAAS,CAEzE;AAED,wBAAgB,aAAa,CAC3B,MAAM,EAAE,KAAK,EAAE,EACf,KAAK,EAAE,MAAM,EAAE,GACd,KAAK,EAAE,CAQT;AAED,wBAAgB,eAAe,CAAC,GAAG,MAAM,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,CAW/D;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,KAAK,EAAE,EAChB,GAAG,UAAU,EAAE,MAAM,EAAE,EAAE,GACxB,KAAK,EAAE,CAET;AAED,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,KAAK,EAAE,EACf,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,GACrC,MAAM,CAMR;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAiBpD"}
|