reeval 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +4 -4
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +84 -33
- package/dist/agent-BYy0Tc6P.js +334 -0
- package/dist/agent-BYy0Tc6P.js.map +1 -0
- package/dist/{agent-D-6z0Z3F.js → agent-DcQAvomp.js} +4 -4
- package/dist/{agent-D-6z0Z3F.js.map → agent-DcQAvomp.js.map} +1 -1
- package/dist/analysis-CxSCt1YY.js.map +1 -1
- package/dist/assemble-BFsyMhb-.js +2 -0
- package/dist/{assemble-DtouO0iQ.js → assemble-h8s0hzTQ.js} +22 -14
- package/dist/assemble-h8s0hzTQ.js.map +1 -0
- package/dist/{cli-CGuUdOPK.js → cli-DemDdg8J.js} +165 -54
- package/dist/cli-DemDdg8J.js.map +1 -0
- package/dist/{cli-spec-DKAu09Gi.js → cli-spec-BxLPgNN-.js} +8 -2
- package/dist/cli-spec-BxLPgNN-.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{command-CCuYRGsI.js → command-CxwnRrTs.js} +11 -11
- package/dist/{command-CCuYRGsI.js.map → command-CxwnRrTs.js.map} +1 -1
- package/dist/command-DhgZPhsY.js +2 -0
- package/dist/{command-BekzrvZI.js → command-PEbIzJj3.js} +12 -12
- package/dist/{command-BekzrvZI.js.map → command-PEbIzJj3.js.map} +1 -1
- package/dist/{command-CLKuXn6Z.js → command-pHfluVCF.js} +11 -9
- package/dist/command-pHfluVCF.js.map +1 -0
- package/dist/{command-CEm8PZ1N.js → command-y91-aNhu.js} +169 -280
- package/dist/command-y91-aNhu.js.map +1 -0
- package/dist/{commands-A4udfyKZ.js → commands-De-SxIgY.js} +1 -1
- package/dist/{commands-CF8mOfiu.js → commands-DhCuEa6z.js} +6 -6
- package/dist/{commands-CF8mOfiu.js.map → commands-DhCuEa6z.js.map} +1 -1
- package/dist/{config-DqJvSqYo.js → config-CqSLZkwx.js} +39 -14
- package/dist/{config-DqJvSqYo.js.map → config-CqSLZkwx.js.map} +1 -1
- package/dist/{db-DxDzMhtB.js → db-DA4LBwi_.js} +4 -4
- package/dist/{db-DxDzMhtB.js.map → db-DA4LBwi_.js.map} +1 -1
- package/dist/db-DDfq7Tx_.js +2 -0
- package/dist/{dist-BZXYHN2X.js → dist-BSoys6pg.js} +2 -2
- package/dist/{dist-BZXYHN2X.js.map → dist-BSoys6pg.js.map} +1 -1
- package/dist/{docker-DQDZWXF9.js → docker-DMRwn7yh.js} +2 -2
- package/dist/{docker-DQDZWXF9.js.map → docker-DMRwn7yh.js.map} +1 -1
- package/dist/{docker-w0CSdQq0.js → docker-DwG9f76x.js} +4 -4
- package/dist/{docker-w0CSdQq0.js.map → docker-DwG9f76x.js.map} +1 -1
- package/dist/{docker-Buu3a60z.js → docker-jf52ARP7.js} +1 -1
- package/dist/{doctor-Bs1_pG9_.js → doctor-DhvVbNKv.js} +23 -18
- package/dist/doctor-DhvVbNKv.js.map +1 -0
- package/dist/{egress-QJMzVtbw.js → egress-D9vJCATi.js} +35 -19
- package/dist/egress-D9vJCATi.js.map +1 -0
- package/dist/{environment-9WHSWLxH.js → environment-ByCF9s3n.js} +2 -2
- package/dist/{environment-9WHSWLxH.js.map → environment-ByCF9s3n.js.map} +1 -1
- package/dist/files-B16cQ6XX.js +2 -0
- package/dist/files-BC5Yqir-.js +2 -0
- package/dist/{files-bF2fPuT9.js → files-CfsIRN7n.js} +40 -7
- package/dist/files-CfsIRN7n.js.map +1 -0
- package/dist/{files-D2-qn_Dl.js → files-DHBBIYx_.js} +6 -6
- package/dist/{files-D2-qn_Dl.js.map → files-DHBBIYx_.js.map} +1 -1
- package/dist/{git-D6hMeieS.js → git-oX92_Jh-.js} +2 -2
- package/dist/{git-D6hMeieS.js.map → git-oX92_Jh-.js.map} +1 -1
- package/dist/{harness-ls5aX1UO.js → harness-BbMShUC-.js} +4 -4
- package/dist/{harness-ls5aX1UO.js.map → harness-BbMShUC-.js.map} +1 -1
- package/dist/harness.d.ts +2 -2
- package/dist/harness.js +1 -1
- package/dist/{header-DzFDoMCI.js → header-sPQ9Lq2S.js} +2 -2
- package/dist/{header-DzFDoMCI.js.map → header-sPQ9Lq2S.js.map} +1 -1
- package/dist/{images-2m2U-UQc.js → images-C9Y4ijbG.js} +6 -6
- package/dist/{images-2m2U-UQc.js.map → images-C9Y4ijbG.js.map} +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +23 -23
- package/dist/index.js.map +1 -1
- package/dist/init-DPJv7OmX.js +2 -0
- package/dist/{init-CH9UY7Bt.js → init-x55uF8a4.js} +10 -10
- package/dist/{init-CH9UY7Bt.js.map → init-x55uF8a4.js.map} +1 -1
- package/dist/{invocations-rXktgSAn.js → invocations-DSMfs3Dx.js} +3 -3
- package/dist/{invocations-rXktgSAn.js.map → invocations-DSMfs3Dx.js.map} +1 -1
- package/dist/{jiti-BOLXpKyG.js → jiti-DWsGEgxD.js} +4 -4
- package/dist/{jiti-BOLXpKyG.js.map → jiti-DWsGEgxD.js.map} +1 -1
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +8 -3
- package/dist/judge.js.map +1 -1
- package/dist/{live-Bs3LReMV.js → live-DZkCtmNP.js} +2 -2
- package/dist/{live-Bs3LReMV.js.map → live-DZkCtmNP.js.map} +1 -1
- package/dist/{matchers-DyLvm7EG.js → matchers-D51lM8dO.js} +4 -4
- package/dist/{matchers-DyLvm7EG.js.map → matchers-D51lM8dO.js.map} +1 -1
- package/dist/model-CNoc8t2Y.js +2 -0
- package/dist/model-DQQQ3q_n.js +2 -0
- package/dist/model-GFM8iOzM.js +228 -0
- package/dist/model-GFM8iOzM.js.map +1 -0
- package/dist/{model-CDL98xue.js → model-ORtLm2UB.js} +41 -3
- package/dist/{model-CDL98xue.js.map → model-ORtLm2UB.js.map} +1 -1
- package/dist/{node-kG64XxdE.js → node-OXPxBD2N.js} +3 -3
- package/dist/{node-kG64XxdE.js.map → node-OXPxBD2N.js.map} +1 -1
- package/dist/{opencode-providers-DZuD_oYt.js → opencode-providers-DJVocFOU.js} +31 -3
- package/dist/opencode-providers-DJVocFOU.js.map +1 -0
- package/dist/{options-k25fjaER.js → options-CIutEeuy.js} +62 -20
- package/dist/options-CIutEeuy.js.map +1 -0
- package/dist/{plan-gwk6dJ8c.js → plan-DcHTdiJN.js} +21 -5
- package/dist/plan-DcHTdiJN.js.map +1 -0
- package/dist/{preflight-D2Hce5Jp.js → preflight-BQyUr_hW.js} +1 -1
- package/dist/{preflight-B6hfAXhh.js → preflight-vISQIiF1.js} +3 -3
- package/dist/{preflight-B6hfAXhh.js.map → preflight-vISQIiF1.js.map} +1 -1
- package/dist/{prepare-NuWsT11j.js → prepare-BTeAiW9Q.js} +5 -5
- package/dist/{prepare-NuWsT11j.js.map → prepare-BTeAiW9Q.js.map} +1 -1
- package/dist/{prepare-C3fefNnV.js → prepare-Ca0Nxd9b.js} +2 -2
- package/dist/{provenance-Dfb8YY5Y.js → provenance-D4vo7RF6.js} +21 -3
- package/dist/provenance-D4vo7RF6.js.map +1 -0
- package/dist/{providers-Bfx5Mnk9.js → providers-Bjgky1u8.js} +3 -3
- package/dist/{providers-Bfx5Mnk9.js.map → providers-Bjgky1u8.js.map} +1 -1
- package/dist/{providers-BMIGzPPG.js → providers-DfVMglqP.js} +1 -1
- package/dist/query-CWkZnV62.js +2 -0
- package/dist/{query-CFATHZmV.js → query-D2KaStTs.js} +2 -2
- package/dist/{query-CFATHZmV.js.map → query-D2KaStTs.js.map} +1 -1
- package/dist/questions-CzXdVVsh.js +325 -0
- package/dist/questions-CzXdVVsh.js.map +1 -0
- package/dist/{record-DQ8wzyco.js → record-CbuQH4l1.js} +7 -3
- package/dist/record-CbuQH4l1.js.map +1 -0
- package/dist/{results-BtvBJAp-.js → results-BI-zTqQy.js} +4 -4
- package/dist/{results-BtvBJAp-.js.map → results-BI-zTqQy.js.map} +1 -1
- package/dist/{root-CFDjpWuc.js → root-Zeokqt3-.js} +2 -2
- package/dist/{root-CFDjpWuc.js.map → root-Zeokqt3-.js.map} +1 -1
- package/dist/{sandbox-BJPWuQE9.js → sandbox-Dd-4Bx6B.js} +3 -3
- package/dist/{sandbox-BJPWuQE9.js.map → sandbox-Dd-4Bx6B.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-BjbgFX28.js +2 -0
- package/dist/{server-5DWmKKfV.js → server-C7t8WYCR.js} +11 -9
- package/dist/{server-5DWmKKfV.js.map → server-C7t8WYCR.js.map} +1 -1
- package/dist/source-CLE8vMNa.js +2 -0
- package/dist/{source-Bsfa3LZH.js → source-fNUCWsCW.js} +2 -2
- package/dist/{source-Bsfa3LZH.js.map → source-fNUCWsCW.js.map} +1 -1
- package/dist/{src-DXx_29pq.js → src-B1OT_Zhf.js} +46 -46
- package/dist/{src-DXx_29pq.js.map → src-B1OT_Zhf.js.map} +1 -1
- package/dist/{src-BQyMiWri.js → src-BAiYD4eC.js} +48 -7
- package/dist/{src-BQyMiWri.js.map → src-BAiYD4eC.js.map} +1 -1
- package/dist/{src-D8KYzxBX.js → src-BmBFlEIe.js} +12 -7
- package/dist/{src-D8KYzxBX.js.map → src-BmBFlEIe.js.map} +1 -1
- package/dist/{src-N7E_mfcZ.js → src-C91RndLw.js} +1 -1
- package/dist/{src-C-LHyTSV.js → src-CEQFcAJH.js} +1 -1
- package/dist/src-CNp0srm9.js +3 -0
- package/dist/{src-DwwiVLqv.js → src-D3eW1e7Y.js} +19 -15
- package/dist/{src-DwwiVLqv.js.map → src-D3eW1e7Y.js.map} +1 -1
- package/dist/src-DJsBrqo6.js +4 -0
- package/dist/src-DTlmkpJe.js +3 -0
- package/dist/{src-Cmp-7YrD.js → src-DWk0e7ix.js} +5 -5
- package/dist/{src-Cmp-7YrD.js.map → src-DWk0e7ix.js.map} +1 -1
- package/dist/{src-AEwxd1UP.js → src-DfEAlC9s.js} +4 -4
- package/dist/{src-AEwxd1UP.js.map → src-DfEAlC9s.js.map} +1 -1
- package/dist/src-DgYURuJ6.js +3 -0
- package/dist/src-sSPmuaAc.js +3 -0
- package/dist/state-Bx-pH2ON.js +2 -0
- package/dist/{state-DZ7eqb1F.js → state-g6ij_40j.js} +14 -14
- package/dist/{state-DZ7eqb1F.js.map → state-g6ij_40j.js.map} +1 -1
- package/dist/studio-app/assets/index-B1Pf3LAT.css +2 -0
- package/dist/studio-app/assets/index-DciOL7BI.js +58 -0
- package/dist/studio-app/index.html +2 -2
- package/dist/summary-DjbxgCDi.js +192 -0
- package/dist/summary-DjbxgCDi.js.map +1 -0
- package/dist/{summary-D7Ttky0z.js → summary-gawvEpNu.js} +33 -5
- package/dist/summary-gawvEpNu.js.map +1 -0
- package/dist/tools-B7SoW5kU.js +249 -0
- package/dist/tools-B7SoW5kU.js.map +1 -0
- package/dist/{types-CYJldf5N.d.ts → types-C5AEPr29.d.ts} +135 -17
- package/dist/view/index.d.ts +63 -1
- package/dist/view/index.js +5 -5
- package/dist/{views-BrHOv6wg.js → views-CHXQfSWQ.js} +4 -4
- package/dist/{views-BrHOv6wg.js.map → views-CHXQfSWQ.js.map} +1 -1
- package/dist/views-DDei2LHa.js +3 -0
- package/dist/{worker-entry-DnYkANPG.js → worker-entry-B6nuDpBX.js} +3 -3
- package/dist/{worker-entry-DnYkANPG.js.map → worker-entry-B6nuDpBX.js.map} +1 -1
- package/dist/{workspace-uPapQj16.js → workspace-DUUfne8b.js} +4 -4
- package/dist/{workspace-uPapQj16.js.map → workspace-DUUfne8b.js.map} +1 -1
- package/dist/{workspace-BVgNhC9h.js → workspace-DXV77ZqW.js} +1 -1
- package/docker/Dockerfile +2 -0
- package/docker/reeval-path.sh +21 -0
- package/package.json +1 -1
- package/dist/assemble-Dm4gD1TU.js +0 -2
- package/dist/assemble-DtouO0iQ.js.map +0 -1
- package/dist/cli-CGuUdOPK.js.map +0 -1
- package/dist/cli-spec-DKAu09Gi.js.map +0 -1
- package/dist/command-CEm8PZ1N.js.map +0 -1
- package/dist/command-CLKuXn6Z.js.map +0 -1
- package/dist/db-Cjj2ZsfQ.js +0 -2
- package/dist/doctor-Bs1_pG9_.js.map +0 -1
- package/dist/egress-QJMzVtbw.js.map +0 -1
- package/dist/files--6CEUXm7.js +0 -2
- package/dist/files-Bd25OAQH.js +0 -2
- package/dist/files-bF2fPuT9.js.map +0 -1
- package/dist/init-DJuxvYCW.js +0 -2
- package/dist/model-v-0_Ik7S.js +0 -2
- package/dist/opencode-providers-DZuD_oYt.js.map +0 -1
- package/dist/options-k25fjaER.js.map +0 -1
- package/dist/plan-gwk6dJ8c.js.map +0 -1
- package/dist/provenance-Dfb8YY5Y.js.map +0 -1
- package/dist/query-DR4m6CXf.js +0 -2
- package/dist/record-DQ8wzyco.js.map +0 -1
- package/dist/server-buV3adNS.js +0 -2
- package/dist/source-BXzrmof5.js +0 -2
- package/dist/src-55j1zNHH.js +0 -3
- package/dist/src-BDwSuyIo.js +0 -3
- package/dist/src-DEgff9Z7.js +0 -4
- package/dist/src-W9g-y5X5.js +0 -3
- package/dist/src-daS0wKH2.js +0 -3
- package/dist/state-Cyw3QTsN.js +0 -2
- package/dist/studio-app/assets/index-Da1XDzx8.js +0 -58
- package/dist/studio-app/assets/index-xwzWdcE_.css +0 -2
- package/dist/summary-D7Ttky0z.js.map +0 -1
- package/dist/views-CwCWLWqo.js +0 -3
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as registerSecret, i as redactSecrets, t as REDACTED } from "./redact-C4fL_bZP.js";
|
|
2
|
-
import {
|
|
3
|
-
import { n as FLAGS, t as COMMANDS } from "./cli-spec-
|
|
2
|
+
import { a as readableCliInvocation } from "./record-CbuQH4l1.js";
|
|
3
|
+
import { n as FLAGS, t as COMMANDS } from "./cli-spec-BxLPgNN-.js";
|
|
4
4
|
import { existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { homedir } from "node:os";
|
|
6
6
|
import { join } from "node:path";
|
|
@@ -154,4 +154,4 @@ function logInvocation(root, entry) {
|
|
|
154
154
|
//#endregion
|
|
155
155
|
export { redactArgv as a, masks as i, readInvocations as n, registerEnvFileValues as o, commandOf as r, scrubber as s, logInvocation as t };
|
|
156
156
|
|
|
157
|
-
//# sourceMappingURL=invocations-
|
|
157
|
+
//# sourceMappingURL=invocations-DSMfs3Dx.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"invocations-
|
|
1
|
+
{"version":3,"file":"invocations-DSMfs3Dx.js","names":[],"sources":["../src/report/scrub.ts","../src/report/invocations.ts"],"mappings":";;;;;;;;AAcA,MAAM,YAAY,SAA6B;CAC7C,IAAI,CAAC,QAAQ,SAAS,OAAO,KAAK,SAAS,GAAG,OAAO,CAAC;CACtD,MAAM,sBAAM,IAAI,IAAI,CAAC,KAAK,QAAQ,WAAW,EAAE,CAAC,CAAC;CACjD,IAAI;EACF,IAAI,IAAI,aAAa,IAAI,CAAC;CAC5B,QAAQ,CAAC;CACT,KAAK,MAAM,KAAK,CAAC,GAAG,GAAG,GAAG,IAAI,EAAE,SAAS,IAAI,GAAG,IAAI,IAAI,EAAE,WAAW,MAAM,GAAG,CAAC;CAE/E,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM;AACpD;;;;;AAMA,SAAgB,UACd,MACA,MACA,OAAiC,aAAa,QAC1B;CACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,OAAO,KAAA;CAC3B,IAAI;CACJ,IAAI;EACF,OAAO,KAAK,IAAI;CAClB,QAAQ;EACN;CACF;CACA,MAAM,SAAS,MAAc,EAAE,YAAY,CAAC,CAAC,QAAQ,WAAW,EAAE;CAClE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,MAAM,IAAI,CAAC,GAAG,OAAO,KAAA;CACjD,MAAM,SAAS,KAAK,QAAQ,WAAW,EAAE,CAAC,CAAC,MAAM,KAAK,QAAQ,WAAW,EAAE,CAAC,CAAC,MAAM;CACnF,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,OAAO,YAAY,CAAC,GAAG,OAAO,KAAA;CACnE,MAAM,QAAQ,KAAK,QAAQ,WAAW,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,OAAO,MAAM;CACjE,OAAO,MAAM,KAAK,MAAM,MAAM,IAAI,IAAI,KAAA,IAAY;AACpD;AAEA,SAAgB,SAAS,SAIZ;CACX,MAAM,UAAU,SAAS,QAAQ,OAAO;CACxC,MAAM,QAAQ,UAAU,QAAQ,MAAM,QAAQ,IAAI,QAAQ,QAAQ,IAAI,GAAG;CACzE,MAAM,OAAO,CAAC,GAAG,SAAS,QAAQ,IAAI,GAAG,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,SAAS,EAAE,MAAM;CAC/F,MAAM,WAAW,MAAc,OAAiB,OAAe;EAC7D,IAAI,MAAM;EACV,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,WAAW,MAAM,EAAE;EACvD,OAAO;CACT;CACA,MAAM,QAAQ,UACZ,QAAQ,QAAQ,cAAc,OAAO,QAAQ,GAAG,GAAG,SAAS,WAAW,GAAG,MAAM,GAAG;CACrF,MAAM,SAAY,UAAgB;EAChC,IAAI,OAAO,UAAU,UAAU,OAAO,KAAK,KAAK;EAChD,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO,MAAM,IAAI,KAAK;EAChD,IAAI,SAAS,OAAO,UAAU,UAC5B,OAAO,OAAO,YAAY,OAAO,QAAQ,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;EAEtF,OAAO;CACT;CACA,OAAO;EAAE;EAAM;EAAO,OAAO,CAAC,KAAK,GAAI,QAAQ,SAAS,CAAC,WAAW,IAAI,CAAC,CAAE;CAAE;AAC/E;;AAGA,MAAa,SAAS,SAAiB,KAAK,MAAM,QAAQ,CAAC,CAAC,SAAS;;;;;AAMrE,SAAgB,sBAAsB,UAAwB;CAC5D,KAAK,MAAM,SAAS,OAAO,OAAO,SAAS,QAAQ,CAAC,GAClD,IAAI,SAAS,MAAM,UAAU,GAAG,eAAe,KAAK;AAExD;AAEA,MAAM,aAAa,WAChB,OAAO,QAAQ,KAAK,CAAC,CAAiD,MACpE,GAAG,OAAO,WAAW,KAAK,EAAE,UAAU,MACzC,CAAC,GAAG;;;;;;AAON,SAAgB,WAAW,MAA0B;CACnD,MAAM,MAAgB,CAAC;CACvB,IAAI,WAAgD;CACpD,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,IAAI,QAAQ,MAAM;GAChB,IAAI,KAAK,MAAM,GAAG,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,UAAU,OAAO,CAAC;GACtD;EACF;EACA,IAAI,QAAQ,KAAK,GAAG,GAAG;GACrB,IAAI,KAAK,GAAG;GACZ;EACF;EACA,IAAI,IAAI,WAAW,GAAG,KAAK,QAAQ,KAAK;GACtC,MAAM,KAAK,IAAI,QAAQ,GAAG;GAC1B,MAAM,OAAO,OAAO,KAAK,MAAM,IAAI,MAAM,GAAG,EAAE;GAC9C,MAAM,OAAO,KAAK,WAAW,IAAI,IAAI,KAAK,MAAM,CAAC,IAAI,UAAU,KAAK,MAAM,CAAC,CAAC;GAC5E,MAAM,OAAO,QAAQ,QAAQ,QAAQ,MAAM,QAAoB,KAAA;GAC/D,IAAI,CAAC,MAAM;IACT,IAAI,KAAK,QAAQ;IACjB;GACF;GACA,IAAI,OAAO,IAAI,IAAI,KAAK,GAAG,KAAK,SAAS;QACpC;IACH,IAAI,KAAK,IAAI;IACb,IAAI,KAAK,SAAS,YAAY,IAAI,IAAI,KAAK,QAAQ;KACjD,IAAI,KAAK,SAAS;KAClB;IACF;GACF;GACA;EACF;EACA,MAAM,UAAU,WAAW;EAC3B,IAAI,SAAS;GACX,IAAI,KAAK,GAAG;GACZ,WAAW,aAAa,WAAW,QAAQ,MAAM,KAAA;GACjD;EACF;EACA,WAAW,KAAA;EACX,IAAI,KAAK,OAAO;CAClB;CACA,OAAO;AACT;;AAGA,SAAgB,UAAU,MAA+B;CACvD,MAAM,QAAQ,WAAW,IAAI,CAAC,CAAC,QAAQ,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC,EAAE,WAAW,GAAG,CAAC;CACrF,OAAO,MAAM,SAAS,MAAM,KAAK,GAAG,IAAI;AAC1C;;;ACrIA,MAAa,mBAAmB,SAAiB,KAAK,MAAM,WAAW,WAAW;;AAGlF,SAAgB,gBAAgB,MAA+B;CAC7D,MAAM,OAAO,gBAAgB,IAAI;CACjC,IAAI,CAAC,WAAW,IAAI,GAAG,OAAO,CAAC;CAC/B,OAAO,aAAa,MAAM,MAAM,CAAC,CAC9B,MAAM,IAAI,CAAC,CACX,SAAS,SAAS;EACjB,IAAI,CAAC,KAAK,KAAK,GAAG,OAAO,CAAC;EAC1B,IAAI;GACF,OAAO,CAAC,sBAAsB,KAAK,MAAM,IAAI,CAAkB,CAAC;EAClE,QAAQ;GACN,OAAO,CAAC;EACV;CACF,CAAC;AACL;;;;;AAMA,SAAgB,cAAc,MAAc,OAA4B;CACtE,IAAI;EACF,IAAI,CAAC,WAAW,KAAK,MAAM,SAAS,CAAC,GAAG;EACxC,MAAM,QAAQ,SAAS;GAAE,KAAK,QAAQ;GAAK,MAAM,QAAQ;GAAG,SAAS;EAAK,CAAC;EAC3E,MAAM,QAAQ,CAAC,GAAG,gBAAgB,IAAI,GAAG,MAAM,MAAM,KAAK,CAAC,CAAC,CACzD,MAAM,GAAiB,CAAC,CACxB,KAAK,MAAM,KAAK,UAAU,CAAC,CAAC;EAC/B,cAAc,gBAAgB,IAAI,GAAG,GAAG,MAAM,KAAK,IAAI,EAAE,GAAG;CAC9D,QAAQ,CAAC;AACX"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { a as PACKAGE_NAMES, c as ownDir, n as COMPILED, u as ownRequire } from "./root-
|
|
1
|
+
import { a as PACKAGE_NAMES, c as ownDir, n as COMPILED, u as ownRequire } from "./root-Zeokqt3-.js";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { dirname, join } from "node:path";
|
|
4
4
|
import { createJiti } from "jiti";
|
|
5
5
|
//#region src/jiti.ts
|
|
6
6
|
let babel;
|
|
7
7
|
/**
|
|
8
|
-
* jiti's own Babel transform. jiti requires it lazily from beside its code, which a compiled
|
|
8
|
+
* jiti's own Babel transform. jiti requires it lazily from beside its code, which a compiled reeval
|
|
9
9
|
* doesn't have, so reeval requires the same file: from the jiti package on disk, or, compiled, from
|
|
10
10
|
* the copy embedded in it (`vendor/jiti/babel.cjs`), copied out to disk.
|
|
11
11
|
*/
|
|
@@ -17,7 +17,7 @@ const jitiTransform = (options) => {
|
|
|
17
17
|
return babel(options);
|
|
18
18
|
};
|
|
19
19
|
/**
|
|
20
|
-
* A jiti that imports from `id`. Compiled, a file's imports of reeval itself get this
|
|
20
|
+
* A jiti that imports from `id`. Compiled, a file's imports of reeval itself get this reeval's own
|
|
21
21
|
* modules: a copy on disk would register evals, and hold state, the executable can't see.
|
|
22
22
|
*/
|
|
23
23
|
async function esJiti(id, options = {}) {
|
|
@@ -43,4 +43,4 @@ async function ownModules() {
|
|
|
43
43
|
//#endregion
|
|
44
44
|
export { esJiti as t };
|
|
45
45
|
|
|
46
|
-
//# sourceMappingURL=jiti-
|
|
46
|
+
//# sourceMappingURL=jiti-DWsGEgxD.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jiti-
|
|
1
|
+
{"version":3,"file":"jiti-DWsGEgxD.js","names":[],"sources":["../src/jiti.ts"],"mappings":";;;;;AASA,IAAI;;;;;;AAOJ,MAAa,iBAA4B,YAAY;CACnD,IAAI,CAAC,OAAO;EACV,MAAM,OAAO,WACT,KAAK,OAAO,aAAa,GAAG,WAAW,IACvC,KAAK,QAAQ,WAAW,QAAQ,mBAAmB,CAAC,GAAG,gBAAgB;EAC3E,QAAQ,cAAc,IAAI,CAAC,CAAC,IAAI;CAClC;CACA,OAAO,MAAM,OAAO;AACtB;;;;;AAMA,eAAsB,OAAO,IAAY,UAAuB,CAAC,GAAkB;CACjF,MAAM,iBAAiB,WAAW,MAAM,WAAW,IAAI,KAAA;CACvD,OAAO,WAAW,IAAI;EACpB,WAAW;EACX,GAAG;EACH,GAAI,kBAAkB,EAAE,eAAe;CACzC,CAAC;AACH;AAEA,eAAe,aAA+C;CAC5D,MAAM,UAA4C;EAChD,IAAI,OAAO;EACX,YAAY,OAAO;EACnB,YAAY,OAAO;EACnB,UAAU,OAAO;EACjB,SAAS,OAAO;CAClB;CACA,MAAM,UAAmC,CAAC;CAC1C,KAAK,MAAM,CAAC,SAAS,WAAW,OAAO,QAAQ,OAAO,GACpD,KAAK,MAAM,QAAQ,eAAe,QAAQ,OAAO,WAAW,MAAM;CACpE,OAAO;AACT"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { F as FakeNoteInput, J as JudgeModel, M as FakeAnalysisModel, P as FakeJudgeModel, R as FakeVote, Y as JudgeOptions, a as BlindPair, g as ConfigJudgeOptions, gt as Verdict, n as AnalysisConfig, q as JudgeCallOptions } from "./types-C5AEPr29.js";
|
|
2
2
|
//#region src/judge/index.d.ts
|
|
3
3
|
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
4
4
|
export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.13.0";
|
|
@@ -11,11 +11,12 @@ export declare const fakeJudge: ({ score }?: {
|
|
|
11
11
|
}) => FakeJudgeModel;
|
|
12
12
|
/**
|
|
13
13
|
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
14
|
-
* model's structured answer,
|
|
14
|
+
* model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
15
|
+
* `'tie'`, a function of the blinded pair, or `{ error }`), through the same parsing and
|
|
16
|
+
* validation as a real one. With `analysis.agent`, each note calls the same tools a real model
|
|
17
|
+
* would, then answers with `note` (or `note(input)`), or throws `note.error`.
|
|
15
18
|
*/
|
|
16
|
-
export declare const fakeAnalysis: ({ output }:
|
|
17
|
-
output: FakeAnalysisModel['output'];
|
|
18
|
-
}) => FakeAnalysisModel;
|
|
19
|
+
export declare const fakeAnalysis: ({ output, note, vote }: Pick<FakeAnalysisModel, 'output' | 'note' | 'vote'>) => FakeAnalysisModel;
|
|
19
20
|
//#endregion
|
|
20
|
-
export type { AnalysisConfig, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, JudgeCallOptions, JudgeModel, JudgeOptions, Verdict };
|
|
21
|
+
export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeJudgeModel, FakeNoteInput, FakeVote, JudgeCallOptions, JudgeModel, JudgeOptions, Verdict };
|
|
21
22
|
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.js
CHANGED
|
@@ -7,11 +7,16 @@ const fakeJudge = ({ score = 1 } = {}) => ({
|
|
|
7
7
|
});
|
|
8
8
|
/**
|
|
9
9
|
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
10
|
-
* model's structured answer,
|
|
10
|
+
* model's structured answer, and each compare question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
11
|
+
* `'tie'`, a function of the blinded pair, or `{ error }`), through the same parsing and
|
|
12
|
+
* validation as a real one. With `analysis.agent`, each note calls the same tools a real model
|
|
13
|
+
* would, then answers with `note` (or `note(input)`), or throws `note.error`.
|
|
11
14
|
*/
|
|
12
|
-
const fakeAnalysis = ({ output }) => ({
|
|
15
|
+
const fakeAnalysis = ({ output, note, vote }) => ({
|
|
13
16
|
kind: "fake-analysis",
|
|
14
|
-
output
|
|
17
|
+
...output !== void 0 ? { output } : {},
|
|
18
|
+
...note !== void 0 ? { note } : {},
|
|
19
|
+
...vote !== void 0 ? { vote } : {}
|
|
15
20
|
});
|
|
16
21
|
//#endregion
|
|
17
22
|
export { DEFAULT_THRESHOLD, RECOMMENDED_JUDGE_MODEL, fakeAnalysis, fakeJudge };
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;AAKA,MAAa,aAAa,EAAE,QAAQ,MAA0B,CAAC,OAAuB;CACpF,MAAM;CACN;AACF
|
|
1
|
+
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;AAKA,MAAa,aAAa,EAAE,QAAQ,MAA0B,CAAC,OAAuB;CACpF,MAAM;CACN;AACF;;;;;;;;AASA,MAAa,gBAAgB,EAC3B,QACA,MACA,YAC6E;CAC7E,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;AACvC"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { n as fillDefaults } from "./record-
|
|
1
|
+
import { n as fillDefaults } from "./record-CbuQH4l1.js";
|
|
2
2
|
//#region src/view/live.ts
|
|
3
3
|
/** The progress file's name inside the run's staging directory. */
|
|
4
4
|
const LIVE_FILE = "live.json";
|
|
@@ -95,4 +95,4 @@ function readableLiveProgress(value) {
|
|
|
95
95
|
//#endregion
|
|
96
96
|
export { liveApiPath as a, readableLiveProgress as c, isFinished as i, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
|
|
97
97
|
|
|
98
|
-
//# sourceMappingURL=live-
|
|
98
|
+
//# sourceMappingURL=live-DZkCtmNP.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"live-
|
|
1
|
+
{"version":3,"file":"live-DZkCtmNP.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;AAOA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAsDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAA+B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEnG,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC;EAAC;EAAK;EAAQ;EAAM;EAAM;CAAI,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CACnE,IAAI,CAAC,SAAS,SAAS,MAAoB,GAAG,OAAO;CACrD,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACA;EACG;EACF;EACA;EACA;EACN;EACQ;EACR,WAAY,aAAoC;EAChD,YAAa,cAAqC;CACpD;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC;EAAC;EAAO;EAAW;EAAM;CAAW,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3E,IAAI,CAAC,OAAO,SAAS,KAAkB,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAC7E,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,KAAK,MAAM,MAAM,MAAM,IAAI,GAAG,OAAO;CACzC,MAAM,OAAO;CACb,OAAO;EACL,eAAA;EACO;EACI;EACX;EACM;EACO;EACN;EACP,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;CACZ;AACF"}
|
|
@@ -309,7 +309,7 @@ const isTurn = (value) => typeof value === "object" && value !== null && Array.i
|
|
|
309
309
|
const matches = (value, pattern) => pattern === void 0 || (typeof pattern === "string" ? value.includes(pattern) : pattern.test(value));
|
|
310
310
|
/** Passes when the turn has a `type` event whose `field` matches; refuses when it can't know. */
|
|
311
311
|
function webEvent(matcher, actual, type, field, pattern) {
|
|
312
|
-
if (!isTurn(actual)) throw new TypeError(`${matcher}() expects a turn from
|
|
312
|
+
if (!isTurn(actual)) throw new TypeError(`${matcher}() expects a turn from c.send()`);
|
|
313
313
|
const what = type === "web-search" ? "web searches" : "web fetches";
|
|
314
314
|
if (actual.unreported.includes(type)) throw new HarnessCapabilityError(`${matcher}() can't check this turn: its harness doesn't report ${what} ('${type}' is in turn.unreported), so the agent may have made them with no event. Check the reply or files instead, or use a harness that reports them.`);
|
|
315
315
|
const seen = actual.transcript.filter((e) => e.type === type).map(field);
|
|
@@ -328,7 +328,7 @@ const tail = (s, lines = 20, chars = 2e3) => {
|
|
|
328
328
|
const hasFiles = (value) => typeof value === "object" && value !== null && typeof value.readFile === "function";
|
|
329
329
|
const matchers = {
|
|
330
330
|
async toHaveFile(actual, path, pattern) {
|
|
331
|
-
if (!hasFiles(actual)) throw new TypeError("toHaveFile() expects a turn from
|
|
331
|
+
if (!hasFiles(actual)) throw new TypeError("toHaveFile() expects a turn from c.send(), or c");
|
|
332
332
|
const content = await actual.readFile(path);
|
|
333
333
|
if (content === null) return {
|
|
334
334
|
pass: false,
|
|
@@ -360,7 +360,7 @@ const matchers = {
|
|
|
360
360
|
},
|
|
361
361
|
toPass(actual) {
|
|
362
362
|
const v = actual;
|
|
363
|
-
if (typeof v?.score !== "number") throw new TypeError("toPass() expects a
|
|
363
|
+
if (typeof v?.score !== "number") throw new TypeError("toPass() expects a c.judge() verdict");
|
|
364
364
|
const score = v.score.toFixed(2);
|
|
365
365
|
return {
|
|
366
366
|
pass: v.verdict === "pass",
|
|
@@ -371,4 +371,4 @@ const matchers = {
|
|
|
371
371
|
//#endregion
|
|
372
372
|
export { withLease as a, registerGroup as c, attemptExpectError as d, createExpect as f, withAttemptExpect as g, unawaitedError as h, useSchedulerChannel as i, registerTest as l, extendExpect as m, matchers as n, collectingFile as o, expect as p, acquire as r, getRunContext as s, HarnessCapabilityError as t, useEngine as u };
|
|
373
373
|
|
|
374
|
-
//# sourceMappingURL=matchers-
|
|
374
|
+
//# sourceMappingURL=matchers-D51lM8dO.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers-
|
|
1
|
+
{"version":3,"file":"matchers-D51lM8dO.js","names":[],"sources":["../src/engine/expect.ts","../src/engine/runtime.ts","../src/scheduler/client.ts","../src/matchers.ts"],"mappings":";;;;;AAkDA,KAAK,IAAI,UAAwB;AACjC,KAAK,IAAI,cAA4B;AACrC,KAAK,IAAI,sBAAoC;;AAkB7C,SAAgB,aAAa,MAAiC;CAC5D,MAAM,WAAW,OAAgB,YAAqB;EACpD,MAAM,EAAE,mBAAmB,SAAS,MAAM;EAC1C,SAAS,EAAE,gBAAgB,iBAAiB,EAAE,GAAG,MAAM;EACvD,MAAM,YAAY,KAAK,OAAO,OAAO,OAAO;EAC5C,OAAO,OAAO,UAAU,SAAS,IAAI,IAAI;CAC3C;CACA,OAAO,OAAO,QAAQ,KAAK,MAAM;CACjC,OAAO,OAAO,QAAS,WAAsC,2BAA2B;CACxF,OAAO,iBAAiB,SAAuB,MAAM;CACrD,OAAO,YAAY,UAAU,SAAS,OAAO,MAAM;CACnD,SACE;EACE,gBAAgB;EAChB,uBAAuB;EACvB,4BAA4B;EAC5B,0BAA0B;EAC1B,kCAAkC;EAClC,iBAAiB,MAAM,gBAAgB;CACzC,GACA,MACF;CACA,OAAO,SAAS,KAAK;CACrB,OAAO,UAAU,aACd,KAAK,OAAwE,OAC5E,QACA,QACF;CACF,OAAO,sBAAsB,YAAY,yBAAyB,OAAO;CACzE,OAAO,QAAQ,OAAO,YACnB,OAAO,OAAO,OAAO,CAAC,CAAkD,YAAY,EACnF,MAAM,KACR,CAAC;CACH,OAAO,OAAO,WAAW,MAAM;CAC/B,OAAO,eAAe,YAAY;EAChC,KAAK,OAAO,KAAK,WAAW,UAAU,KAAK,QAAQ,MAAM,IAAI,kBAAkB;CACjF;CACA,SAAS,WAAW,UAAkB;EACpC,MAAM,iCACJ,IAAI,MACF,uCAAuC,SAAS,YAAY,OAAO,SAAS,CAAC,CAAC,gBAChF;EACF,SACE;GAAE,0BAA0B;GAAU,kCAAkC;EAAS,GACjF,MACF;CACF;CACA,SAAS,gBAAgB;EAEvB,SACE;GAAE,uBAAuB;GAAM,4CAA4B,IAF3C,MAAM,gDAEyC;EAAE,GACjE,MACF;CACF;CACA,KAAK,KAAK,UAAU,QAAQ,cAAc,UAAU;CACpD,KAAK,KAAK,UAAU,QAAQ,iBAAiB,aAAa;CAC1D,OAAO,OAAO,cAAc;CAC5B,OAAO;AACT;;;;;AAMA,SAAS,WAAW,QAA4C;CAC9D,QAAQ,IAAI,UAAU,CAAC,MAAM;EAC3B,MAAM,EAAE,WAAW,IAAI,UAAU,KAAM,YAAY;EACnD,MAAM,YAAY,OAAO,MAAM,OAAO;EACtC,KAAK,KAAK,KAAK,WAAW,QAAQ,IAAI;EACtC,MAAM,QAAgB,IAAI,MAAM,WAAW,EACzC,IAAI,QAAQ,KAAK,UAAU;GACzB,MAAM,UAAU,QAAQ,IAAI,QAAQ,KAAK,QAAQ;GACjD,IAAI,OAAO,YAAY,YACrB,OAAO,mBAAmB,KAAK,YAAY,QAAQ;GACrD,QAAQ,GAAG,SAAoB;IAC7B,MAAM,2BAAW,IAAI,MAAM,aAAa;IACxC,QAAQ,YAAY;KAClB,MAAM,WAAW,KAAK,IAAI,IAAI;KAC9B,MAAM,SAAS,YAAY,QAAQ,OAAO;KAC1C,SACE,IAAI;MACF,KAAK,KAAK,KAAK,WAAW,UAAU,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC;MACxD,OAAO,MAAM,QAAQ,MAAM,SAAS,WAAW,IAAI;KACrD,SAAS,OAAO;MACd,IAAI,KAAK,IAAI,IAAI,YAAY,UAAU;OACrC,IAAI,iBAAiB,SAAS,SAAS,OACrC,MAAM,QAAQ,SAAS,MAAM,QAAQ,SAAS,SAAS,MAAM,OAAO;OACtE,MAAM;MACR;MACA,MAAM,IAAI,SAAS,YAAY,WAAW,SAAS,QAAQ,CAAC;KAC9D;IAEJ,EAAA,CAAG;GACL;EACF,EACF,CAAC;EACD,OAAO;CACT;AACF;;AAGA,SAAgB,mBAAmB,QAAsB,MAAqC;CAC5F,MAAM,CAAC,QAAQ,KAAK,QAAQ,UAAU,CAAC;CAEvC,IAAI,MAAM,OAAO,OAAO,uBAAO,IAAI,MAAM,GAAG,IAAI;CAChD,MAAM,QAAQ,OAAO,SAAS;CAC9B,IACE,MAAM,6BAA6B,QACnC,MAAM,6BAA6B,MAAM,gBAEzC,OAAQ,MAAM,iCAAiD;CACjE,IAAI,MAAM,yBAAyB,MAAM,mBAAmB,GAC1D,OAAO,MAAM;AAEjB;AAEA,MAAM,eAAe,aAAa;AAClC,OAAO,eAAe,YAAY,eAAe;CAC/C,OAAO;CACP,UAAU;CACV,cAAc;AAChB,CAAC;AAED,MAAM,gBAAgB,IAAI,kBAAgC;AAI1D,MAAM,iBAAiB,IAAI,kBAA6B;AAExD,MAAM,cAAc,UAClB,OAAO,UAAU,YACjB,UAAU,QACV,OAAQ,MAA2B,SAAS;;;;;AAM9C,SAAS,QAA0B,WAAc,SAAoB,MAAiB;CACpF,OAAO,IAAI,MAAM,WAAW,EAC1B,IAAI,QAAQ,UAAU;EACpB,MAAM,QAAQ,QAAQ,IAAI,QAAQ,QAAQ;EAC1C,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,QAAQ;EACvC,IAAI,OAAO,UAAU,YACnB,QAAQ,GAAG,SAAoB;GAC7B,MAAM,SAAS,QAAQ,MAAM,OAAO,QAAQ,IAAI;GAChD,IAAI,CAAC,WAAW,MAAM,GAAG,OAAO;GAChC,MAAM,QAAiB;IACrB,WAAW,GAAG,KAAK;IACnB,SAAS,QAAQ,QAAQ,MAAM,CAAC,CAAC,WACzB,KAAA,IACL,UAAmB,yBAAS,IAAI,MAAM,GAAG,KAAK,UAAU,CAC3D;IACA,UAAU;GACZ;GACA,QAAQ,KAAK,KAAK;GAClB,MAAM,WAAc,OAAmC;IACrD,MAAM,WAAW;IACjB,OAAO,GAAG,QAAQ,QAAQ,MAAM,CAAC;GACnC;GACA,OAAO;IAEL,OAAO,aAAqB,eAC1B,SAAS,MAAM,EAAE,KAAK,aAAa,UAAU,CAAC;IAChD,QAAQ,eAAuB,SAAS,MAAM,EAAE,MAAM,UAAU,CAAC;IACjE,UAAU,cAAsB,SAAS,MAAM,EAAE,QAAQ,SAAS,CAAC;KAClE,OAAO,cAAc;GACxB;EACF;EAEF,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,WAAW,KAAK,IACnE,QAAQ,OAAO,SAAS,IAAI,IAC5B;CACN,EACF,CAAC;AACH;AAEA,MAAM,SAAS,WAAoB,SAAiB;CAClD,MAAM,UAAU,eAAe,SAAS;CACxC,OAAO,WAAW,OAAO,cAAc,YAAY,cAAc,OAC7D,QAAQ,WAAW,SAAS,IAAI,IAChC;AACN;;AAGA,MAAM,SAAiC;CAAE,MAAM;CAAkB,MAAM;AAAiB;AAExF,MAAa,SAAuB,IAAI,MAAM,cAAc;CAC1D,QAAQ,QAAQ,SAAS,SACvB,MAAM,QAAQ,MAAM,cAAc,SAAS,KAAK,QAAQ,SAAS,IAAI,GAAG,WAAW;CACrF,MAAM,QAAQ,aAAa;EACzB,MAAM,QAAQ,cAAc,SAAS,KAAK;EAC1C,MAAM,QAAQ,QAAQ,IAAI,OAAO,QAAQ;EAEzC,IAAI,OAAO,UAAU,cAAc,aAAa,UAAU,OAAO;EACjE,MAAM,QAAQ,OAAO;EAErB,QAAQ,GAAG,SAAoB;GAC7B,MAAM,SAAS,QAAQ,MAAM,OAAO,OAAO,IAAI;GAC/C,OAAO,QAAQ,MAAM,QAAQ,KAAK,IAAI;EACxC;CACF;AACF,CAAC;;AAGD,SAAgB,kBACd,KACA,SACA,MACY;CACZ,OAAO,cAAc,IAAI,WAAW,eAAe,IAAI,SAAS,IAAI,CAAC;AACvE;;;;;AAMA,eAAsB,eAAe,SAAgD;CACnF,MAAM,SAAS,QAAQ,QAAQ,MAAM,CAAC,EAAE,QAAQ;CAChD,IAAI,OAAO,WAAW,GAAG,OAAO,KAAA;CAChC,KAAK,MAAM,KAAK,QAAQ,EAAE,WAAW;CACrC,MAAM,WAAW,MAAM,QAAQ,IAAI,OAAO,KAAK,MAAM,EAAE,OAAO,CAAC;CAC/D,MAAM,IAAI,KAAK,IACb,GACA,SAAS,WAAW,MAAM,MAAM,KAAA,CAAS,CAC3C;CACA,MAAM,OAAO,GAAG,OAAO,EAAE,EAAE,UAAU,qDAAqD,OAAO,EAAE,EAAE,UAAU;CAC/G,MAAM,UAAU,SAAS;CACzB,IAAI,mBAAmB,OAAO;EAC5B,QAAQ,UAAU,GAAG,QAAQ,QAAQ,MAAM;EAC3C,OAAO;CACT;CACA,MAAM,QAAQ,IAAI,MAAM,IAAI;CAC5B,MAAM,OAAO;CACb,OAAO;AACT;AAQA,SAAgB,aAAa,UAA0B;CACrD,aAAa,OAAO,QAAiD;AACvE;;;ACvQA,IAAI;;AAGJ,SAAgB,UAAU,MAAoB;CAC5C,SAAS;AACX;AAEA,SAAS,SAAiB;CACxB,IAAI,CAAC,QACH,MAAM,IAAI,WACR,iGACF;CACF,OAAO;AACT;AAEA,MAAa,gBAAwC,MAAM,SAAS,SAClE,OAAO,CAAC,CAAC,aAAa,MAAM,SAAS,IAAI;AAE3C,MAAa,iBAA0C,MAAM,MAAM,SACjE,OAAO,CAAC,CAAC,cAAc,MAAM,MAAM,IAAI;AAEzC,MAAa,sBAAkC,OAAO,CAAC,CAAC,cAAc;AAEtE,MAAa,uBAA2C,OAAO,CAAC,CAAC,eAAe;;;ACzDhF,IAAM,iBAAN,MAA0C;CACxC,2BAAoB,IAAI,IAAqB;CAC7C,aAAsB,IAAI,WAAW,MAAM;EACzC,KAAK,SAAS,IAAI,EAAE,EAAE,CAAC,EAAE,QAAQ;EACjC,KAAK,SAAS,OAAO,EAAE,EAAE;CAC3B,CAAC;CAED,QAAQ,IAAY,QAAgB,MAAwB,SAAkB;EAC5E,KAAK,SAAS,IAAI,IAAI,OAAO;EAC7B,KAAK,WAAW,QAAQ;GAAE;GAAI,OAAO;GAAS;GAAQ,GAAI,OAAO,EAAE,KAAK,IAAI,CAAC;EAAG,CAAC;CACnF;CAEA,QAAQ,IAAY;EAClB,KAAK,SAAS,OAAO,EAAE;EACvB,KAAK,WAAW,QAAQ,EAAE;CAC5B;AACF;;AAGA,IAAM,mBAAN,MAA4C;CAC1C,2BAAoB,IAAI,IAAqB;CAC7C;CAEA,YAAY,MAAwC;EAClD,KAAK,QAAQ;CACf;CAEA,MAAM,IAAY;EAChB,MAAM,UAAU,KAAK,SAAS,IAAI,EAAE;EACpC,KAAK,SAAS,OAAO,EAAE;EACvB,SAAS,QAAQ;CACnB;CAEA,QAAQ,IAAY,QAAgB,MAAwB,SAAkB;EAC5E,KAAK,SAAS,IAAI,IAAI,OAAO;EAC7B,KAAK,MAAM;GAAE,GAAG;GAAW;GAAI;GAAQ,GAAI,OAAO,EAAE,KAAK,IAAI,CAAC;EAAG,CAAC;CACpE;CAEA,QAAQ,IAAY;EAClB,KAAK,SAAS,OAAO,EAAE;EACvB,KAAK,MAAM;GAAE,GAAG;GAAW;EAAG,CAAC;CACjC;AACF;AAEA,IAAI;AACJ,IAAI,MAAM;;AAGV,SAAgB,oBAAoB,MAA8D;CAChG,MAAM,UAAU,IAAI,iBAAiB,IAAI;CACzC,YAAY;CACZ,QAAQ,OAAO,QAAQ,MAAM,EAAE;AACjC;;;;;AAMA,SAAgB,QACd,QACA,UAAiD,CAAC,GAClC;CAChB,MAAM,EAAE,MAAM,WAAW;CACzB,IAAI,QAAQ,SAAS,OAAO,QAAQ,OAAO,OAAO,MAAM;CACxD,cAAc,IAAI,eAAe;CACjC,MAAM,MAAM;CACZ,MAAM,KAAK,OAAO,EAAE,GAAG;CACvB,IAAI,WAAW;CACf,MAAM,QAAe,EACnB,UAAU;EACR,IAAI,UAAU;EACd,WAAW;EACX,IAAI,QAAQ,EAAE;CAChB,EACF;CACA,OAAO,IAAI,SAAgB,SAAS,WAAW;EAC7C,MAAM,gBAAgB;GACpB,MAAM,QAAQ;GACd,OAAO,QAAQ,MAAM;EACvB;EACA,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;EACzD,IAAI,QAAQ,IAAI,QAAQ,MAAM;GAC5B,eAAe;IACb,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,QAAQ,KAAK;GACf;GACA,SAAS,UAAU;IACjB,QAAQ,oBAAoB,SAAS,OAAO;IAC5C,OAAO,KAAK;GACd;EACF,CAAC;CACH,CAAC;AACH;;AAGA,eAAsB,UACpB,QACA,IACA,QACY;CACZ,MAAM,QAAQ,MAAM,QAAQ,QAAQ,SAAS,EAAE,OAAO,IAAI,CAAC,CAAC;CAC5D,IAAI;EACF,OAAO,MAAM,GAAG;CAClB,UAAU;EACR,MAAM,QAAQ;CAChB;AACF;;;;ACvHA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAgB;AAClB;AAEA,MAAM,UAAU,UACd,OAAO,UAAU,YACjB,UAAU,QACV,MAAM,QAAS,MAAe,UAAU,KACxC,MAAM,QAAS,MAAe,UAAU;AAE1C,MAAM,WAAW,OAAe,YAC9B,YAAY,KAAA,MACX,OAAO,YAAY,WAAW,MAAM,SAAS,OAAO,IAAI,QAAQ,KAAK,KAAK;;AAG7E,SAAS,SACP,SACA,QACA,MACA,OACA,SACQ;CACR,IAAI,CAAC,OAAO,MAAM,GAAG,MAAM,IAAI,UAAU,GAAG,QAAQ,gCAAgC;CACpF,MAAM,OAAO,SAAS,eAAe,iBAAiB;CACtD,IAAI,OAAO,WAAW,SAAS,IAAI,GACjC,MAAM,IAAI,uBACR,GAAG,QAAQ,uDAAuD,KAAK,KAAK,KAAK,+IACnF;CAEF,MAAM,OAAO,OAAO,WAAW,QAAQ,MAAM,EAAE,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK;CACvE,MAAM,OAAO,KAAK,MAAM,MAAM,QAAQ,GAAG,OAAO,CAAC;CACjD,MAAM,SAAS,YAAY,KAAA,IAAY,KAAK,aAAa;CACzD,OAAO;EACL;EACA,eACE,OACI,eAAe,OAAO,OAAO,QAAQ,KAAK,UAAU,IAAI,MACxD,YAAY,OAAO,OAAO,QAAQ,KAAK,SAAS,KAAK,UAAU,IAAI,IAAI;CAC/E;AACF;;AAGA,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,OAAO,UAAU,YAAY,UAAU,QAAQ,OAAQ,MAAmB,aAAa;AAEzF,MAAa,WAAW;CACtB,MAAM,WAAW,QAAiB,MAAc,SAA4C;EAC1F,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,UAAU,iDAAiD;EAC5F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF;CAEA,MAAM,UAAU,QAAkC;EAChD,MAAM,IAAK,MAAM;EACjB,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,IAAI,UAAU,mCAAmC;EAC5F,OAAO;GACL,MAAM,EAAE,aAAa;GACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;EAC3E;CACF;CAEA,eAAe,QAAiB,OAAiC;EAC/D,OAAO,SACL,kBACA,QACA,eACC,MAAO,WAAW,IAAI,EAAE,QAAQ,IACjC,KACF;CACF;CAEA,cAAc,QAAiB,KAA+B;EAC5D,OAAO,SAAS,iBAAiB,QAAQ,cAAc,MAAO,SAAS,IAAI,EAAE,MAAM,IAAK,GAAG;CAC7F;CAEA,OAAO,QAAyB;EAC9B,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,IAAI,UAAU,sCAAsC;EAC5F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF;AACF"}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
import { a as providerModel, c as resolveAnalysisModel, i as isFake, n as callAnalysisModel, o as questionModel, r as cannedModel, s as questionModelLabel, t as analysisModelName } from "./model-GFM8iOzM.js";
|
|
2
|
+
export { analysisModelName, callAnalysisModel, cannedModel, isFake, providerModel, questionModel, questionModelLabel, resolveAnalysisModel };
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import { t as UsageError } from "./errors-CSxfJ4qu.js";
|
|
2
|
+
import { a as isFakeJudge, c as judgeRoute } from "./judge-D8Uf4dA9.js";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { Output, generateText } from "ai";
|
|
5
|
+
//#region src/analysis/schema.ts
|
|
6
|
+
const confidence = z.enum([
|
|
7
|
+
"high",
|
|
8
|
+
"medium",
|
|
9
|
+
"low"
|
|
10
|
+
]);
|
|
11
|
+
const cites = z.array(z.string()).describe("Ids from the digest: F<n>, E<n>, E<n>:L<line>, an attempt id, or <attempt id>:check<n>");
|
|
12
|
+
const claim = z.object({
|
|
13
|
+
text: z.string(),
|
|
14
|
+
cites,
|
|
15
|
+
confidence,
|
|
16
|
+
quotes: z.array(z.string()).optional().describe("Exact spans copied from the cited evidence, if the claim quotes any")
|
|
17
|
+
});
|
|
18
|
+
const analysisSchema = z.object({
|
|
19
|
+
tldr: z.string().describe("2-3 sentences: what happened, and does the thing under test help"),
|
|
20
|
+
whatHappened: z.array(claim),
|
|
21
|
+
results: z.array(claim).describe("Pass rate and effort, each with its noise verdict"),
|
|
22
|
+
why: z.array(claim).describe("Causes: the thing under test, the eval itself, or reeval/infrastructure"),
|
|
23
|
+
anomalies: z.array(z.object({
|
|
24
|
+
text: z.string(),
|
|
25
|
+
kind: z.enum([
|
|
26
|
+
"es-bug",
|
|
27
|
+
"eval-design",
|
|
28
|
+
"judge",
|
|
29
|
+
"noise",
|
|
30
|
+
"other"
|
|
31
|
+
]),
|
|
32
|
+
cites
|
|
33
|
+
})),
|
|
34
|
+
actions: z.array(z.object({
|
|
35
|
+
title: z.string(),
|
|
36
|
+
target: z.string().describe("The file, skill, eval or reeval setting to change"),
|
|
37
|
+
kind: z.enum([
|
|
38
|
+
"skill",
|
|
39
|
+
"docs",
|
|
40
|
+
"eval",
|
|
41
|
+
"judge",
|
|
42
|
+
"es",
|
|
43
|
+
"harness"
|
|
44
|
+
]),
|
|
45
|
+
change: z.string(),
|
|
46
|
+
expectedEffect: z.string().describe("What should measurably change"),
|
|
47
|
+
verify: z.string().describe("One reeval command from the CLI list that would show the effect"),
|
|
48
|
+
impact: z.enum([
|
|
49
|
+
"high",
|
|
50
|
+
"medium",
|
|
51
|
+
"low"
|
|
52
|
+
]),
|
|
53
|
+
effort: z.enum([
|
|
54
|
+
"low",
|
|
55
|
+
"medium",
|
|
56
|
+
"high"
|
|
57
|
+
]),
|
|
58
|
+
cites
|
|
59
|
+
})).max(5),
|
|
60
|
+
confidence: z.object({
|
|
61
|
+
overall: confidence,
|
|
62
|
+
why: z.string()
|
|
63
|
+
})
|
|
64
|
+
});
|
|
65
|
+
const SYSTEM_PROMPT = `You analyse the results of a reeval run for the engineer who wrote the evals.
|
|
66
|
+
reeval runs coding agents in sandboxes. A compare eval runs a control cell and variant cells (with a skill, docs or MCP, say) and reports pass-rate lift with a 95% interval, and effort (tool calls, agent time) next to it.
|
|
67
|
+
|
|
68
|
+
You get a DIGEST: facts computed deterministically by reeval (ids F1...), evidence excerpts (ids E1...; transcript lines start with their line number L<n>), an attempts table (ids like r2/fix-config/control#1), the RULES reeval used for every verdict, and the CLI.
|
|
69
|
+
|
|
70
|
+
Rules:
|
|
71
|
+
- Every claim cites at least one id from the digest: F<n>, E<n>, E<n>:L<line> for a line the evidence shows, an attempt id, or <attempt id>:check<n>. Never cite anything that isn't in the digest.
|
|
72
|
+
- Take numbers from the facts. Don't compute new statistics beyond simple ratios of cited numbers.
|
|
73
|
+
- The verdicts in the facts are final: if a lift or effort verdict says within noise or insufficient, say so; never call it an effect. Name each factor claim's tier (controlled, pooled, observed).
|
|
74
|
+
- Say what can't be attributed (factors that are not_measured).
|
|
75
|
+
- Separate causes: the thing under test (skill, docs), the eval itself (task, checks, judge question), and reeval or infrastructure (bugs, fingerprints, harness).
|
|
76
|
+
- Explain a failed judge verdict by comparing the cited output with the judge question, and say it's your reading: judges record no reason.
|
|
77
|
+
- Actions: at most 5, each with a concrete target, the change, the expected measurable effect, and a verify command that is exactly one of the reeval commands in the CLI list, with only the flags listed for it. "No actions" is a valid answer when nothing is worth doing. No generic advice ("add more tests") without a concrete target.
|
|
78
|
+
- Quotes, if any, must be copied exactly from the cited evidence.
|
|
79
|
+
- Plain, direct English. No hype.`;
|
|
80
|
+
//#endregion
|
|
81
|
+
//#region src/analysis/model.ts
|
|
82
|
+
const isFake = (m) => typeof m === "object" && m !== null && m.kind === "fake-analysis";
|
|
83
|
+
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
84
|
+
/** `analysis.model`, else the judge's when it's a language model; undefined when neither is. */
|
|
85
|
+
function defaultModel(config) {
|
|
86
|
+
const model = config.analysis.model;
|
|
87
|
+
if (model !== void 0) return model;
|
|
88
|
+
const judge = config.judge.model;
|
|
89
|
+
if (typeof judge === "string" && !isEvaluationModel(judge)) return judge;
|
|
90
|
+
}
|
|
91
|
+
/** ` (the judge, typesafe-ai/jev-1.13.0, is an evaluation model: it scores yes/no questions and can't <task>)`, when it is one. */
|
|
92
|
+
function judgeAside(config, task) {
|
|
93
|
+
const judge = config.judge.model;
|
|
94
|
+
if (typeof judge !== "string" && !isFakeJudge(judge)) return "";
|
|
95
|
+
return ` (the judge, ${isFakeJudge(judge) ? "a fake judge" : judge}, is an evaluation model: it scores yes/no questions and can't ${task})`;
|
|
96
|
+
}
|
|
97
|
+
/** Which model analyses: `analysis.model`, else the judge's when it's a language model. */
|
|
98
|
+
function analysisModelName(config, use = "analyze") {
|
|
99
|
+
const model = defaultModel(config);
|
|
100
|
+
if (model !== void 0) return model;
|
|
101
|
+
throw new UsageError(`${use === "agent" ? "config.analysis.agent" : "reeval analyze"} needs a language model. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(config, "write an analysis")}${use === "agent" ? ", or turn agent off" : ", or run reeval analyze --context to hand the digest to your own coding agent"}.`);
|
|
102
|
+
}
|
|
103
|
+
/** The plan's name for the model a compare question without its own is answered by; null when none. */
|
|
104
|
+
function questionModelLabel(config) {
|
|
105
|
+
const model = defaultModel(config);
|
|
106
|
+
if (model === void 0) return null;
|
|
107
|
+
return isFake(model) ? "fake" : model;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* The model that answers a compare question: its own, else the analysis model. Throws a
|
|
111
|
+
* UsageError (exit 2) when that's missing or an evaluation model.
|
|
112
|
+
*/
|
|
113
|
+
function questionModel(config, question, where) {
|
|
114
|
+
if (question.model !== void 0) return question.model;
|
|
115
|
+
const model = defaultModel(config);
|
|
116
|
+
const fix = `give the question its own, { question: '…', model: 'google/gemini-3.8-flash' }`;
|
|
117
|
+
if (model === void 0) throw new UsageError(`compare.question "${question.question}" (${where}) needs a language model to answer it. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(config, "pick between two attempts")}, or ${fix}.`);
|
|
118
|
+
if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`compare.question "${question.question}" (${where}) would be answered by config.analysis.model "${model}", an evaluation model: it scores yes/no questions and can't pick between two attempts. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
|
|
119
|
+
return model;
|
|
120
|
+
}
|
|
121
|
+
/** Checks the model and its credentials before any work; throws a UsageError (exit 2). */
|
|
122
|
+
function resolveAnalysisModel(config, use = "analyze") {
|
|
123
|
+
const model = analysisModelName(config, use);
|
|
124
|
+
if (isFake(model)) return {
|
|
125
|
+
name: "fake",
|
|
126
|
+
route: "fake",
|
|
127
|
+
fake: model
|
|
128
|
+
};
|
|
129
|
+
if (isEvaluationModel(model)) throw new UsageError(`config.analysis.model "${model}" is an evaluation model: it scores yes/no questions and can't write an analysis. Use a language model, e.g. 'google/gemini-3.8-flash'.`);
|
|
130
|
+
try {
|
|
131
|
+
return {
|
|
132
|
+
name: model,
|
|
133
|
+
route: judgeRoute(model).kind
|
|
134
|
+
};
|
|
135
|
+
} catch (error) {
|
|
136
|
+
const message = error.message.replace("judge model", "analysis model");
|
|
137
|
+
throw new UsageError(use === "agent" ? message : `${message} Or run reeval analyze --context, which needs no key.`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/** A language model that answers `text`, through the AI SDK's own parsing, or throws `error`. */
|
|
141
|
+
function cannedModel(answer) {
|
|
142
|
+
return {
|
|
143
|
+
specificationVersion: "v4",
|
|
144
|
+
provider: "fake",
|
|
145
|
+
modelId: "fake-analysis",
|
|
146
|
+
supportedUrls: {},
|
|
147
|
+
doGenerate: async () => {
|
|
148
|
+
if ("error" in answer) throw new Error(answer.error);
|
|
149
|
+
return {
|
|
150
|
+
content: [{
|
|
151
|
+
type: "text",
|
|
152
|
+
text: answer.text
|
|
153
|
+
}],
|
|
154
|
+
finishReason: {
|
|
155
|
+
unified: "stop",
|
|
156
|
+
raw: "stop"
|
|
157
|
+
},
|
|
158
|
+
usage: {
|
|
159
|
+
inputTokens: {
|
|
160
|
+
total: 0,
|
|
161
|
+
noCache: 0,
|
|
162
|
+
cacheRead: 0,
|
|
163
|
+
cacheWrite: 0
|
|
164
|
+
},
|
|
165
|
+
outputTokens: {
|
|
166
|
+
total: 0,
|
|
167
|
+
text: 0,
|
|
168
|
+
reasoning: 0
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
warnings: [],
|
|
172
|
+
response: { modelId: "fake-analysis" }
|
|
173
|
+
};
|
|
174
|
+
},
|
|
175
|
+
doStream: async () => {
|
|
176
|
+
throw new Error("the fake analysis model does not stream");
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
/** A language model that answers with the fake's output. */
|
|
181
|
+
function fakeModel(fake, digest) {
|
|
182
|
+
if (fake.output === void 0) throw new Error("fakeAnalysis() has no output, so it has no analysis to return");
|
|
183
|
+
const output = typeof fake.output === "function" ? fake.output(digest) : fake.output;
|
|
184
|
+
return cannedModel({ text: typeof output === "string" ? output : JSON.stringify(output) });
|
|
185
|
+
}
|
|
186
|
+
async function languageModel(model, digest) {
|
|
187
|
+
if (model.fake) return fakeModel(model.fake, digest);
|
|
188
|
+
return providerModel(model);
|
|
189
|
+
}
|
|
190
|
+
/** The provider's own model for a direct route, else the AI Gateway's by name. */
|
|
191
|
+
async function providerModel(model) {
|
|
192
|
+
if (model.route === "gateway") return model.name;
|
|
193
|
+
const provider = model.name.slice(0, model.name.indexOf("/"));
|
|
194
|
+
const id = model.name.slice(provider.length + 1);
|
|
195
|
+
if (provider === "google") {
|
|
196
|
+
const { google } = await import("@ai-sdk/google");
|
|
197
|
+
return google(id);
|
|
198
|
+
}
|
|
199
|
+
return model.name;
|
|
200
|
+
}
|
|
201
|
+
async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
202
|
+
const started = Date.now();
|
|
203
|
+
const result = await generateText({
|
|
204
|
+
model: await languageModel(model, digest),
|
|
205
|
+
system: SYSTEM_PROMPT,
|
|
206
|
+
prompt,
|
|
207
|
+
output: Output.object({
|
|
208
|
+
schema: analysisSchema,
|
|
209
|
+
name: "analysis"
|
|
210
|
+
}),
|
|
211
|
+
temperature: .2,
|
|
212
|
+
maxOutputTokens,
|
|
213
|
+
maxRetries: 3
|
|
214
|
+
});
|
|
215
|
+
return {
|
|
216
|
+
output: result.output,
|
|
217
|
+
resolvedModel: result.response?.modelId ?? null,
|
|
218
|
+
usage: {
|
|
219
|
+
inputTokens: result.usage?.inputTokens ?? null,
|
|
220
|
+
outputTokens: result.usage?.outputTokens ?? null
|
|
221
|
+
},
|
|
222
|
+
latencyMs: Date.now() - started
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
//#endregion
|
|
226
|
+
export { providerModel as a, resolveAnalysisModel as c, isFake as i, SYSTEM_PROMPT as l, callAnalysisModel as n, questionModel as o, cannedModel as r, questionModelLabel as s, analysisModelName as t };
|
|
227
|
+
|
|
228
|
+
//# sourceMappingURL=model-GFM8iOzM.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-GFM8iOzM.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SACC,wFACF;AAEF,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EAAE,OAAO,CAAC,CAAC,SAAS,kEAAkE;CAC5F,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EAAE,MAAM,KAAK,CAAC,CAAC,SAAS,mDAAmD;CACpF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAU;GAAe;GAAS;GAAS;EAAO,CAAC;EACjE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAM;EAAS,CAAC;EAChE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SAAS,iEAAiE;EAC7E,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;ACzC7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;AAG5E,SAAS,aAAa,QAAgE;CACpF,MAAM,QAAQ,OAAO,SAAS;CAC9B,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,QAAQ,OAAO,MAAM;CAC3B,IAAI,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,GAAG,OAAO;AAErE;;AAGA,SAAS,WAAW,QAAwB,MAAsB;CAChE,MAAM,QAAQ,OAAO,MAAM;CAC3B,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAMA,SAAgB,kBACd,QACA,MAAmB,WACS;CAC5B,MAAM,QAAQ,aAAa,MAAM;CACjC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,GAAG,QAAQ,UAAU,0BAA0B,iBAAiB,iFAAiF,OAAO,OAAO,WAAW,QAAQ,mBAAmB,IAAI,QAAQ,UAAU,wBAAwB,gFAAgF,EACrU;AACF;;AAGA,SAAgB,mBAAmB,QAAuC;CACxE,MAAM,QAAQ,aAAa,MAAM;CACjC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OAC4B;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,QAAQ,aAAa,MAAM;CACjC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,qBAAqB,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,QAAQ,2BAA2B,EAAE,OAAO,IAAI,EAC5N;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,qBAAqB,SAAS,SAAS,KAAK,MAAM,gDAAgD,MAAM,mJAAmJ,IAAI,EACjQ;CAEF,OAAO;AACT;;AAGA,SAAgB,qBACd,QACA,MAAmB,WACI;CACvB,MAAM,QAAQ,kBAAkB,QAAQ,GAAG;CAC3C,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,0BAA0B,MAAM,wIAClC;CAEF,IAAI;EACF,OAAO;GAAE,MAAM;GAAO,OAAO,WAAW,KAAK,CAAC,CAAC;EAAK;CACtD,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB,QAAQ,QAAQ,eAAe,gBAAgB;EAChF,MAAM,IAAI,WACR,QAAQ,UAAU,UAAU,GAAG,QAAQ,sDACzC;CACF;AACF;;AAGA,SAAgB,YAAY,QAA6D;CACvF,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,YAAY;GACtB,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ,MAAM,OAAO;IAAK,CAAC;IAC7C,cAAc;KAAE,SAAS;KAAQ,KAAK;IAAO;IAC7C,OAAO;KACL,aAAa;MAAE,OAAO;MAAG,SAAS;MAAG,WAAW;MAAG,YAAY;KAAE;KACjE,cAAc;MAAE,OAAO;MAAG,MAAM;MAAG,WAAW;KAAE;IAClD;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;AAGA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,IAAI,aAAa,UAAU;EACzB,MAAM,EAAE,WAAW,MAAM,OAAO;EAChC,OAAO,OAAO,EAAE;CAClB;CACA,OAAO,MAAM;AACf;AASA,eAAsB,kBACpB,OACA,QACA,QACA,iBACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,cAAc,OAAO,MAAM;EACxC,QAAQ;EACR;EACA,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAgB,MAAM;EAAW,CAAC;EAClE,aAAa;EACb;EACA,YAAY;CACd,CAAC;CACD,OAAO;EACL,QAAQ,OAAO;EACf,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-CBvm6pap.js";
|
|
2
2
|
import { a as quantile, i as minimumP, n as mannWhitney, r as median } from "./stats-BNAHlNMD.js";
|
|
3
|
-
import {
|
|
3
|
+
import { c as readableTranscriptLine } from "./record-CbuQH4l1.js";
|
|
4
4
|
import { n as promptTokens } from "./usage-DsYUqIUM.js";
|
|
5
5
|
//#region src/view/compare.ts
|
|
6
6
|
const STACK_FIELDS = [
|
|
@@ -419,6 +419,23 @@ function harnessLabel(h) {
|
|
|
419
419
|
const parts = [...h.model ? [`model ${h.model}`] : [], ...h.resolvedModels.length ? [`served ${h.resolvedModels.join(", ")}`] : []];
|
|
420
420
|
return parts.length ? `${h.id} (${parts.join(", ")})` : h.id;
|
|
421
421
|
}
|
|
422
|
+
/**
|
|
423
|
+
* The runtime reeval ran on, as `node 22.14.0`, or `bun 1.4.2 (node 26.3.0)` with the Node version
|
|
424
|
+
* it's compatible with. A run recorded before `stack.runtime` ran on Node.
|
|
425
|
+
*/
|
|
426
|
+
function runtimeLabel(stack) {
|
|
427
|
+
const runtime = stack.runtime ?? {
|
|
428
|
+
name: "node",
|
|
429
|
+
version: stack.node
|
|
430
|
+
};
|
|
431
|
+
return runtime.name === "node" ? `node ${runtime.version}` : `${runtime.name} ${runtime.version} (node ${stack.node})`;
|
|
432
|
+
}
|
|
433
|
+
/** The evals' language as `typescript evals` or `dart 3.13.5 evals`; null when not recorded. */
|
|
434
|
+
function languageLabel(stack) {
|
|
435
|
+
const language = stack.language;
|
|
436
|
+
if (!language) return null;
|
|
437
|
+
return `${language.version ? `${language.name} ${language.version}` : language.name} evals`;
|
|
438
|
+
}
|
|
422
439
|
/** Harness-reported tokens as `127k in (98k cached) · 776 out`; null when none were reported. */
|
|
423
440
|
function formatUsage(usage) {
|
|
424
441
|
if (!usage) return null;
|
|
@@ -583,6 +600,27 @@ function cellRows(e) {
|
|
|
583
600
|
attempts: e.attempts.filter((a) => a.cell === cell.key)
|
|
584
601
|
}));
|
|
585
602
|
}
|
|
603
|
+
/** One row per compare question, with each cut's win rates in the record's order. */
|
|
604
|
+
function answerRows(e) {
|
|
605
|
+
return (e.answers ?? []).map((a) => {
|
|
606
|
+
const rates = (cut) => a.rates.filter((r) => r.cut === cut).map((r) => ({
|
|
607
|
+
name: r.name,
|
|
608
|
+
wins: r.wins,
|
|
609
|
+
n: r.n,
|
|
610
|
+
rate: r.n ? r.wins / r.n : null
|
|
611
|
+
}));
|
|
612
|
+
return {
|
|
613
|
+
question: a.question,
|
|
614
|
+
model: a.resolvedModel ?? a.model,
|
|
615
|
+
votes: a.votes.length,
|
|
616
|
+
ties: a.votes.filter((v) => v.winner === "tie").length,
|
|
617
|
+
within: rates("within"),
|
|
618
|
+
across: rates("across"),
|
|
619
|
+
error: a.error ?? null,
|
|
620
|
+
noPairs: a.votes.length === 0 && a.error === void 0
|
|
621
|
+
};
|
|
622
|
+
});
|
|
623
|
+
}
|
|
586
624
|
/** Whether an eval needs the cells table: several cells, or several runs of one. */
|
|
587
625
|
const isMatrix = (e) => e.cells.length > 1 || e.cells.some((c) => c.runs > 1);
|
|
588
626
|
/** One eval across runs, oldest first. */
|
|
@@ -720,6 +758,6 @@ const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId)
|
|
|
720
758
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
721
759
|
const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
|
|
722
760
|
//#endregion
|
|
723
|
-
export {
|
|
761
|
+
export { VERDICT_TEXT as A, effortVerdict as B, runRows as C, splitPatch as D, splitEvalId as E, effortChangeLabel as F, compare as G, formatRatio as H, effortLabel as I, whyChanged as J, sourceChanges as K, effortOf as L, cellEffort as M, cellSamples as N, transcriptItems as O, contrastEffort as P, effortSample as R, runOutcome as S, shortSha as T, formatSeconds as U, formatCount as V, cellsChanged as W, indexFiles as _, cellRows as a, languageLabel as b, evalHistory as c, findEval as d, formatDuration as f, hasUngatedCells as g, harnessLabel as h, buildIndex as i, cellContrast as j, EFFORT_ALPHA as k, evalTree as l, formatUsage as m, activityLabel as n, compareView as o, formatPercent as p, stackChanges as q, answerRows as r, defaultAttempt as s, COMPARE_ORDER as t, findAttempt as u, isMatrix as v, runtimeLabel as w, parseTranscript as x, isUngatedMiss as y, effortSummary as z };
|
|
724
762
|
|
|
725
|
-
//# sourceMappingURL=model-
|
|
763
|
+
//# sourceMappingURL=model-ORtLm2UB.js.map
|