reeval 0.17.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/{adapter-DyvDj4ZN.js → adapter-CptK032x.js} +2 -2
- package/dist/{adapter-DyvDj4ZN.js.map → adapter-CptK032x.js.map} +1 -1
- package/dist/{agent-v036GEgG.js → agent-B63OzUX9.js} +6 -6
- package/dist/{agent-v036GEgG.js.map → agent-B63OzUX9.js.map} +1 -1
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-sgqwfjtn.js} +4 -4
- package/dist/app-files-sgqwfjtn.js.map +1 -0
- package/dist/assemble-CxBHkavM.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-Me7Whp-y.js} +58 -194
- package/dist/assemble-Me7Whp-y.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
- package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-Cik0SoIn.js} +17 -9
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
- package/dist/{capture-Bdcx9Gap.js → capture-C3RmjPph.js} +392 -613
- package/dist/capture-C3RmjPph.js.map +1 -0
- package/dist/capture-YPwxp_iK.js +1 -0
- package/dist/{cli-CGd51thd.js → cli-BBOlXKPG.js} +1075 -457
- package/dist/cli-BBOlXKPG.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
- package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
- package/dist/client-tZ2WBDxf.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-CFQ9c1hk.js} +13 -13
- package/dist/{command-TLmY-rYp.js.map → command-CFQ9c1hk.js.map} +1 -1
- package/dist/{command-CtTdCF3B.js → command-DhiwsZl1.js} +9 -9
- package/dist/{command-CtTdCF3B.js.map → command-DhiwsZl1.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-fO0BFsTq.js} +102 -37
- package/dist/command-fO0BFsTq.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-qGS802uX.js} +16 -14
- package/dist/command-qGS802uX.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CC60TOT5.js} +64 -18
- package/dist/commands-CC60TOT5.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-xYTMNTPv.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-C2UtS9vy.js} +718 -76
- package/dist/config-C2UtS9vy.js.map +1 -0
- package/dist/{contrasts-DtSX8E_V.js → contrasts-BjOg7wyr.js} +3 -3
- package/dist/{contrasts-DtSX8E_V.js.map → contrasts-BjOg7wyr.js.map} +1 -1
- package/dist/cost-BJU0MAji.js +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BTK_apL-.js} +2 -2
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
- package/dist/{db-sHGpaLo9.js → db-C5SZ0510.js} +72 -14
- package/dist/db-C5SZ0510.js.map +1 -0
- package/dist/db-puzLzX4k.js +2 -0
- package/dist/{default-effort-BZRa-SQP.js → default-effort-BVmx7r7L.js} +8 -5
- package/dist/default-effort-BVmx7r7L.js.map +1 -0
- package/dist/{default-effort-fpdbYVKm.js → default-effort-FEidNAIi.js} +1 -1
- package/dist/{docker-CWuSRj7a.js → docker-B2F8-PGC.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-B2F8-PGC.js.map} +1 -1
- package/dist/{docker-CIzHzwlC.js → docker-BlSopHpZ.js} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-Dto8ROCq.js} +5 -5
- package/dist/{docker--2ytOHcu.js.map → docker-Dto8ROCq.js.map} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-BcvfKpBX.js} +20 -20
- package/dist/{doctor-eE65YpEv.js.map → doctor-BcvfKpBX.js.map} +1 -1
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-CC1RV21n.js} +3 -3
- package/dist/{environment-Cy3FVLR5.js.map → environment-CC1RV21n.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-CzxdgFBW.js} +18 -10
- package/dist/eval-source-CzxdgFBW.js.map +1 -0
- package/dist/evaluate-DToSB2bt.d.ts +101 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-NTNJ8T9H.js} +129 -51
- package/dist/evaluate-NTNJ8T9H.js.map +1 -0
- package/dist/{files-tnMgu7-X.js → files-CZXrkPja.js} +5 -5
- package/dist/{files-tnMgu7-X.js.map → files-CZXrkPja.js.map} +1 -1
- package/dist/files-DFtuRQJx.js +1 -0
- package/dist/{files-CQU08uh4.js → files-DsXULUi5.js} +3 -3
- package/dist/{files-CQU08uh4.js.map → files-DsXULUi5.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-aAzoGNCB.js} +4 -4
- package/dist/{files-CIX-pLl4.js.map → files-aAzoGNCB.js.map} +1 -1
- package/dist/files-oOYNR7ZH.js +1 -0
- package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
- package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-C1MB5bjr.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-C1MB5bjr.js.map} +1 -1
- package/dist/harness-DGsRZDSo.js +105 -0
- package/dist/harness-DGsRZDSo.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
- package/dist/harness-DYSC4zMh.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-BVZHDf_z.js → images-CNayXCp2.js} +8 -8
- package/dist/{images-BVZHDf_z.js.map → images-CNayXCp2.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5 -5
- package/dist/init-8o6W3_fn.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-D5HtmyMI.js} +5 -5
- package/dist/{init-BWnMbUnc.js.map → init-D5HtmyMI.js.map} +1 -1
- package/dist/ink-list-Ca-5dunO.js +184 -0
- package/dist/ink-list-Ca-5dunO.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-DrGXF9dT.js} +3 -3
- package/dist/{invocations-BGRs1r_C.js.map → invocations-DrGXF9dT.js.map} +1 -1
- package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
- package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-m-mb0zv8.js +246 -0
- package/dist/judgments-m-mb0zv8.js.map +1 -0
- package/dist/list-CM_YSJXq.js +374 -0
- package/dist/list-CM_YSJXq.js.map +1 -0
- package/dist/{live-d_5a5iUv.js → live-B71o2TkX.js} +21 -3
- package/dist/live-B71o2TkX.js.map +1 -0
- package/dist/live-DQc8jr6o.js +1 -0
- package/dist/{live-ColieOai.js → live-uRURFDMZ.js} +7 -7
- package/dist/{live-ColieOai.js.map → live-uRURFDMZ.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
- package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-BxBWFOZn.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-BxBWFOZn.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-CXVVB4RK.js} +1 -1
- package/dist/{model-DOiCiMCD.js → model-Ce9l9_rF.js} +19 -255
- package/dist/model-Ce9l9_rF.js.map +1 -0
- package/dist/model-Cxzo4cjw.js +1 -0
- package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
- package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
- package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
- package/dist/{patch-DvLr8wCZ.js → patch-ug5HpMOj.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-ug5HpMOj.js.map} +1 -1
- package/dist/{plan-CIfOFhth.js → plan-DqnzfDH1.js} +15 -14
- package/dist/plan-DqnzfDH1.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-B2KQ0y3P.js} +4 -4
- package/dist/{plugins-BEs2Am3L.js.map → plugins-B2KQ0y3P.js.map} +1 -1
- package/dist/{preflight-BAIRsTpl.js → preflight-DVXEvzFU.js} +1 -1
- package/dist/{preflight-50Qw5e5K.js → preflight-DyzmvK7i.js} +3 -3
- package/dist/{preflight-50Qw5e5K.js.map → preflight-DyzmvK7i.js.map} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-21WMQZyt.js} +7 -7
- package/dist/{prepare-8XhzhLHX.js.map → prepare-21WMQZyt.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-BcSRa7-3.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
- package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-iRxjp4Pu.js} +4 -4
- package/dist/{provenance-Cc8fFE7Z.js.map → provenance-iRxjp4Pu.js.map} +1 -1
- package/dist/{providers-CP_bBtYm.js → providers-D0ob_tra.js} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-vXMXuwn-.js} +5 -5
- package/dist/{providers-DZSTlC9R.js.map → providers-vXMXuwn-.js.map} +1 -1
- package/dist/{query-DXpxaaTH.js → query-BhaKQ07P.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-BhaKQ07P.js.map} +1 -1
- package/dist/query-CvgUsvep.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-BzdLdV3C.js} +39 -9
- package/dist/questions-BzdLdV3C.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-CauU408t.js} +231 -25
- package/dist/remarks-CauU408t.js.map +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/{remarks-DeZZT8PK.js → remarks-NMgPd30U.js} +4 -3
- package/dist/{remarks-DeZZT8PK.js.map → remarks-NMgPd30U.js.map} +1 -1
- package/dist/{remote-DZ00A8V_.js → remote-Dwy3fyo3.js} +5 -5
- package/dist/{remote-DZ00A8V_.js.map → remote-Dwy3fyo3.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-B1zMnPwK.js} +7 -7
- package/dist/{resolve-B_JlQ0La.js.map → resolve-B1zMnPwK.js.map} +1 -1
- package/dist/resolve-BIIeVT12.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-CfC0id_u.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-CfC0id_u.js.map} +1 -1
- package/dist/{resume-CexsWuGK.js → resume-Dkhrpjow.js} +123 -8
- package/dist/resume-Dkhrpjow.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-DcvQcr36.js} +2 -2
- package/dist/{runtime-CSZ1hjlN.js.map → runtime-DcvQcr36.js.map} +1 -1
- package/dist/{sandbox-B5W8lLL1.js → sandbox-8zCrtuCf.js} +2 -2
- package/dist/{sandbox-B5W8lLL1.js.map → sandbox-8zCrtuCf.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-0tuj8VSa.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Dplq8g0r.js} +7 -7
- package/dist/{server-BfN6q2lH.js.map → server-Dplq8g0r.js.map} +1 -1
- package/dist/{source-BD6Bg0Tr.js → source-NWHqn5-o.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-NWHqn5-o.js.map} +1 -1
- package/dist/source-llwLewOU.js +1 -0
- package/dist/{src-BC2WXULN.js → src-B9oE_mNS.js} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-BQ6BiXoc.js} +19 -11
- package/dist/{src-Cpy9MNS3.js.map → src-BQ6BiXoc.js.map} +1 -1
- package/dist/{src-Dp-fWOfG.js → src-BQXdf-Gw.js} +2 -2
- package/dist/{src-Dp-fWOfG.js.map → src-BQXdf-Gw.js.map} +1 -1
- package/dist/{src-DurP--_D.js → src-BXnv0ViU.js} +1 -1
- package/dist/src-BiS8YNN_.js +3 -0
- package/dist/{src-DVaAKCsH.js → src-BkH4GOxu.js} +1 -1
- package/dist/{src-CM7xdJFA.js → src-C-tH5ebd.js} +1 -1
- package/dist/{src-Cx0HJIJ-.js → src-CUpoVVb8.js} +1 -1
- package/dist/{src-C9elednc.js → src-Cg1bFs9g.js} +1 -1
- package/dist/{src-BKDZO3ra.js → src-DV_UorZv.js} +1 -1
- package/dist/{src-CWEGknZo.js → src-XxUpr2sa.js} +1 -1
- package/dist/{state-CdSjXIfE.js → state-DDCGUhva.js} +5 -5
- package/dist/{state-CdSjXIfE.js.map → state-DDCGUhva.js.map} +1 -1
- package/dist/state-DZbAO95a.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-M8_7xio7.js +1294 -0
- package/dist/suite-M8_7xio7.js.map +1 -0
- package/dist/suite-u2KJnGpY.js +2 -0
- package/dist/{summary-zP3cCXDN.js → summary-Uzc9osfV.js} +47 -13
- package/dist/summary-Uzc9osfV.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
- package/dist/{types-BHR1lGPh.d.ts → types-DGvexkcp.d.ts} +71 -63
- package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
- package/dist/vertex-BLQMayc1.js.map +1 -0
- package/dist/vertex-h_peaZTN.js +1 -0
- package/dist/view/index.d.ts +140 -35
- package/dist/view/index.js +8 -7
- package/dist/{views-CKoN5NAx.js → views-BISrsIjw.js} +6 -6
- package/dist/{views-CKoN5NAx.js.map → views-BISrsIjw.js.map} +1 -1
- package/dist/views-Bfab8fM9.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-DJD_nHkS.js} +5 -5
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
- package/dist/workspace-B8EOCuxl.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-BO1VhUR9.js} +5 -5
- package/dist/{workspace-Cho30qlM.js.map → workspace-BO1VhUR9.js.map} +1 -1
- package/package.json +3 -1
- package/skills/eval-rubric/SKILL.md +22 -3
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -56,6 +56,22 @@ function readableAttempt(v) {
|
|
|
56
56
|
durationMs: durationMs ?? null
|
|
57
57
|
};
|
|
58
58
|
}
|
|
59
|
+
const strings = (v) => Array.isArray(v) && v.every(str);
|
|
60
|
+
/** A saved invocation, or undefined when it isn't one; its numbers are whole and at least 1. */
|
|
61
|
+
function readableInvocation(v) {
|
|
62
|
+
if (!isObj(v)) return void 0;
|
|
63
|
+
const { runs, profile, concurrency, filters, tags } = v;
|
|
64
|
+
const count = (n) => n === void 0 || Number.isInteger(n) && n >= 1;
|
|
65
|
+
if (!count(runs) || !count(concurrency) || !(profile === void 0 || str(profile))) return void 0;
|
|
66
|
+
if (!strings(filters) || !strings(tags)) return void 0;
|
|
67
|
+
return {
|
|
68
|
+
...runs === void 0 ? {} : { runs },
|
|
69
|
+
...profile === void 0 ? {} : { profile },
|
|
70
|
+
...concurrency === void 0 ? {} : { concurrency },
|
|
71
|
+
filters,
|
|
72
|
+
tags
|
|
73
|
+
};
|
|
74
|
+
}
|
|
59
75
|
/**
|
|
60
76
|
* A progress file as the studio reads it, or null when it isn't one it can read: half-written,
|
|
61
77
|
* corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
|
|
@@ -68,6 +84,7 @@ function readableLiveProgress(value) {
|
|
|
68
84
|
if (!isOneOf(PHASES, phase) || !Array.isArray(attempts)) return null;
|
|
69
85
|
const list = attempts.map(readableAttempt);
|
|
70
86
|
if (!list.every((a) => a !== null)) return null;
|
|
87
|
+
const invocation = readableInvocation(value.invocation);
|
|
71
88
|
return {
|
|
72
89
|
schemaVersion: 1,
|
|
73
90
|
runId,
|
|
@@ -78,9 +95,10 @@ function readableLiveProgress(value) {
|
|
|
78
95
|
phase,
|
|
79
96
|
planned: list.length,
|
|
80
97
|
counts: liveCounts(list),
|
|
81
|
-
attempts: list
|
|
98
|
+
attempts: list,
|
|
99
|
+
...invocation ? { invocation } : {}
|
|
82
100
|
};
|
|
83
101
|
}
|
|
84
|
-
export { liveApiPath as a,
|
|
102
|
+
export { liveApiPath as a, readableInvocation as c, isFinished as i, readableLiveProgress as l, LIVE_FILE as n, liveCounts as o, LIVE_SCHEMA_VERSION as r, liveTranscriptPath as s, FINISHED as t };
|
|
85
103
|
|
|
86
|
-
//# sourceMappingURL=live-
|
|
104
|
+
//# sourceMappingURL=live-B71o2TkX.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live-B71o2TkX.js","names":[],"sources":["../src/view/live.ts"],"mappings":";;;;AASA,MAAa,YAAY;AACzB,MAAa,sBAAsB;AAwDnC,MAAa,cAAc;AAC3B,MAAa,sBAAsB,OAAe,eAChD,YAAY,MAAM,sBAAsB,mBAAmB,UAAU;AAEvE,MAAa,WAAkC;CAAC;CAAQ;CAAQ;CAAe;AAAS;AACxF,MAAa,cAAc,WAAuB,SAAS,SAAS,MAAM;AAE1E,SAAgB,WAAW,UAA8D;CACvF,MAAM,SAAS,GAAG,aAChB,SAAS,QAAQ,MAAM,SAAS,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD,OAAO;EACL,MAAM,MAAM,GAAG,QAAQ;EACvB,QAAQ,MAAM,MAAM;EACpB,QAAQ,MAAM,MAAM;EACpB,SAAS,MAAM,aAAa;EAC5B,SAAS,MAAM,SAAS;EACxB,SAAS,MAAM,SAAS;CAC1B;AACF;AAEA,MAAM,WAAkC;CAAC;CAAU;CAAW,GAAG;AAAQ;AACzE,MAAM,SAA+B;CAAC;CAAW;CAAc;AAAW;AAC1E,MAAM,SAAS,MACb,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AACzD,MAAM,OAAO,MAA4B,OAAO,MAAM;AACtD,MAAM,OAAO,MAA4B,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC;AACnF,MAAM,UAAa,GAAY,OAC7B,MAAM,QAAQ,MAAM,KAAA,KAAa,GAAG,CAAC;AAEvC,SAAS,gBAAgB,GAAgC;CACvD,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO;CACtB,MAAM,EAAE,KAAK,QAAQ,MAAM,MAAM,MAAM,GAAG,QAAQ,WAAW,eAAe;CAC5E,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,UAAU,MAAM,GAAG,OAAO;CACvC,IAAI,CAAC,OAAO,WAAW,GAAG,KAAK,CAAC,OAAO,YAAY,GAAG,GAAG,OAAO;CAChE,OAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW,aAAa;EACxB,YAAY,cAAc;CAC5B;AACF;AAEA,MAAM,WAAW,MAA8B,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG;;AAG9E,SAAgB,mBAAmB,GAAuC;CACxE,IAAI,CAAC,MAAM,CAAC,GAAG,OAAO,KAAA;CACtB,MAAM,EAAE,MAAM,SAAS,aAAa,SAAS,SAAS;CACtD,MAAM,SAAS,MAAe,MAAM,KAAA,KAAc,OAAO,UAAU,CAAC,KAAM,KAAgB;CAC1F,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,WAAW,KAAK,EAAE,YAAY,KAAA,KAAa,IAAI,OAAO,IAC/E,OAAO,KAAA;CACT,IAAI,CAAC,QAAQ,OAAO,KAAK,CAAC,QAAQ,IAAI,GAAG,OAAO,KAAA;CAChD,OAAO;EACL,GAAI,SAAS,KAAA,IAAY,CAAC,IAAI,EAAQ,KAAe;EACrD,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;EAC3C,GAAI,gBAAgB,KAAA,IAAY,CAAC,IAAI,EAAe,YAAsB;EAC1E;EACA;CACF;AACF;;;;;AAMA,SAAgB,qBAAqB,OAAqC;CACxE,IAAI,CAAC,MAAM,KAAK,KAAK,MAAM,kBAAA,GAAuC,OAAO;CACzE,aAAa,iBAAiB,KAAK;CACnC,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,aAAa,OAAO,aAAa;CACtE,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,WAAW,KAAK,CAAC,IAAI,GAAG,GAAG,OAAO;CAC3F,IAAI,CAAC,QAAQ,QAAQ,KAAK,KAAK,CAAC,MAAM,QAAQ,QAAQ,GAAG,OAAO;CAChE,MAAM,OAAO,SAAS,IAAI,eAAe;CACzC,IAAI,CAAC,KAAK,OAAO,MAAM,MAAM,IAAI,GAAG,OAAO;CAC3C,MAAM,aAAa,mBAAmB,MAAM,UAAU;CACtD,OAAO;EACL,eAAA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS,KAAK;EACd,QAAQ,WAAW,IAAI;EACvB,UAAU;EACV,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;CACrC;AACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { n as listLive } from "./live-uRURFDMZ.js";
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { t as RUN_ID } from "./paths-Dwmi7L4X.js";
|
|
2
2
|
import { s as recordFileName } from "./ids-DuK6xl4w.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { l as recordFile, p as scratchDir } from "./files-
|
|
6
|
-
import { t as fileInside } from "./source-
|
|
7
|
-
import {
|
|
3
|
+
import { p as redactJsonText } from "./redact-BhjtnjrU.js";
|
|
4
|
+
import { i as isFinished, l as readableLiveProgress, n as LIVE_FILE } from "./live-B71o2TkX.js";
|
|
5
|
+
import { l as recordFile, p as scratchDir } from "./files-CZXrkPja.js";
|
|
6
|
+
import { t as fileInside } from "./source-NWHqn5-o.js";
|
|
7
|
+
import { c as previousAttempt, n as isCompletedAttempt, o as loadPrevious } from "./resume-Dkhrpjow.js";
|
|
8
8
|
import { existsSync } from "node:fs";
|
|
9
9
|
import { hostname } from "node:os";
|
|
10
10
|
import { join } from "node:path";
|
|
@@ -85,7 +85,7 @@ async function liveTranscript(source, runId, key) {
|
|
|
85
85
|
const file = await fileInside(stagingRoot(source.root), [runId], ["evals", recordFileName(key)]);
|
|
86
86
|
if (!file) return null;
|
|
87
87
|
const record = JSON.parse(await readFile(file, "utf8"));
|
|
88
|
-
const { transcriptLines } = await import("./assemble-
|
|
88
|
+
const { transcriptLines } = await import("./assemble-CxBHkavM.js");
|
|
89
89
|
return redactJsonText(transcriptLines(record));
|
|
90
90
|
} catch {
|
|
91
91
|
return null;
|
|
@@ -117,4 +117,4 @@ function watchLive(source, onChange, pollMs = 400) {
|
|
|
117
117
|
}
|
|
118
118
|
export { watchLive as a, stagingRoot as i, listLive as n, liveTranscript as r, STALE_MS as t };
|
|
119
119
|
|
|
120
|
-
//# sourceMappingURL=live-
|
|
120
|
+
//# sourceMappingURL=live-uRURFDMZ.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"live-
|
|
1
|
+
{"version":3,"file":"live-uRURFDMZ.js","names":[],"sources":["../src/studio/live.ts"],"mappings":";;;;;;;;;;;;;AAuBA,MAAa,WAAW;AAExB,MAAa,eAAe,SAAiB,KAAK,MAAM,WAAW,SAAS;;AAK5E,SAAS,OAAO,UAAiC;CAC/C,IAAI,SAAS,SAAS,SAAS,GAAG,OAAO;CACzC,IAAI;EACF,QAAQ,KAAK,SAAS,KAAK,CAAC;EAC5B,OAAO;CACT,SAAS,OAAO;EACd,OAAQ,MAAgC,SAAS;CACnD;AACF;AAEA,eAAe,aAAa,MAAc,OAA6C;CACrF,MAAM,OAAO,MAAM,WAAW,YAAY,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,SAAS,CAAC;CACrE,MAAM,OAAO,QAAS,MAAM,SAAS,MAAM,MAAM,CAAC,CAAC,YAAY,IAAI;CACnE,IAAI,CAAC,MAAM,OAAO;CAClB,IAAI;EACF,MAAM,WAAW,qBAAqB,KAAK,MAAM,IAAI,CAAC;EACtD,OAAO,UAAU,UAAU,QAAQ,WAAW;CAChD,QAAQ;EACN,OAAO;CACT;AACF;AAEA,MAAM,UAAU,QAAsB,UACpC,CAAC,OAAO,YAAY,WAAW,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,QAAQ,WAAW,WAAW,KAAK,KAAK,CAAC,CAAC;;;;;AAM/F,eAAsB,SACpB,QACA,UAAuB,CAAC,GACJ;CACpB,IAAI,CAAC,OAAO,SAAS,OAAO,CAAC;CAC7B,MAAM,MAAM,QAAQ,OAAO,KAAK,IAAI;CACpC,MAAM,UAAU,QAAQ,WAAA;CACxB,MAAM,QAAQ,MAAM,QAAQ,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,YAAsB,CAAC,CAAC;CAC9E,MAAM,OAAkB,CAAC;CACzB,KAAK,MAAM,SAAS,MACjB,QAAQ,MAAM,OAAO,KAAK,CAAC,CAAC,CAAC,CAC7B,KAAK,CAAC,CACN,QAAQ,GAAG;EACZ,MAAM,WAAW,MAAM,aAAa,OAAO,MAAM,KAAK;EACtD,IAAI,CAAC,YAAY,OAAO,QAAQ,KAAK,GAAG;EACxC,MAAM,QAAQ,MAAM,KAAK,MAAM,SAAS,WAAW,IAAI;EACvD,MAAM,EAAE,eAAe,GAAG,KAAK,IAAI,MAAM,KAAK,aAAa,MAAM,GAAG,QAAQ;EAC5E,KAAK,KAAK;GAAE,GAAG;GAAK,OAAO,SAAS,OAAO,QAAQ,IAAI,YAAY,SAAS;EAAM,CAAC;CACrF;CACA,OAAO;AACT;;;;;AAMA,eAAe,eAAe,MAAc,OAAe,KAAa;CACtE,MAAM,EAAE,WAAW,MAAM,aAAa,KAAK,YAAY,IAAI,GAAG,KAAK,CAAC;CACpE,MAAM,OAAO,gBAAgB,QAAQ,GAAG;CACxC,MAAM,MAAM,QAAQ,mBAAmB,KAAK,SAAS,KAAK,KAAK,IAAI,KAAK,MAAM,aAAa;CAC3F,IAAI,CAAC,KAAK,OAAO,KAAA;CACjB,MAAM,OAAO,CAAC,sBAAsB,GAAG,IAAI,KAAK,MAAM,GAAG,CAAC;CAC1D,MAAM,OAAO,MAAM,WAAW,YAAY,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI;CAC9D,OAAO,OAAO,eAAe,MAAM,SAAS,MAAM,MAAM,CAAC,IAAI,KAAA;AAC/D;;;;;;AAOA,eAAsB,eACpB,QACA,OACA,KACwB;CACxB,IAAI,CAAC,OAAO,WAAW,CAAC,OAAO,KAAK,KAAK,GAAG,OAAO;CAEnD,MAAM,WAAU,MADO,aAAa,OAAO,MAAM,KAAK,EAAA,EAC5B,SAAS,MAAM,MAAM,EAAE,QAAQ,GAAG;CAC5D,IAAI,CAAC,WAAW,CAAC,WAAW,QAAQ,MAAM,GAAG,OAAO;CACpD,IAAI;EACF,MAAM,OAAO,MAAM,eAAe,OAAO,MAAM,OAAO,GAAG;EACzD,IAAI,SAAS,KAAA,GAAW,OAAO;EAC/B,MAAM,OAAO,MAAM,WAAW,YAAY,OAAO,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,SAAS,eAAe,GAAG,CAAC,CAAC;EAC/F,IAAI,CAAC,MAAM,OAAO;EAClB,MAAM,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;EACtD,MAAM,EAAE,oBAAoB,MAAM,OAAO;EACzC,OAAO,eAAe,gBAAgB,MAAM,CAAC;CAC/C,QAAQ;EACN,OAAO;CACT;AACF;;AAGA,SAAgB,UAAU,QAAsB,UAAqC,SAAS,KAAK;CACjG,IAAI,YAAY;CAChB,IAAI,OAAO;CACX,MAAM,QAAQ,YAAY;EACxB,IAAI,MAAM;EACV,OAAO;EACP,IAAI;GACF,MAAM,OAAO,MAAM,SAAS,MAAM;GAClC,MAAM,OAAO,KAAK,UAAU,IAAI;GAChC,IAAI,SAAS,WAAW;IACtB,MAAM,QAAQ,cAAc;IAC5B,YAAY;IACZ,IAAI,CAAC,OAAO,SAAS,IAAI;GAC3B;EACF,UAAU;GACR,OAAO;EACT;CACF;CACA,MAAW;CACX,MAAM,OAAO,kBAAkB,KAAK,MAAM,GAAG,MAAM;CACnD,KAAK,MAAM;CACX,aAAa,cAAc,IAAI;AACjC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { r as isThenable, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
|
-
import {
|
|
3
|
-
import { a as plural } from "./text-
|
|
2
|
+
import { c as calledTool, d as fetched, f as ran, l as changed, m as subjectError, o as calledMcp, p as searched, s as calledSkill, u as edited } from "./client-tZ2WBDxf.js";
|
|
3
|
+
import { a as plural } from "./text-B8-W3Cvn.js";
|
|
4
4
|
import { isDeepStrictEqual } from "node:util";
|
|
5
5
|
// src/matchers.ts
|
|
6
6
|
const matcher = (run) => run;
|
|
@@ -144,4 +144,4 @@ const matchers = Object.fromEntries(Object.keys(MATCHERS).map((name) => [name, f
|
|
|
144
144
|
const evalMatchers = Object.fromEntries(Object.keys(matchers).filter((name) => !GENERIC_MATCHERS.has(name)).map((name) => [name, matchers[name]]));
|
|
145
145
|
export { runMatcher as i, evalMatchers as n, isMatcherName as r, MATCHERS as t };
|
|
146
146
|
|
|
147
|
-
//# sourceMappingURL=matchers-
|
|
147
|
+
//# sourceMappingURL=matchers-Cs8aDqdU.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers-
|
|
1
|
+
{"version":3,"file":"matchers-Cs8aDqdU.js","names":[],"sources":["../src/matchers.ts"],"mappings":";;;;;AAiCA,MAAM,WAAoE,QAAmB;;AAG7F,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,aAAa,KAAK,KAAK,OAAO,MAAM,aAAa;AAEnD,MAAM,QAAQ,UAA2B;CACvC,IAAI,OAAO,UAAU,UAAU,OAAO,IAAI,MAAM;CAChD,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF;AAEA,SAAS,UAAU,QAAyB;CAC1C,MAAM,IAAI;CACV,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,aAAa,uCAAuC;CAC/F,OAAO;EACL,MAAM,EAAE,aAAa;EACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;CAC3E;AACF;AAEA,MAAM,aAAa,UACjB,aAAa,KAAK,KAClB,OAAO,MAAM,aAAa,YAC1B,MAAM,QAAQ,MAAM,MAAM,KAC1B,MAAM,QAAQ,MAAM,OAAO;;AAG7B,MAAM,aAAa,MACjB,OAAO,EAAE,SAAS,WAAW,YAAY,EAAE,KAAK,OAAO,EAAE,aAAa,EAAE;;AAG1E,MAAM,eAAe,GAAoC,MAAM,MAAM;CACnE,MAAM,QAAQ,EAAE,OACb,MAAM,GAAG,GAAG,CAAC,CACb,KAAK,MAAM,KAAK,EAAE,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,GAAG,GAAG,CAAC;CAChE,MAAM,OAAO,EAAE,OAAO,SAAS,MAAM;CACrC,MAAM,OAAO,OAAO,EAAE,SAAS,WAAW,OAAO,EAAE,KAAK,iBAAiB;CACzE,OAAO,CAAC,GAAG,OAAO,GAAI,OAAO,IAAI,CAAC,KAAK,KAAK,OAAO,MAAM,IAAI,CAAC,CAAE,CAAC,CAAC,KAAK,IAAI;AAC7E;AAEA,MAAa,WAAW;CACtB,MAAM,SAAS,QAAQ,CAAC,cAAyB;EAC/C,MAAM,OAAO,OAAO,GAAG,QAAQ,QAAQ;EACvC,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,0BACrD,YAAY,KAAK,MAAM,EAAE,SAAS,KAAK,QAAQ,EAAE;EACzD;CACF,CAAC;CAED,SAAS,SAAS,QAAQ,CAAC,cAAyB;EAClD,MAAM,OAAO,kBAAkB,QAAQ,QAAQ;EAC/C,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,uBAAuB,KAAK,QAAQ,MAC7D,YAAY,KAAK,MAAM,EAAE,mBAAmB,KAAK,QAAQ;EACjE;CACF,CAAC;CAED,UAAU,SAAS,QAAQ,OAAW;EACpC,MAAM,WAAW;EACjB,eACE,WAAW,OACP,YAAY,KAAK,MAAM,EAAE,mBACzB,YAAY,KAAK,MAAM,EAAE;CACjC,EAAE;CAEF,YAAY,SAAS,QAAQ,OAAW;EACtC,MAAM,CAAC,CAAC;EACR,eACE,SAAS,YAAY,KAAK,MAAM,EAAE,gBAAgB,YAAY,KAAK,MAAM,EAAE;CAC/E,EAAE;CAEF,WAAW,SAAS,QAAQ,CAAC,YAAuB;EAClD,MAAM,OACJ,OAAO,WAAW,YAAY,OAAO,WAAW,WAC5C,OAAO,SAAS,MAAM,IACtB,MAAM,QAAQ,MAAM,KAAK,OAAO,SAAS,MAAM;EACrD,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,kBAAkB,KAAK,MAAM,MACtD,YAAY,KAAK,MAAM,EAAE,cAAc,KAAK,MAAM;EAC1D;CACF,CAAC;CAED,YAAY,QACV,OAAO,QAAQ,CAAC,MAAM,aAAwD;EAC5E,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,aAAa,iDAAiD;EAC3F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF,CACF;CAGA,WAAW,SAAS,QAAQ,MAC1B,WAAW,MAAM,IAAI,QAAQ,QAAQ,MAAM,CAAC,CAAC,KAAK,SAAS,IAAI,UAAU,MAAM,CACjF;CAEA,gBAAgB,SAAS,QAAQ,CAAC,WAAsC,SAAS,QAAQ,KAAK,CAAC;CAE/F,eAAe,SAAS,QAAQ,CAAC,SAAkC,QAAQ,QAAQ,GAAG,CAAC;CAEvF,mBAAmB,SAAS,QAAQ,CAAC,UAA0B,YAAY,QAAQ,IAAI,CAAC;CAExF,iBAAiB,SACd,QAAQ,CAAC,QAAQ,MAAM,WACtB,UAAU,QAAQ,QAAQ,MAAM,KAAK,CACzC;CAEA,kBAAkB,SACf,QAAQ,CAAC,MAAM,WACd,WAAW,QAAQ,MAAM,KAAK,CAClC;CAEA,WAAW,SAAS,QAAQ,CAAC,aAA0C,IAAI,QAAQ,OAAO,CAAC;CAE3F,cAAc,SAAS,QAAQ,CAAC,UAAoC,OAAO,QAAQ,IAAI,CAAC;CAExF,eAAe,SAAS,QAAQ,CAAC,WAAgC,QAAQ,QAAQ,KAAK,CAAC;CAEvF,cAAc,SAAS,QAAQ,CAAC,YAAoD;EAClF,IAAI,CAAC,UAAU,MAAM,GAAG,MAAM,aAAa,6CAA6C;EACxF,MAAM,QAAQ,OAAO,WAAW,YAAY,YAAY,KAAK,MAAM,IAAI,OAAO,OAAO,EAAE,IAAI;EAG3F,IAAI,CAAC,SAAS,EADZ,OAAO,WAAW,YAAY,OAAO,UAAU,MAAM,KAAK,UAAU,OAAO,UAAU,MAErF,MAAM,aACJ,uFAAuF,KAAK,UAAU,MAAM,GAC9G;EAEF,MAAM,OACJ,OAAO,WAAW,KAAA,MACjB,QAAQ,KAAK,MAAM,OAAO,SAAS,GAAG,MAAM,QAAQ,OAAO,WAAW;EACzE,OAAO;GACL;GACA,eACE,OACI,YAAY,UAAU,MAAM,EAAE,iBAAiB,WAC/C,OAAO,WAAW,KAAA,IAChB,YAAY,UAAU,MAAM,EAAE,aAAa,OAAO,0BAClD,YAAY,UAAU,MAAM,EAAE,aAAa,OAAO,QAAQ,OAAO;EAC3E;CACF,CAAC;CAED,oBAAoB,SAAS,QAAQ,MAAkB;EACrD,IAAI,CAAC,UAAU,MAAM,GAAG,MAAM,aAAa,mDAAmD;EAC9F,MAAM,OAAO,OAAO,OAAO,WAAW;EACtC,OAAO;GACL;GACA,eACE,OACI,YAAY,UAAU,MAAM,EAAE,wBAC9B,8BAA8B,UAAU,MAAM,EAAE,QAAQ,OAAO,OAAO,OAAO,QAAQ,YAAY,EAAE,IAAI,YAAY,MAAM;EACjI;CACF,CAAC;CAED,QAAQ,SAAS,QAAQ,GAAO,EAAE,UAAkB;EAClD,MAAM,UAAU;EAChB,IAAI,SAAS,YAAY,WAAW,OAAO,QAAQ,UAAU,UAE3D,OAAO;GACL,MAAM;GACN,eACE,qBAAqB,QAAQ,SAAS,KAAK,QAAQ,MAAM,KAAK,QAAQ;EAC1E;EAEF,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,aAAa,sCAAsC;EAC3F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF,CAAC;AACH;;AAKA,MAAM,mCAAmB,IAAI,IAAiB;CAC5C;CACA;CACA;CACA;CACA;AACF,CAAC;AAKD,MAAa,iBAAiB,SAAsC,OAAO,OAAO,UAAU,IAAI;;AAGhG,MAAa,cACX,MACA,QACA,MACA,UAEC,SAAS,KAAK,CAA8C,QAAQ,MAAM,KAAK;;AASlF,MAAa,WAAW,OAAO,YAC5B,OAAO,KAAK,QAAQ,CAAC,CAAmB,KAAK,SAAS,CACrD,MACA,SAAiD,QAAiB,GAAG,MAAiB;CACpF,OAAO,WAAW,MAAM,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC;AAC9D,CACF,CAAC,CACH;;AAGA,MAAa,eAAe,OAAO,YAChC,OAAO,KAAK,QAAQ,CAAC,CACnB,QAAQ,SAAS,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC,CAC7C,KAAK,SAAS,CAAC,MAAM,SAAS,KAAK,CAAC,CACzC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { a as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, r as OPENROUTER_JUDGE_MODEL, u as judgeRoute } from "./judge-
|
|
2
|
-
import { u as vertexDecisionModel, v as vertexSettings } from "./vertex-
|
|
3
|
-
import { i as debugTrace } from "./attempt-
|
|
4
|
-
import { n as openRouterDecisionModel, t as openRouterBilling } from "./openrouter-models-
|
|
1
|
+
import { a as RECOMMENDED_JUDGE_MODEL, n as DIRECT_PROVIDERS, r as OPENROUTER_JUDGE_MODEL, u as judgeRoute } from "./judge-DYOF69Oe.js";
|
|
2
|
+
import { u as vertexDecisionModel, v as vertexSettings } from "./vertex-BLQMayc1.js";
|
|
3
|
+
import { i as debugTrace } from "./attempt-Dp7Wtm8M.js";
|
|
4
|
+
import { n as openRouterDecisionModel, t as openRouterBilling } from "./openrouter-models-DNvLRYM7.js";
|
|
5
5
|
import { experimental_decide } from "ai";
|
|
6
6
|
// src/judge/model.ts
|
|
7
7
|
/**
|
|
@@ -73,4 +73,4 @@ async function scoreWithModel(model, question, state, providerOptions, signal, g
|
|
|
73
73
|
}
|
|
74
74
|
export { scoreWithModel };
|
|
75
75
|
|
|
76
|
-
//# sourceMappingURL=model-
|
|
76
|
+
//# sourceMappingURL=model-1L19Zukh.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-
|
|
1
|
+
{"version":3,"file":"model-1L19Zukh.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;;;;;;AAmBA,eAAe,aACb,OACA,SACA,WACA,QACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,eAAe,OAAO;CACnE,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,IAAI,MAAM,SAAS,cAAc,OAAO,wBAAwB,OAAO,SAAS;CAChF,IAAI,MAAM,SAAS,UAAU,OAAO,oBAAoB,OAAO,UAAU,eAAe,CAAC;CACzF,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,QAAQ,MAAM,UAAd;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,cAAc,EAAE;EACvE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,SACE,QAAQ,MAAM,OAAO,uBAAA,CAAwB,WAAW,cAAc,EAAE;CAC5E;AACF;;AAGA,MAAM,wBAAwB;;;;;;AAS9B,SAAS,kBACP,OACA,SACA,OACoB;CACpB,MAAM,IAAI;CACV,MAAM,WAAW,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CAClD,IACE,YAAY,iBACX,wBAAwB,KAAK,GAAG,WAAW,EAAE,KAAK,GAAG,eAAe,MAErE,OAAO,oBAAoB,MAAM,qFAAqF,EAAE,QAAQ,oDAAoD,uBAAuB;CAE7M,IAAI,YAAY,YAAY,4CAA4C,KAAK,GAAG,WAAW,EAAE,GAAG;EAC9F,MAAM,SAAS,iBAAiB;EAChC,MAAM,UACJ,aAAa,WACT,gBAAgB,sBAAsB,4DAA4D,QAAQ,QAC1G,SACE,iFAAiF,OAAO,QACxF;EACR,OAAO,oBAAoB,MAAM,4HAA4H,wBAAwB,eAAe,sBAAsB,aAAa,QAAQ;CACjP;CACA,MAAM,KAAK,MAAM,MAAM,SAAS,SAAS,CAAC;CAC1C,KACG,WAAW,cAAc,YAC1B,aAAa,YACb,OAAO,yBACP,GAAG,MAAM,OAAO,SAAS,mBAEzB,OAAO,oBAAoB,MAAM,sGAAsG,sBAAsB,kBAAkB,EAAE,QAAQ,iBAAiB,sBAAsB;AAGpO;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QACA,SACA,YAAY,OACZ,QAMC;CACD,MAAM,WAAW,MAAM,aAAa,OAAO,SAAS,WAAW,MAAM;CACrE,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO;EACP;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC,CAAC,CAAC,OAAO,UAAmB;EAC3B,MAAM,UAAU,OAAO,UAAU,YAAY,kBAAkB,OAAO,SAAS,KAAK;EACpF,MAAM,UAAU,IAAI,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC,IAAI;CACzD,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EACL,OAAO,OAAO,QAAQ,OAAO;EAC7B,eAAe,OAAO,UAAU;EAChC,GAAI,YAAY,eAAe,kBAAkB,gBAAgB,IAAI,CAAC;CACxE;AACF"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import { b as resolveModelGateway, n as analysisModelOf, v as gatewayScope } from "./config-
|
|
3
|
-
import { s as isFakeJudge, u as judgeRoute } from "./judge-
|
|
4
|
-
import { p as vertexLanguageModel, v as vertexSettings, y as vertexSettingsIn } from "./vertex-
|
|
5
|
-
import { i as openRouterLanguageModel, r as openRouterFallbacks } from "./openrouter-models-
|
|
2
|
+
import { b as resolveModelGateway, n as analysisModelOf, v as gatewayScope } from "./config-C2UtS9vy.js";
|
|
3
|
+
import { s as isFakeJudge, u as judgeRoute } from "./judge-DYOF69Oe.js";
|
|
4
|
+
import { p as vertexLanguageModel, v as vertexSettings, y as vertexSettingsIn } from "./vertex-BLQMayc1.js";
|
|
5
|
+
import { i as openRouterLanguageModel, r as openRouterFallbacks } from "./openrouter-models-DNvLRYM7.js";
|
|
6
6
|
import { z } from "zod";
|
|
7
|
-
import { Output, generateText } from "ai";
|
|
7
|
+
import { NoObjectGeneratedError, NoOutputGeneratedError, Output, generateText } from "ai";
|
|
8
8
|
// src/analysis/schema.ts
|
|
9
9
|
const confidence = z.enum([
|
|
10
10
|
"high",
|
|
@@ -245,21 +245,45 @@ function hostModel(name, gateway, what, { fallbacks = false, vertex } = {}) {
|
|
|
245
245
|
...route === "vertex" && vertex ? { vertex } : {}
|
|
246
246
|
});
|
|
247
247
|
}
|
|
248
|
-
|
|
248
|
+
/**
|
|
249
|
+
* A provider refusing `maxOutputTokens` above the model's own limit, e.g. "max_tokens is too large",
|
|
250
|
+
* "The maximum tokens you requested exceeds the model limit of 8192".
|
|
251
|
+
*/
|
|
252
|
+
const OVER_CAP = /\bmax(?:imum)?[ _](?:output[ _])?tokens\b/i;
|
|
253
|
+
/** The model stopped at `maxOutputTokens` before its analysis was whole; names the option in `file`. */
|
|
254
|
+
function cutOff(model, maxOutputTokens, file, usage) {
|
|
255
|
+
const reasoning = usage?.outputTokenDetails?.reasoningTokens;
|
|
256
|
+
return new UsageError(`The analysis model, ${model.name}, stopped at analysis.maxOutputTokens (${maxOutputTokens} tokens${reasoning ? `, ${reasoning} of them reasoning` : ""}) before its analysis was complete. Raise analysis: { maxOutputTokens: ${maxOutputTokens * 2} } in ${file}.`);
|
|
257
|
+
}
|
|
258
|
+
async function callAnalysisModel(model, digest, prompt, maxOutputTokens, file) {
|
|
249
259
|
const started = Date.now();
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
260
|
+
let result;
|
|
261
|
+
try {
|
|
262
|
+
result = await generateText({
|
|
263
|
+
model: await languageModel(model, digest),
|
|
264
|
+
system: SYSTEM_PROMPT,
|
|
265
|
+
prompt,
|
|
266
|
+
output: Output.object({
|
|
267
|
+
schema: analysisSchema,
|
|
268
|
+
name: "analysis"
|
|
269
|
+
}),
|
|
270
|
+
maxOutputTokens,
|
|
271
|
+
maxRetries: 3
|
|
272
|
+
});
|
|
273
|
+
} catch (error) {
|
|
274
|
+
if (NoObjectGeneratedError.isInstance(error) && error.finishReason === "length") throw cutOff(model, maxOutputTokens, file, error.usage);
|
|
275
|
+
if (error instanceof Error && OVER_CAP.test(error.message)) throw new UsageError(`${error.message} The analysis model, ${model.name}, was asked for up to ${maxOutputTokens} output tokens: set analysis: { maxOutputTokens } in ${file} to at most its output limit.`);
|
|
276
|
+
throw error;
|
|
277
|
+
}
|
|
278
|
+
let output;
|
|
279
|
+
try {
|
|
280
|
+
output = result.output;
|
|
281
|
+
} catch (error) {
|
|
282
|
+
if (NoOutputGeneratedError.isInstance(error) && result.finishReason === "length") throw cutOff(model, maxOutputTokens, file, result.usage);
|
|
283
|
+
throw error;
|
|
284
|
+
}
|
|
261
285
|
return {
|
|
262
|
-
output
|
|
286
|
+
output,
|
|
263
287
|
resolvedModel: result.response?.modelId ?? null,
|
|
264
288
|
usage: {
|
|
265
289
|
inputTokens: result.usage?.inputTokens ?? null,
|
|
@@ -270,4 +294,4 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
|
270
294
|
}
|
|
271
295
|
export { isFake as a, questionModelLabel as c, hostModel as i, resolveAnalysisModel as l, callAnalysisModel as n, providerModel as o, cannedModel as r, questionModel as s, analysisModelName as t, SYSTEM_PROMPT as u };
|
|
272
296
|
|
|
273
|
-
//# sourceMappingURL=model-
|
|
297
|
+
//# sourceMappingURL=model-BxBWFOZn.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-
|
|
1
|
+
{"version":3,"file":"model-BxBWFOZn.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,uFAAuF;AAEnG,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EACH,OAAO,CAAC,CACR,SACC,2HACF;CACF,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EACN,MAAM,KAAK,CAAC,CACZ,SACC,uLACF;CACF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAc;GAAe;GAAS;GAAS;EAAO,CAAC;EACrE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAU;EAAS,CAAC;EACpE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SACC,sIACF;EACF,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;AC5B7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;;;;;AAO5E,SAAS,cACP,QACA,QAAgC,OAAO,MAAM,OACL;CACxC,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,IAAI,QAAQ,KAAA;AAC1E;;AAGA,SAAS,WAAW,OAA+B,MAAsB;CACvE,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAGA,SAAgB,kBAAkB,QAAoD;CACpF,MAAM,QAAQ,cAAc,MAAM;CAClC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,gGAAgG,OAAO,OAAO,WAAW,OAAO,MAAM,OAAO,mBAAmB,EAAE,+EACpK;AACF;;;;;AAMA,SAAgB,mBACd,QACA,QAAgC,OAAO,MAAM,OAC9B;CACf,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OACA,QAAgC,OAAO,MAAM,OACjB;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,OAAO,SAAS,WAAW,KAAA,IAAY,8BAA8B;CAC3E,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,OAAO,IAAI,EAAE,OAAO,IAAI,EAC7L;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,2CAA2C,OAAO,KAAK,KAAK,MAAM,+DAA+D,KAAK,6DAA6D,IAAI,EACpP;CAEF,OAAO;AACT;;AAGA,SAAgB,qBAAqB,QAA+C;CAClF,MAAM,QAAQ,kBAAkB,MAAM;CACtC,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,qBAAqB,OAAO,KAAK,OAAO,MAAM,sIAChD;CAEF,MAAM,QAAQ,aAAa,MAAM;CACjC,MAAM,UAAU,oBAAoB,OAAO,KAAA,GAAW,OAAO,gBAAgB;CAC7E,IAAI;EACF,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,kBAAkB,OAAO,CAAC,CAAC;EACxE,OAAO;GACL,MAAM;GACN;GACA,GAAI,UAAU,eAAe,EAAE,WAAW,oBAAoB,MAAM,QAAQ,EAAE,IAAI,CAAC;GACnF,GAAI,UAAU,WAAW,EAAE,QAAQ,iBAAiB,MAAM,QAAQ,EAAE,IAAI,CAAC;EAC3E;CACF,SAAS,OAAO;EACd,MAAM,IAAI,WACR,GAAI,MAAgB,QAAQ,sDAC9B;CACF;AACF;;;;;;AAOA,SAAgB,YACd,QACA,cAAc,OACC;CACf,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAY;GAC7B,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,MAAM,QAAQ,cAAc,KAAK,KAAK,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI;GACnF,IAAI,SAAS,cAAc,KAAK,KAAK,OAAO,KAAK,SAAS,CAAC,IAAI;GAC/D,IAAI,OAAO,OAAO;GAElB,MAAM,MACJ,eAAe,QAAQ,oBAAoB,KAAA,KAAa,SAAS,QAAQ;GAC3E,IAAI,KAAK;IACP,SAAS,QAAQ;IACjB,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;GACjC;GACA,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ;IAAK,CAAC;IAChC,cAAc,MACV;KAAE,SAAS;KAAU,KAAK;IAAa,IACvC;KAAE,SAAS;KAAQ,KAAK;IAAO;IACnC,OAAO;KACL,aAAa;MAAE,OAAO;MAAO,SAAS;MAAO,WAAW;MAAG,YAAY;KAAE;KACzE,cAAc;MAAE,OAAO;MAAQ,MAAM;MAAQ,WAAW;KAAE;IAC5D;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;;;;AAMA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,IAAI,MAAM,UAAU,cAAc,OAAO,wBAAwB,MAAM,MAAM,CAAC,CAAC,MAAM,SAAS;CAC9F,IAAI,MAAM,UAAU,UAClB,OAAO,oBAAoB,MAAM,MAAM,MAAM,UAAU,eAAe,CAAC;CAEzE,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,QAAQ,UAAR;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,EAAE;EACzD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,SACE,OAAO,MAAM;CACjB;AACF;;AAMA,SAAgB,UACd,MACA,SACA,MACA,EAAE,YAAY,OAAO,WAA8B,CAAC,GAC5B;CACxB,MAAM,QAAQ,WAAW,MAAM,QAAQ,KAAK,MAAM,OAAO,CAAC,CAAC;CAC3D,OAAO,cAAc;EACnB;EACA;EACA,GAAI,UAAU,eAAe,EAAE,UAAU,IAAI,CAAC;EAC9C,GAAI,UAAU,YAAY,SAAS,EAAE,OAAO,IAAI,CAAC;CACnD,CAAC;AACH;;;;;AAaA,MAAM,WAAW;;AAGjB,SAAS,OACP,OACA,iBACA,MACA,OACY;CACZ,MAAM,YAAY,OAAO,oBAAoB;CAC7C,OAAO,IAAI,WACT,uBAAuB,MAAM,KAAK,yCAAyC,gBAAgB,SAAS,YAAY,KAAK,UAAU,sBAAsB,GAAG,yEAAyE,kBAAkB,EAAE,QAAQ,KAAK,EACpQ;AACF;AAEA,eAAsB,kBACpB,OACA,QACA,QACA,iBACA,MACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,aAAa;GAC1B,OAAO,MAAM,cAAc,OAAO,MAAM;GACxC,QAAQ;GACR;GACA,QAAQ,OAAO,OAAO;IAAE,QAAQ;IAAgB,MAAM;GAAW,CAAC;GAClE;GACA,YAAY;EACd,CAAC;CACH,SAAS,OAAO;EACd,IAAI,uBAAuB,WAAW,KAAK,KAAK,MAAM,iBAAiB,UACrE,MAAM,OAAO,OAAO,iBAAiB,MAAM,MAAM,KAAK;EAExD,IAAI,iBAAiB,SAAS,SAAS,KAAK,MAAM,OAAO,GACvD,MAAM,IAAI,WACR,GAAG,MAAM,QAAQ,uBAAuB,MAAM,KAAK,wBAAwB,gBAAgB,uDAAuD,KAAK,8BACzJ;EAEF,MAAM;CACR;CACA,IAAI;CACJ,IAAI;EACF,SAAS,OAAO;CAClB,SAAS,OAAO;EAEd,IAAI,uBAAuB,WAAW,KAAK,KAAK,OAAO,iBAAiB,UACtE,MAAM,OAAO,OAAO,iBAAiB,MAAM,OAAO,KAAK;EAEzD,MAAM;CACR;CACA,OAAO;EACL;EACA,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { t as analysisModelName, n as callAnalysisModel, r as cannedModel, i as hostModel, a as isFake, o as providerModel, s as questionModel, c as questionModelLabel, l as resolveAnalysisModel } from "./model-
|
|
1
|
+
export { t as analysisModelName, n as callAnalysisModel, r as cannedModel, i as hostModel, a as isFake, o as providerModel, s as questionModel, c as questionModelLabel, l as resolveAnalysisModel } from "./model-BxBWFOZn.js";
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
|
|
2
|
-
import { a as plural
|
|
3
|
-
import { a as rescore, h as sampleReasons, n as gateLabel } from "./gate-
|
|
4
|
-
import {
|
|
5
|
-
import { a as totalTokens,
|
|
2
|
+
import { a as plural } from "./text-B8-W3Cvn.js";
|
|
3
|
+
import { a as rescore, h as sampleReasons, n as gateLabel } from "./gate-Cp_pQoEe.js";
|
|
4
|
+
import { l as median, o as mannWhitney, p as quantile, u as minimumP } from "./stats-C18W4xPX.js";
|
|
5
|
+
import { a as totalTokens, n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
|
|
6
|
+
import { m as uniqueJudgeLabels, n as cellUsage, t as cellJudgeRates } from "./judgments-m-mb0zv8.js";
|
|
6
7
|
import { u as readableTranscriptLine } from "./record-DhzwkSXc.js";
|
|
7
8
|
// src/view/compare.ts
|
|
8
9
|
const STACK_FIELDS = [
|
|
@@ -493,7 +494,7 @@ const effortSample = (a) => ({
|
|
|
493
494
|
});
|
|
494
495
|
/** The files the workspace diff changed, a collapsed directory counting each of its files. */
|
|
495
496
|
const filesChanged = (files) => files.reduce((n, f) => n + (f.count ?? 1), 0);
|
|
496
|
-
const counted
|
|
497
|
+
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
497
498
|
const spread = (values) => ({
|
|
498
499
|
median: median(values) ?? 0,
|
|
499
500
|
q1: quantile(values, .25) ?? 0,
|
|
@@ -592,7 +593,7 @@ function contrastEffort(variant, control) {
|
|
|
592
593
|
};
|
|
593
594
|
}
|
|
594
595
|
/** The counted attempts of one cell of an eval, as effort samples. */
|
|
595
|
-
const cellSamples = (e, cellKey) => e.attempts.filter((a) => a.cell === cellKey && counted
|
|
596
|
+
const cellSamples = (e, cellKey) => e.attempts.filter((a) => a.cell === cellKey && counted(a)).map(effortSample);
|
|
596
597
|
const cellEffort = (e, cellKey) => effortOf(cellSamples(e, cellKey));
|
|
597
598
|
/** A variant cell's effort against its control's, in the same run; null for controls. */
|
|
598
599
|
function cellContrast(e, cellKey) {
|
|
@@ -659,246 +660,6 @@ function effortChangeLabel(base, head) {
|
|
|
659
660
|
const tok = ratio(head.tokens.median, base.tokens.median);
|
|
660
661
|
return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
|
|
661
662
|
}
|
|
662
|
-
// src/view/judgments.ts
|
|
663
|
-
/** Show each key until this many; then the minimum score plus a count. */
|
|
664
|
-
const COLLAPSE_AFTER = 2;
|
|
665
|
-
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
666
|
-
/** Short names stay at this many characters; a collision takes the distinctive tail instead. */
|
|
667
|
-
const LABEL_MAX = 20;
|
|
668
|
-
const judgeText = (question) => question.replace(/\?+$/, "").trim();
|
|
669
|
-
/** Last `max` characters, cut at a word boundary where there is one, marked with a leading `…`. */
|
|
670
|
-
function clipTail(text, max) {
|
|
671
|
-
if (text.length <= max) return text;
|
|
672
|
-
const cut = text.slice(-(max - 1));
|
|
673
|
-
const space = cut.search(/\s/);
|
|
674
|
-
return `…${(space >= 0 && space < max / 2 ? cut.slice(space + 1) : cut).trimStart()}`;
|
|
675
|
-
}
|
|
676
|
-
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
677
|
-
function judgeLabel(question) {
|
|
678
|
-
return clip(judgeText(question), LABEL_MAX);
|
|
679
|
-
}
|
|
680
|
-
/**
|
|
681
|
-
* One short label per key. Shared prefixes take the distinctive tail so
|
|
682
|
-
* `Does this change…` does not collapse several checks; a tail that still
|
|
683
|
-
* collides falls back to the key.
|
|
684
|
-
*/
|
|
685
|
-
function uniqueJudgeLabels(items) {
|
|
686
|
-
const first = /* @__PURE__ */ new Map();
|
|
687
|
-
for (const item of items) if (!first.has(item.key)) first.set(item.key, item);
|
|
688
|
-
const labels = /* @__PURE__ */ new Map();
|
|
689
|
-
const used = /* @__PURE__ */ new Map();
|
|
690
|
-
const setLabel = (key, label) => {
|
|
691
|
-
const prev = labels.get(key);
|
|
692
|
-
if (prev !== void 0) {
|
|
693
|
-
const group = used.get(prev);
|
|
694
|
-
if (group) {
|
|
695
|
-
const next = group.filter((k) => k !== key);
|
|
696
|
-
if (next.length) used.set(prev, next);
|
|
697
|
-
else used.delete(prev);
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
labels.set(key, label);
|
|
701
|
-
const group = used.get(label) ?? [];
|
|
702
|
-
group.push(key);
|
|
703
|
-
used.set(label, group);
|
|
704
|
-
};
|
|
705
|
-
for (const row of first.values()) setLabel(row.key, judgeLabel(row.question));
|
|
706
|
-
for (const [label, keys] of [...used]) {
|
|
707
|
-
if (keys.length < 2) continue;
|
|
708
|
-
const tails = /* @__PURE__ */ new Map();
|
|
709
|
-
for (const key of keys) {
|
|
710
|
-
const row = first.get(key);
|
|
711
|
-
if (!row) continue;
|
|
712
|
-
const tail = clipTail(judgeText(row.question), LABEL_MAX);
|
|
713
|
-
const group = tails.get(tail) ?? [];
|
|
714
|
-
group.push(key);
|
|
715
|
-
tails.set(tail, group);
|
|
716
|
-
}
|
|
717
|
-
for (const [tail, tkeys] of tails) {
|
|
718
|
-
const [only] = tkeys;
|
|
719
|
-
const taken = (used.get(tail) ?? []).some((k) => !tkeys.includes(k));
|
|
720
|
-
if (only && tkeys.length === 1 && tail !== label && !taken) setLabel(only, tail);
|
|
721
|
-
else for (const key of tkeys) setLabel(key, key);
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
for (const [, keys] of [...used]) {
|
|
725
|
-
if (keys.length < 2) continue;
|
|
726
|
-
for (const key of keys) setLabel(key, key);
|
|
727
|
-
}
|
|
728
|
-
return labels;
|
|
729
|
-
}
|
|
730
|
-
function keyTallies(e, cellKey) {
|
|
731
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
732
|
-
const expectLabels = /* @__PURE__ */ new Map();
|
|
733
|
-
for (const a of e.attempts) {
|
|
734
|
-
if (a.cell !== cellKey || !counted(a)) continue;
|
|
735
|
-
for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
|
|
736
|
-
const row = byKey.get(j.key) ?? {
|
|
737
|
-
label: "",
|
|
738
|
-
labelled: false,
|
|
739
|
-
question: j.question,
|
|
740
|
-
pass: 0,
|
|
741
|
-
counted: 0
|
|
742
|
-
};
|
|
743
|
-
row.counted++;
|
|
744
|
-
if (j.verdict === "pass") row.pass++;
|
|
745
|
-
byKey.set(j.key, row);
|
|
746
|
-
if (labeled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
|
|
750
|
-
key,
|
|
751
|
-
question: row.question
|
|
752
|
-
})));
|
|
753
|
-
for (const [key, row] of byKey) {
|
|
754
|
-
const label = expectLabels.get(key);
|
|
755
|
-
row.labelled = label !== void 0;
|
|
756
|
-
row.label = label ?? labels.get(key) ?? judgeLabel(row.question);
|
|
757
|
-
}
|
|
758
|
-
return byKey;
|
|
759
|
-
}
|
|
760
|
-
function pairedControl(e, cellKey) {
|
|
761
|
-
const cell = e.cells.find((c) => c.key === cellKey);
|
|
762
|
-
if (cell?.role !== "variant") return void 0;
|
|
763
|
-
return e.cells.find((x) => x.role === "control" && x.harness === cell.harness && x.sandbox === cell.sandbox);
|
|
764
|
-
}
|
|
765
|
-
function liftForKey(variant, control) {
|
|
766
|
-
if (!control || variant.counted < 4 || control.counted < 4) return null;
|
|
767
|
-
const { points, low, high } = liftInterval({
|
|
768
|
-
pass: variant.pass,
|
|
769
|
-
n: variant.counted
|
|
770
|
-
}, {
|
|
771
|
-
pass: control.pass,
|
|
772
|
-
n: control.counted
|
|
773
|
-
});
|
|
774
|
-
return {
|
|
775
|
-
points: round3(points),
|
|
776
|
-
control: {
|
|
777
|
-
pass: control.pass,
|
|
778
|
-
counted: control.counted
|
|
779
|
-
},
|
|
780
|
-
variant: {
|
|
781
|
-
pass: variant.pass,
|
|
782
|
-
counted: variant.counted
|
|
783
|
-
},
|
|
784
|
-
withinNoise: containsZero(low, high)
|
|
785
|
-
};
|
|
786
|
-
}
|
|
787
|
-
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
|
|
788
|
-
function cellJudgeRates(e, cellKey) {
|
|
789
|
-
const tallies = keyTallies(e, cellKey);
|
|
790
|
-
const control = pairedControl(e, cellKey);
|
|
791
|
-
const controlTallies = control ? keyTallies(e, control.key) : void 0;
|
|
792
|
-
return [...tallies].map(([key, row]) => ({
|
|
793
|
-
key,
|
|
794
|
-
...row,
|
|
795
|
-
lift: liftForKey(row, controlTallies?.get(key))
|
|
796
|
-
}));
|
|
797
|
-
}
|
|
798
|
-
/**
|
|
799
|
-
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
800
|
-
* of the cell's checks against its control is already there.
|
|
801
|
-
*/
|
|
802
|
-
function summaryJudgeRates(e, cell) {
|
|
803
|
-
const checked = new Set((cell.checks ?? []).filter((k) => k.control && k.verdict).map((k) => k.label));
|
|
804
|
-
return cellJudgeRates(e, cell.key).filter((r) => !(r.labelled && checked.has(r.label)));
|
|
805
|
-
}
|
|
806
|
-
const signedPoints = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
|
|
807
|
-
/** `login 4/5 · checkout 0/5`, with `(+80 pts)` on a key that has lift. */
|
|
808
|
-
function judgeRatesLabel(rates) {
|
|
809
|
-
return rates.map((r) => {
|
|
810
|
-
const n = `${r.label} ${r.pass}/${r.counted}`;
|
|
811
|
-
return r.lift ? `${n} (${signedPoints(r.lift.points)} pts)` : n;
|
|
812
|
-
}).join(" · ");
|
|
813
|
-
}
|
|
814
|
-
/** Each key, or the lowest score plus how many judges there were. */
|
|
815
|
-
function collapsedJudges(judgments) {
|
|
816
|
-
const labels = uniqueJudgeLabels(judgments);
|
|
817
|
-
const items = judgments.map((j) => ({
|
|
818
|
-
verdict: j.verdict,
|
|
819
|
-
score: j.score,
|
|
820
|
-
threshold: j.threshold,
|
|
821
|
-
label: labels.get(j.key) ?? judgeLabel(j.question),
|
|
822
|
-
key: j.key
|
|
823
|
-
}));
|
|
824
|
-
if (items.length <= COLLAPSE_AFTER) return {
|
|
825
|
-
shown: items,
|
|
826
|
-
extra: 0
|
|
827
|
-
};
|
|
828
|
-
return {
|
|
829
|
-
shown: [items.reduce((a, b) => a.score <= b.score ? a : b)],
|
|
830
|
-
extra: items.length
|
|
831
|
-
};
|
|
832
|
-
}
|
|
833
|
-
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
834
|
-
function collapsedJudgeLabel(judgments) {
|
|
835
|
-
if (!judgments.length) return null;
|
|
836
|
-
const { shown, extra } = collapsedJudges(judgments);
|
|
837
|
-
const parts = shown.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold.toFixed(2)}`);
|
|
838
|
-
if (extra) parts.push(`${extra} judges`);
|
|
839
|
-
return parts.join(" · ");
|
|
840
|
-
}
|
|
841
|
-
const JUDGE_ERROR = /^judge failed for /;
|
|
842
|
-
/**
|
|
843
|
-
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
844
|
-
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
845
|
-
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
846
|
-
* reaches is named by its question.
|
|
847
|
-
*/
|
|
848
|
-
function labeledJudgments(a) {
|
|
849
|
-
const passes = (a.assertions ?? []).filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
|
|
850
|
-
const zip = passes.length === a.judgements.length;
|
|
851
|
-
return a.judgements.map((judgment, i) => {
|
|
852
|
-
const x = zip ? passes[i] : void 0;
|
|
853
|
-
return {
|
|
854
|
-
judgment,
|
|
855
|
-
label: x?.labelled ? x.label : judgeText(judgment.question),
|
|
856
|
-
labelled: x?.labelled === true,
|
|
857
|
-
...x ? { check: x.label } : {}
|
|
858
|
-
};
|
|
859
|
-
});
|
|
860
|
-
}
|
|
861
|
-
/** Sum of counted attempts' usage; null when none of them reported any. */
|
|
862
|
-
function cellUsage(e, cellKey) {
|
|
863
|
-
return sumUsage(e.attempts.filter((a) => a.cell === cellKey && counted(a)).map((a) => a.usage));
|
|
864
|
-
}
|
|
865
|
-
/**
|
|
866
|
-
* Each `c.exec` check's `expect` label, by position: the attempt's `toSucceed` assertions in call
|
|
867
|
-
* order, when they are exactly as many as its checks; undefined where no label reaches.
|
|
868
|
-
*/
|
|
869
|
-
function checkLabels(a) {
|
|
870
|
-
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
871
|
-
const zip = succeeds.length === a.checks.length;
|
|
872
|
-
return a.checks.map((_, i) => {
|
|
873
|
-
const x = zip ? succeeds[i] : void 0;
|
|
874
|
-
return x?.labelled ? x.label : void 0;
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
/**
|
|
878
|
-
* Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
|
|
879
|
-
* are exactly as many as its checks; empty otherwise.
|
|
880
|
-
*/
|
|
881
|
-
function checksByLabel(a) {
|
|
882
|
-
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
883
|
-
if (succeeds.length !== a.checks.length) return /* @__PURE__ */ new Map();
|
|
884
|
-
return new Map(succeeds.map((x, i) => [x.label, a.checks[i]]));
|
|
885
|
-
}
|
|
886
|
-
/**
|
|
887
|
-
* How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
|
|
888
|
-
* when it had none or passed on its first try.
|
|
889
|
-
*/
|
|
890
|
-
function triesNote(exitCode, tried, of) {
|
|
891
|
-
if (of <= 1 || exitCode === 0 && tried <= 1) return null;
|
|
892
|
-
if (exitCode === 0) return `passed on try ${tried} of ${of}`;
|
|
893
|
-
return tried >= of ? `failed all ${of} tries` : `failed on try ${tried} of ${of}`;
|
|
894
|
-
}
|
|
895
|
-
/** {@link triesNote} for a recorded check. */
|
|
896
|
-
const checkTries = (c) => c.retries && c.tries ? triesNote(c.exitCode, c.tries.length, c.retries + 1) : null;
|
|
897
|
-
/** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
|
|
898
|
-
function commandLine(command, max = 80) {
|
|
899
|
-
const lines = command.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
900
|
-
return clip(lines.length > 1 ? `${lines[0]} …` : lines[0] ?? "", max);
|
|
901
|
-
}
|
|
902
663
|
// src/view/model.ts
|
|
903
664
|
/** Splits `file › group › … › name` back into its parts. */
|
|
904
665
|
function splitEvalId(id) {
|
|
@@ -1006,19 +767,17 @@ const shortSha = (sha) => sha ? sha.slice(0, 7) : "no git";
|
|
|
1006
767
|
/** Reasoning time, or `n/a` when the harness couldn't measure it: never `0s` for unknown. */
|
|
1007
768
|
const reasoningLabel = (ms) => ms === null ? "n/a" : formatDuration(ms);
|
|
1008
769
|
/**
|
|
1009
|
-
* An attempt's effort,
|
|
1010
|
-
*
|
|
1011
|
-
* With the workspace diff's count, `files changed` follows: an edit made from the shell is in it,
|
|
1012
|
-
* never in the file edits.
|
|
770
|
+
* An attempt's effort: tool calls, then the files the workspace diff changed, which counts a file
|
|
771
|
+
* the shell wrote, then the calls to the agent's edit tools, which doesn't.
|
|
1013
772
|
*/
|
|
1014
773
|
function activityLabel(a, changed) {
|
|
1015
|
-
|
|
1016
|
-
return `${plural(a.tools, "tool call")} · ${plural(a.fileEdits, "file edit")}${files}`;
|
|
774
|
+
return `${plural(a.tools, "tool call")} · ${plural(changed, "file")} changed · ${plural(a.fileEdits, "edit tool call")}`;
|
|
1017
775
|
}
|
|
1018
776
|
/**
|
|
1019
777
|
* A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
|
|
1020
778
|
* failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
|
|
1021
|
-
* default) don't. An interrupted run is `skipped`, whatever ran
|
|
779
|
+
* default) don't. An interrupted run is `skipped`, whatever ran; a resume its filters left
|
|
780
|
+
* `interrupted` exits on its outcome as if it were complete.
|
|
1022
781
|
*/
|
|
1023
782
|
function runOutcome(run) {
|
|
1024
783
|
if (run.status === "interrupted") return "skipped";
|
|
@@ -1028,6 +787,11 @@ function runOutcome(run) {
|
|
|
1028
787
|
if (outcomes.includes("infra_error")) return "infra_error";
|
|
1029
788
|
return "pass";
|
|
1030
789
|
}
|
|
790
|
+
/**
|
|
791
|
+
* The eval hasn't run yet: its run stopped (interrupted, or out of budget) or a filtered resume
|
|
792
|
+
* left it out, and it's `skipped` without being a todo or declared skipped. A resume runs it.
|
|
793
|
+
*/
|
|
794
|
+
const notRunYet = (run, e) => run.status !== "complete" && e.outcome === "skipped" && !e.todo && !e.skip;
|
|
1031
795
|
function andList(items) {
|
|
1032
796
|
if (items.length <= 1) return items[0] ?? "";
|
|
1033
797
|
if (items.length === 2) return `${items[0]} and ${items[1]}`;
|
|
@@ -1388,6 +1152,6 @@ const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
|
|
|
1388
1152
|
const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
|
|
1389
1153
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
1390
1154
|
const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
|
|
1391
|
-
export {
|
|
1155
|
+
export { formatCount as $, runRows as A, bothUnderATenth as B, isUngatedMiss as C, reasoningLabel as D, parseTranscript as E, DIRECTIONAL_MIN as F, directional as G, cellEffort as H, EFFORT_ALPHA as I, effortOf as J, effortChangeLabel as K, EFFORT_MIN_RUNS as L, shortSha as M, splitEvalId as N, runLead as O, transcriptItems as P, filesChanged as Q, TOKENS_UNREPORTED as R, isNotGatedPass as S, notRunYet as T, cellSamples as U, cellContrast as V, contrastEffort as W, effortSummary as X, effortSample as Y, effortVerdict as Z, formatUsage as _, buildIndex as a, compare as at, indexFiles as b, defaultAttempt as c, sourceChanges as ct, evalTree as d, formatRatio as et, failedJudgeLabels as f, formatPercent as g, formatDuration as h, attemptAnswerRows as i, cellsChanged as it, runtimeLabel as j, runOutcome as k, evalHistory as l, stackChanges as lt, findEval as m, activityLabel as n, tokenValues as nt, cellRows as o, newRows as ot, findAttempt as p, effortLabel as q, answerRows as r, addedRow as rt, compareView as s, rowKey as st, COMPARE_ORDER as t, formatSeconds as tt, evalMarkOutcome as u, whyChanged as ut, harnessLabel as v, languageLabel as w, isMatrix as x, hasUngatedCells as y, VERDICT_TEXT as z };
|
|
1392
1156
|
|
|
1393
|
-
//# sourceMappingURL=model-
|
|
1157
|
+
//# sourceMappingURL=model-Ce9l9_rF.js.map
|