reeval 0.17.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/{adapter-DyvDj4ZN.js → adapter-CptK032x.js} +2 -2
- package/dist/{adapter-DyvDj4ZN.js.map → adapter-CptK032x.js.map} +1 -1
- package/dist/{agent-v036GEgG.js → agent-B63OzUX9.js} +6 -6
- package/dist/{agent-v036GEgG.js.map → agent-B63OzUX9.js.map} +1 -1
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-sgqwfjtn.js} +4 -4
- package/dist/app-files-sgqwfjtn.js.map +1 -0
- package/dist/assemble-CxBHkavM.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-Me7Whp-y.js} +58 -194
- package/dist/assemble-Me7Whp-y.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
- package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-Cik0SoIn.js} +17 -9
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
- package/dist/{capture-Bdcx9Gap.js → capture-C3RmjPph.js} +392 -613
- package/dist/capture-C3RmjPph.js.map +1 -0
- package/dist/capture-YPwxp_iK.js +1 -0
- package/dist/{cli-CGd51thd.js → cli-BBOlXKPG.js} +1075 -457
- package/dist/cli-BBOlXKPG.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
- package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
- package/dist/client-tZ2WBDxf.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-CFQ9c1hk.js} +13 -13
- package/dist/{command-TLmY-rYp.js.map → command-CFQ9c1hk.js.map} +1 -1
- package/dist/{command-CtTdCF3B.js → command-DhiwsZl1.js} +9 -9
- package/dist/{command-CtTdCF3B.js.map → command-DhiwsZl1.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-fO0BFsTq.js} +102 -37
- package/dist/command-fO0BFsTq.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-qGS802uX.js} +16 -14
- package/dist/command-qGS802uX.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CC60TOT5.js} +64 -18
- package/dist/commands-CC60TOT5.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-xYTMNTPv.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-C2UtS9vy.js} +718 -76
- package/dist/config-C2UtS9vy.js.map +1 -0
- package/dist/{contrasts-DtSX8E_V.js → contrasts-BjOg7wyr.js} +3 -3
- package/dist/{contrasts-DtSX8E_V.js.map → contrasts-BjOg7wyr.js.map} +1 -1
- package/dist/cost-BJU0MAji.js +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BTK_apL-.js} +2 -2
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
- package/dist/{db-sHGpaLo9.js → db-C5SZ0510.js} +72 -14
- package/dist/db-C5SZ0510.js.map +1 -0
- package/dist/db-puzLzX4k.js +2 -0
- package/dist/{default-effort-BZRa-SQP.js → default-effort-BVmx7r7L.js} +8 -5
- package/dist/default-effort-BVmx7r7L.js.map +1 -0
- package/dist/{default-effort-fpdbYVKm.js → default-effort-FEidNAIi.js} +1 -1
- package/dist/{docker-CWuSRj7a.js → docker-B2F8-PGC.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-B2F8-PGC.js.map} +1 -1
- package/dist/{docker-CIzHzwlC.js → docker-BlSopHpZ.js} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-Dto8ROCq.js} +5 -5
- package/dist/{docker--2ytOHcu.js.map → docker-Dto8ROCq.js.map} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-BcvfKpBX.js} +20 -20
- package/dist/{doctor-eE65YpEv.js.map → doctor-BcvfKpBX.js.map} +1 -1
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-CC1RV21n.js} +3 -3
- package/dist/{environment-Cy3FVLR5.js.map → environment-CC1RV21n.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-CzxdgFBW.js} +18 -10
- package/dist/eval-source-CzxdgFBW.js.map +1 -0
- package/dist/evaluate-DToSB2bt.d.ts +101 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-NTNJ8T9H.js} +129 -51
- package/dist/evaluate-NTNJ8T9H.js.map +1 -0
- package/dist/{files-tnMgu7-X.js → files-CZXrkPja.js} +5 -5
- package/dist/{files-tnMgu7-X.js.map → files-CZXrkPja.js.map} +1 -1
- package/dist/files-DFtuRQJx.js +1 -0
- package/dist/{files-CQU08uh4.js → files-DsXULUi5.js} +3 -3
- package/dist/{files-CQU08uh4.js.map → files-DsXULUi5.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-aAzoGNCB.js} +4 -4
- package/dist/{files-CIX-pLl4.js.map → files-aAzoGNCB.js.map} +1 -1
- package/dist/files-oOYNR7ZH.js +1 -0
- package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
- package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-C1MB5bjr.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-C1MB5bjr.js.map} +1 -1
- package/dist/harness-DGsRZDSo.js +105 -0
- package/dist/harness-DGsRZDSo.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
- package/dist/harness-DYSC4zMh.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-BVZHDf_z.js → images-CNayXCp2.js} +8 -8
- package/dist/{images-BVZHDf_z.js.map → images-CNayXCp2.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5 -5
- package/dist/init-8o6W3_fn.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-D5HtmyMI.js} +5 -5
- package/dist/{init-BWnMbUnc.js.map → init-D5HtmyMI.js.map} +1 -1
- package/dist/ink-list-Ca-5dunO.js +184 -0
- package/dist/ink-list-Ca-5dunO.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-DrGXF9dT.js} +3 -3
- package/dist/{invocations-BGRs1r_C.js.map → invocations-DrGXF9dT.js.map} +1 -1
- package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
- package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-m-mb0zv8.js +246 -0
- package/dist/judgments-m-mb0zv8.js.map +1 -0
- package/dist/list-CM_YSJXq.js +374 -0
- package/dist/list-CM_YSJXq.js.map +1 -0
- package/dist/{live-d_5a5iUv.js → live-B71o2TkX.js} +21 -3
- package/dist/live-B71o2TkX.js.map +1 -0
- package/dist/live-DQc8jr6o.js +1 -0
- package/dist/{live-ColieOai.js → live-uRURFDMZ.js} +7 -7
- package/dist/{live-ColieOai.js.map → live-uRURFDMZ.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
- package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-BxBWFOZn.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-BxBWFOZn.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-CXVVB4RK.js} +1 -1
- package/dist/{model-DOiCiMCD.js → model-Ce9l9_rF.js} +19 -255
- package/dist/model-Ce9l9_rF.js.map +1 -0
- package/dist/model-Cxzo4cjw.js +1 -0
- package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
- package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
- package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
- package/dist/{patch-DvLr8wCZ.js → patch-ug5HpMOj.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-ug5HpMOj.js.map} +1 -1
- package/dist/{plan-CIfOFhth.js → plan-DqnzfDH1.js} +15 -14
- package/dist/plan-DqnzfDH1.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-B2KQ0y3P.js} +4 -4
- package/dist/{plugins-BEs2Am3L.js.map → plugins-B2KQ0y3P.js.map} +1 -1
- package/dist/{preflight-BAIRsTpl.js → preflight-DVXEvzFU.js} +1 -1
- package/dist/{preflight-50Qw5e5K.js → preflight-DyzmvK7i.js} +3 -3
- package/dist/{preflight-50Qw5e5K.js.map → preflight-DyzmvK7i.js.map} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-21WMQZyt.js} +7 -7
- package/dist/{prepare-8XhzhLHX.js.map → prepare-21WMQZyt.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-BcSRa7-3.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
- package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-iRxjp4Pu.js} +4 -4
- package/dist/{provenance-Cc8fFE7Z.js.map → provenance-iRxjp4Pu.js.map} +1 -1
- package/dist/{providers-CP_bBtYm.js → providers-D0ob_tra.js} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-vXMXuwn-.js} +5 -5
- package/dist/{providers-DZSTlC9R.js.map → providers-vXMXuwn-.js.map} +1 -1
- package/dist/{query-DXpxaaTH.js → query-BhaKQ07P.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-BhaKQ07P.js.map} +1 -1
- package/dist/query-CvgUsvep.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-BzdLdV3C.js} +39 -9
- package/dist/questions-BzdLdV3C.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-CauU408t.js} +231 -25
- package/dist/remarks-CauU408t.js.map +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/{remarks-DeZZT8PK.js → remarks-NMgPd30U.js} +4 -3
- package/dist/{remarks-DeZZT8PK.js.map → remarks-NMgPd30U.js.map} +1 -1
- package/dist/{remote-DZ00A8V_.js → remote-Dwy3fyo3.js} +5 -5
- package/dist/{remote-DZ00A8V_.js.map → remote-Dwy3fyo3.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-B1zMnPwK.js} +7 -7
- package/dist/{resolve-B_JlQ0La.js.map → resolve-B1zMnPwK.js.map} +1 -1
- package/dist/resolve-BIIeVT12.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-CfC0id_u.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-CfC0id_u.js.map} +1 -1
- package/dist/{resume-CexsWuGK.js → resume-Dkhrpjow.js} +123 -8
- package/dist/resume-Dkhrpjow.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-DcvQcr36.js} +2 -2
- package/dist/{runtime-CSZ1hjlN.js.map → runtime-DcvQcr36.js.map} +1 -1
- package/dist/{sandbox-B5W8lLL1.js → sandbox-8zCrtuCf.js} +2 -2
- package/dist/{sandbox-B5W8lLL1.js.map → sandbox-8zCrtuCf.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-0tuj8VSa.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Dplq8g0r.js} +7 -7
- package/dist/{server-BfN6q2lH.js.map → server-Dplq8g0r.js.map} +1 -1
- package/dist/{source-BD6Bg0Tr.js → source-NWHqn5-o.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-NWHqn5-o.js.map} +1 -1
- package/dist/source-llwLewOU.js +1 -0
- package/dist/{src-BC2WXULN.js → src-B9oE_mNS.js} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-BQ6BiXoc.js} +19 -11
- package/dist/{src-Cpy9MNS3.js.map → src-BQ6BiXoc.js.map} +1 -1
- package/dist/{src-Dp-fWOfG.js → src-BQXdf-Gw.js} +2 -2
- package/dist/{src-Dp-fWOfG.js.map → src-BQXdf-Gw.js.map} +1 -1
- package/dist/{src-DurP--_D.js → src-BXnv0ViU.js} +1 -1
- package/dist/src-BiS8YNN_.js +3 -0
- package/dist/{src-DVaAKCsH.js → src-BkH4GOxu.js} +1 -1
- package/dist/{src-CM7xdJFA.js → src-C-tH5ebd.js} +1 -1
- package/dist/{src-Cx0HJIJ-.js → src-CUpoVVb8.js} +1 -1
- package/dist/{src-C9elednc.js → src-Cg1bFs9g.js} +1 -1
- package/dist/{src-BKDZO3ra.js → src-DV_UorZv.js} +1 -1
- package/dist/{src-CWEGknZo.js → src-XxUpr2sa.js} +1 -1
- package/dist/{state-CdSjXIfE.js → state-DDCGUhva.js} +5 -5
- package/dist/{state-CdSjXIfE.js.map → state-DDCGUhva.js.map} +1 -1
- package/dist/state-DZbAO95a.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-M8_7xio7.js +1294 -0
- package/dist/suite-M8_7xio7.js.map +1 -0
- package/dist/suite-u2KJnGpY.js +2 -0
- package/dist/{summary-zP3cCXDN.js → summary-Uzc9osfV.js} +47 -13
- package/dist/summary-Uzc9osfV.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
- package/dist/{types-BHR1lGPh.d.ts → types-DGvexkcp.d.ts} +71 -63
- package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
- package/dist/vertex-BLQMayc1.js.map +1 -0
- package/dist/vertex-h_peaZTN.js +1 -0
- package/dist/view/index.d.ts +140 -35
- package/dist/view/index.js +8 -7
- package/dist/{views-CKoN5NAx.js → views-BISrsIjw.js} +6 -6
- package/dist/{views-CKoN5NAx.js.map → views-BISrsIjw.js.map} +1 -1
- package/dist/views-Bfab8fM9.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-DJD_nHkS.js} +5 -5
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
- package/dist/workspace-B8EOCuxl.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-BO1VhUR9.js} +5 -5
- package/dist/{workspace-Cho30qlM.js.map → workspace-BO1VhUR9.js.map} +1 -1
- package/package.json +3 -1
- package/skills/eval-rubric/SKILL.md +22 -3
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,61 +1,3 @@
|
|
|
1
|
-
/** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
|
|
2
|
-
type DeclaredMode = 'run' | 'skip' | 'only';
|
|
3
|
-
/** A `group`: options its evals and groups inherit, and its children in declaration order. */
|
|
4
|
-
type DeclaredGroup = {
|
|
5
|
-
type: 'group';
|
|
6
|
-
/** The group's name, as passed to `group`. */
|
|
7
|
-
name: string;
|
|
8
|
-
mode: DeclaredMode;
|
|
9
|
-
/** The group's own options, before the cascade. */
|
|
10
|
-
options: EvalOptionsData;
|
|
11
|
-
children: DeclaredNode[];
|
|
12
|
-
};
|
|
13
|
-
/** An `evaluate`: one eval, planned into a test per attempt. */
|
|
14
|
-
type DeclaredEval = {
|
|
15
|
-
type: 'eval';
|
|
16
|
-
/** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
|
|
17
|
-
name: string;
|
|
18
|
-
mode: DeclaredMode;
|
|
19
|
-
/** The eval's own options, before the cascade. */
|
|
20
|
-
options: EvalOptionsData;
|
|
21
|
-
/** Names the eval's body to whoever runs it. */
|
|
22
|
-
body: string;
|
|
23
|
-
source: DeclaredSource;
|
|
24
|
-
/** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
|
|
25
|
-
family?: DeclaredFamily;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
|
|
29
|
-
* text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
|
|
30
|
-
* whitespace, non-ASCII characters as themselves).
|
|
31
|
-
*/
|
|
32
|
-
type DeclaredSource = {
|
|
33
|
-
/** sha256 of the body function's source text. */
|
|
34
|
-
body: string;
|
|
35
|
-
/** sha256 of the case's data; set on each case of an `evaluate.each`. */
|
|
36
|
-
case?: string;
|
|
37
|
-
};
|
|
38
|
-
/** Where a case of a pooled `evaluate.each` sits in its list. */
|
|
39
|
-
type DeclaredFamily = {
|
|
40
|
-
/** The name template passed to `evaluate.each`. */
|
|
41
|
-
name: string;
|
|
42
|
-
/** The case's index, from 0. */
|
|
43
|
-
case: number;
|
|
44
|
-
/** How many cases the list has. */
|
|
45
|
-
cases: number;
|
|
46
|
-
};
|
|
47
|
-
/** `evaluate.todo` or `group.todo`. */
|
|
48
|
-
type DeclaredTodo = {
|
|
49
|
-
type: 'todo';
|
|
50
|
-
/** The name passed to `.todo`. */
|
|
51
|
-
name: string;
|
|
52
|
-
};
|
|
53
|
-
/** One node of the tree: a group, an eval or a todo. */
|
|
54
|
-
type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
|
|
55
|
-
/** An eval file's top-level groups, evals and todos, in declaration order. */
|
|
56
|
-
type DeclarationTree = {
|
|
57
|
-
children: DeclaredNode[];
|
|
58
|
-
};
|
|
59
1
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
60
2
|
declare const EFFORT_ALPHA = 0.05;
|
|
61
3
|
/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
|
|
@@ -516,6 +458,11 @@ type AttemptRecord = {
|
|
|
516
458
|
* in the attempt's directory, `<n>` the question's place in the eval's `questions`.
|
|
517
459
|
*/
|
|
518
460
|
answers?: AttemptAnswer[];
|
|
461
|
+
/**
|
|
462
|
+
* Each `c.observe(name, value)` the attempt made, one per name in first-call order with its last
|
|
463
|
+
* value; unset without any. They annotate: `outcome` never depends on them.
|
|
464
|
+
*/
|
|
465
|
+
observations?: Observation[];
|
|
519
466
|
};
|
|
520
467
|
type EvalEntry = {
|
|
521
468
|
/** `file › group › … › name`, as shown. */
|
|
@@ -555,6 +502,12 @@ type EvalEntry = {
|
|
|
555
502
|
outcome: Outcome;
|
|
556
503
|
/** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
|
|
557
504
|
todo: boolean;
|
|
505
|
+
/**
|
|
506
|
+
* Declared not to run: `evaluate.skip`, inside a `group.skip`, or left out by an `.only` in its
|
|
507
|
+
* file. Its outcome is `skipped`. Unset otherwise, so in a run that isn't `complete` a skipped
|
|
508
|
+
* eval with neither `todo` nor `skip` hasn't run yet. Unset on records from before it.
|
|
509
|
+
*/
|
|
510
|
+
skip?: true;
|
|
558
511
|
/** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
|
|
559
512
|
passRate: number | null;
|
|
560
513
|
cells: CellRecord[];
|
|
@@ -817,6 +770,22 @@ type Cost = {
|
|
|
817
770
|
usage: TokenUsage | null;
|
|
818
771
|
source: 'reported' | 'unreported';
|
|
819
772
|
};
|
|
773
|
+
/**
|
|
774
|
+
* The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
|
|
775
|
+
* them, so it finishes the attempts the run planned.
|
|
776
|
+
*/
|
|
777
|
+
type RunInvocation = {
|
|
778
|
+
/** `--runs`; unset when each cell ran its own `runs`. */
|
|
779
|
+
runs?: number;
|
|
780
|
+
/** The profile the run was sized with; unset when it had none. */
|
|
781
|
+
profile?: string;
|
|
782
|
+
/** `--concurrency` or `REEVAL_CONCURRENCY`; unset when the config or the machine sized it. */
|
|
783
|
+
concurrency?: number;
|
|
784
|
+
/** The filter arguments. */
|
|
785
|
+
filters: string[];
|
|
786
|
+
/** `--tag`, without the profile's own tags. */
|
|
787
|
+
tags: string[];
|
|
788
|
+
};
|
|
820
789
|
type RunRecord = {
|
|
821
790
|
kind: 'reeval-run';
|
|
822
791
|
schemaVersion: typeof SCHEMA_VERSION;
|
|
@@ -863,6 +832,8 @@ type RunRecord = {
|
|
|
863
832
|
harness?: string[];
|
|
864
833
|
};
|
|
865
834
|
} | null;
|
|
835
|
+
/** How `reeval run` was started; `--resume` plans with it again. Unset in older records. */
|
|
836
|
+
invocation?: RunInvocation;
|
|
866
837
|
/**
|
|
867
838
|
* What the run planned before it started, and each level whose sandbox ran as root or with
|
|
868
839
|
* capabilities added back.
|
|
@@ -1519,6 +1490,8 @@ type JudgeInputs = {
|
|
|
1519
1490
|
transcript?: boolean | 'tools' | 'full';
|
|
1520
1491
|
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
|
|
1521
1492
|
checks?: boolean | string[];
|
|
1493
|
+
/** `c.observe()` values so far, by name; a list shows only those names. */
|
|
1494
|
+
observations?: boolean | string[];
|
|
1522
1495
|
/**
|
|
1523
1496
|
* Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
|
|
1524
1497
|
* capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
|
|
@@ -1553,6 +1526,8 @@ type ResolvedJudgeStateOptions = {
|
|
|
1553
1526
|
transcript: boolean | 'tools' | 'full';
|
|
1554
1527
|
turns: 'last' | 'all';
|
|
1555
1528
|
checks: boolean | string[];
|
|
1529
|
+
/** Set only when the call turned it on, so never `false` as written. */
|
|
1530
|
+
observations?: boolean | string[];
|
|
1556
1531
|
};
|
|
1557
1532
|
type Verdict = {
|
|
1558
1533
|
verdict: 'pass' | 'fail';
|
|
@@ -1628,6 +1603,12 @@ type AttemptContext = {
|
|
|
1628
1603
|
* made to it afterwards.
|
|
1629
1604
|
*/
|
|
1630
1605
|
writeFile(path: string, contents: string | Uint8Array): Promise<void>;
|
|
1606
|
+
/**
|
|
1607
|
+
* Records a fact about the attempt under `name`, such as what the agent installed. It never
|
|
1608
|
+
* affects the outcome. Calling it again with a name replaces its value. The value is JSON, at
|
|
1609
|
+
* most 4 KB, and is masked like the rest of the record.
|
|
1610
|
+
*/
|
|
1611
|
+
observe(name: string, value: JSONValue): void;
|
|
1631
1612
|
/** Every turn so far, oldest first. */
|
|
1632
1613
|
readonly turns: readonly Turn[];
|
|
1633
1614
|
/** Aborted on timeout or Ctrl-C. */
|
|
@@ -2612,6 +2593,8 @@ type QuestionInputs = {
|
|
|
2612
2593
|
transcript?: boolean | 'tools' | 'full';
|
|
2613
2594
|
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
|
|
2614
2595
|
checks?: boolean | string[];
|
|
2596
|
+
/** `c.observe()` values, by name; a list shows only those names. */
|
|
2597
|
+
observations?: boolean | string[];
|
|
2615
2598
|
/** Each attempt's own captures, by name; `true` sends all of them. */
|
|
2616
2599
|
artifacts?: true | string[];
|
|
2617
2600
|
};
|
|
@@ -2624,6 +2607,8 @@ type ResolvedQuestionStateOptions = {
|
|
|
2624
2607
|
transcript: boolean | 'tools' | 'full';
|
|
2625
2608
|
turns: 'last' | 'all';
|
|
2626
2609
|
checks: boolean | string[];
|
|
2610
|
+
/** Set only when the question turned it on, so never `false` as written. */
|
|
2611
|
+
observations?: boolean | string[];
|
|
2627
2612
|
};
|
|
2628
2613
|
/** Defaults for every question below it. What a question is shown is its own, never a default. */
|
|
2629
2614
|
type QuestionOptions = {
|
|
@@ -2794,7 +2779,14 @@ type Revision = number | string;
|
|
|
2794
2779
|
* An eval's own source as hashes: its body and case data as declared, and its options and its
|
|
2795
2780
|
* enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
|
|
2796
2781
|
*/
|
|
2797
|
-
type EvalSource =
|
|
2782
|
+
type EvalSource = {
|
|
2783
|
+
/** sha256 of the body function's source text. */
|
|
2784
|
+
body: string;
|
|
2785
|
+
/**
|
|
2786
|
+
* sha256 of the case's data as canonical JSON, with secrets in its strings masked as the record
|
|
2787
|
+
* masks them; set on each case of an `evaluate.each`.
|
|
2788
|
+
*/
|
|
2789
|
+
case?: string;
|
|
2798
2790
|
/** sha256 of the canonical options. */
|
|
2799
2791
|
options: string;
|
|
2800
2792
|
};
|
|
@@ -2985,6 +2977,14 @@ type BlindSide = BlindTurn & {
|
|
|
2985
2977
|
command: string;
|
|
2986
2978
|
notRun: true;
|
|
2987
2979
|
}>;
|
|
2980
|
+
/** Its `c.observe()` values; a name the question lists that it never observed is `missing`. */
|
|
2981
|
+
observations?: Array<{
|
|
2982
|
+
name: string;
|
|
2983
|
+
value: JSONValue;
|
|
2984
|
+
} | {
|
|
2985
|
+
name: string;
|
|
2986
|
+
missing: true;
|
|
2987
|
+
}>;
|
|
2988
2988
|
/** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
|
|
2989
2989
|
artifacts?: Array<{
|
|
2990
2990
|
label: string;
|
|
@@ -3006,7 +3006,7 @@ type AnalysisConfig = {
|
|
|
3006
3006
|
instructions?: string;
|
|
3007
3007
|
/** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
|
|
3008
3008
|
maxInputTokens?: number;
|
|
3009
|
-
/** The most tokens the model may answer with (default
|
|
3009
|
+
/** The most tokens the model may answer with, reasoning included (default 32000). reeval has no prices, so tokens are the cap. */
|
|
3010
3010
|
maxOutputTokens?: number;
|
|
3011
3011
|
};
|
|
3012
3012
|
type ReevalConfig = {
|
|
@@ -3415,6 +3415,14 @@ type AssertionRecord = {
|
|
|
3415
3415
|
/** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
|
|
3416
3416
|
durationMs?: number;
|
|
3417
3417
|
};
|
|
3418
|
+
/** A `c.observe(name, value)` call: the last value given for the name. */
|
|
3419
|
+
type Observation = {
|
|
3420
|
+
name: string;
|
|
3421
|
+
/** Masked like the rest of the record. */
|
|
3422
|
+
value: JSONValue;
|
|
3423
|
+
/** How many turns the attempt had sent when the value was given: 0 before the first. */
|
|
3424
|
+
turn: number;
|
|
3425
|
+
};
|
|
3418
3426
|
/**
|
|
3419
3427
|
* A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
|
|
3420
3428
|
* folder), or why it wasn't.
|
|
@@ -3505,7 +3513,7 @@ type SandboxWarning = {
|
|
|
3505
3513
|
file: string | null;
|
|
3506
3514
|
/** The capabilities added back, for `capabilities`. */
|
|
3507
3515
|
capabilities?: string[];
|
|
3508
|
-
/** The line reeval prints
|
|
3516
|
+
/** The line reeval prints after `⚠ Warning:`, in red for root and orange for capabilities. */
|
|
3509
3517
|
message: string;
|
|
3510
3518
|
};
|
|
3511
3519
|
/** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
|
|
@@ -3516,5 +3524,5 @@ type ResolvedGateway = {
|
|
|
3516
3524
|
why: string;
|
|
3517
3525
|
}>;
|
|
3518
3526
|
};
|
|
3519
|
-
export {
|
|
3520
|
-
//# sourceMappingURL=types-
|
|
3527
|
+
export { FakeAnalysisModel as $, effortVerdict as $n, StatusRange as $t, CompareCut as A, UpstreamCount as An, OpenRouterGatewayDescriptor as At, EgressPolicy as B, TOKENS_UNREPORTED as Bn, ProfileLimit as Bt, CaptureWindow as C, HistoryCell as Cn, JudgeModel as Ct, ClaudeCodeHarnessDescriptor as D, RunInvocation as Dn, McpServer as Dt, ClaudeCodeEffort as E, Lift as En, JudgeSubject as Et, Diff as F, EffortContrast as Fn, PinnableKey as Ft, EnvironmentInput as G, cellSamples as Gn, ResolvedGateway as Gt, EnvVars as H, bothUnderATenth as Hn, QuestionInputs as Ht, DirectGatewayDescriptor as I, EffortMetric as In, PluginPart as It, EvalFn as J, effortChangeLabel as Jn, Secret as Jt, EnvironmentSpec as K, contrastEffort as Kn, SandboxDescriptor as Kt, DirectoryOptions as L, EffortSample as Ln, PluginSource as Lt, ConfigHelpers as M, DIRECTIONAL_MIN as Mn, PairFactor as Mt, ConfigJudgeOptions as N, EFFORT_ALPHA as Nn, PairVote as Nt, CodexHarnessDescriptor as O, RunRecord as On, Observation as Ot, CursorHarnessDescriptor as P, EFFORT_MIN_RUNS as Pn, PiHarnessDescriptor as Pt, ExecRetry as Q, effortSummary as Qn, SkillSource as Qt, DockerSandboxDescriptor as R, EffortVerdict as Rn, Profile as Rt, CaptureWait as S, FileRef as Sn, JudgeInputs as St, CellRole as T, JudgmentRecord as Tn, JudgeState as Tt, Environment as U, cellContrast as Un, QuestionOptions as Ut, EgressRecord as V, VERDICT_TEXT as Vn, Question as Vt, EnvironmentHelpers as W, cellEffort as Wn, ReevalConfig as Wt, EvalOptionsData as X, effortOf as Xn, ServeHandle as Xt, EvalOptions as Y, effortLabel as Yn, SecretTemplate as Yt, ExecResult as Z, effortSample as Zn, SetupStep as Zt, CapturePage as _, FamilyLift as _n, HostFile as _t, AttemptContext as a, VercelGatewayDescriptor as an, FakeVote as at, CaptureUseAction as b, FamilyRecord as bn, JudgeCallOptions as bt, BlindPair as c, VertexPublisher as cn, GatewayName as ct, CaptureCall as d, WebEventType as dn, GroupOptions as dt, TokenUsage as en, filesChanged as er, FakeAnswer as et, CaptureConsoleMessage as f, ArtifactRef as fn, HarnessData as ft, CaptureOptions as g, EvalEntry as gn, HistoryConfig as gt, CaptureKey as h, CheckTally as hn, HarnessOption as ht, AttemptAnswer as i, TurnStatus as in, tokenValues as ir, FakeScriptStep as it, CompareOptions as j, CellEffort as jn, Outcome as jt, CompareAnswer as k, StackSummary as kn, OpenCodeHarnessDescriptor as kt, Budget as l, Viewport as ln, GooseHarnessDescriptor as lt, CaptureInstruction as m, CellRecord as mn, HarnessEdit as mt, AnalysisConfig as n, Turn as nn, formatRatio as nr, FakeJudgeModel as nt, BackgroundHandle as o, Verdict as on, FxHarnessDescriptor as ot, CaptureFailedRequest as p, AttemptRecord as pn, HarnessDescriptor as pt, ErrorKind as q, directional as qn, SandboxWarning as qt, AntigravityHarnessDescriptor as r, TurnData as rn, formatSeconds as rr, FakeSandboxDescriptor as rt, BlindAttempt as s, VertexGatewayDescriptor as sn, GatewayDescriptor as st, AgentHarnessDescriptor as t, TranscriptEvent as tn, formatCount as tr, FakeHarnessDescriptor as tt, Capture as u, Wait as un, GrokBuildHarnessDescriptor as ut, CapturePageFile as v, FamilyLiftSummary as vn, JSONValue as vt, Cell as w, HistoryEntry as wn, JudgeOptions as wt, CaptureUseScript as x, FamilyVerdict as xn, JudgeFailure as xt, CaptureUse as y, FamilyRate as yn, JudgeArtifactRef as yt, Duration as z, Spread as zn, ProfileDefaults as zt };
|
|
3528
|
+
//# sourceMappingURL=types-DGvexkcp.d.ts.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
+
import { a as claudeCodeVertexAliasRefusal } from "./harness-DYSC4zMh.js";
|
|
2
3
|
import { generateKeyPairSync } from "node:crypto";
|
|
3
4
|
// src/gateway/token.ts
|
|
4
5
|
/** How long before a token expires it's replaced. */
|
|
@@ -397,10 +398,12 @@ function vertexModelProbe(model, settings) {
|
|
|
397
398
|
/**
|
|
398
399
|
* Why Vertex AI can't serve a routed harness's model, or undefined when it can: fx's Chat
|
|
399
400
|
* Completions endpoint and OpenCode's `google-vertex` provider serve Google's models, and Claude
|
|
400
|
-
* only over Anthropic's own API there
|
|
401
|
+
* only over Anthropic's own API there; Claude Code's `sonnet` and `opusplan` aliases name a
|
|
402
|
+
* retiring model there.
|
|
401
403
|
* A bare fx id is OpenAI's everywhere else, so only a bare `gemini-…` one is taken as Google's.
|
|
402
404
|
*/
|
|
403
405
|
function vertexModelRefusalFor(h) {
|
|
406
|
+
if (h.kind === "claude-code") return claudeCodeVertexAliasRefusal(h.model);
|
|
404
407
|
const provider = /^([^/]+)\//.exec(h.model)?.[1];
|
|
405
408
|
if (h.kind === "opencode") return provider === "google" || provider === "google-vertex" ? void 0 : `OpenCode reaches Vertex AI only on its google-vertex provider, which serves Google's models, not ${JSON.stringify(h.model)}: use a Gemini model, such as google/gemini-3.8-flash`;
|
|
406
409
|
if (h.kind !== "fx") return void 0;
|
|
@@ -453,4 +456,4 @@ async function vertexDecisionModel(model, settings, source = vertexTokens()) {
|
|
|
453
456
|
}
|
|
454
457
|
export { brokerToken as S, vertexRegion as _, VERTEX_ROUTES as a, vertexTokens as b, openCodeVertexId as c, vertexHarnessModel as d, vertexHost as f, vertexModelRefusalFor as g, vertexModelRefusal as h, VERTEX_REFUSALS as i, vertexClaudeId as l, vertexModelProbe as m, VERTEX_MODEL_PROVIDERS as n, adcTokenSource as o, vertexLanguageModel as p, VERTEX_PROJECT_ENV as r, fxVertexId as s, SANDBOX_ACCESS_TOKEN as t, vertexDecisionModel as u, vertexSettings as v, vertexXaiId as x, vertexSettingsIn as y };
|
|
455
458
|
|
|
456
|
-
//# sourceMappingURL=vertex-
|
|
459
|
+
//# sourceMappingURL=vertex-BLQMayc1.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vertex-BLQMayc1.js","names":[],"sources":["../src/gateway/token.ts","../src/gateway/vertex.ts"],"mappings":";;;;;AAkBA,MAAa,mBAAmB;;AAEhC,MAAa,iBAAiB;;AAE9B,MAAa,WAAW;;;;;AAMxB,SAAgB,kBAAkB,QAAqB,MAAM,KAAK,KAAkB;CAClF,IAAI;CACJ,IAAI;CACJ,aAAa;EACX,IAAI,WAAW,QAAQ,YAAA,MAA+B,IAAI,GAAG,OAAO,QAAQ,QAAQ,OAAO;EAC3F,YAAY,OAAO,CAAC,CACjB,MAAM,UAAU;GACf,UAAU;GACV,OAAO;EACT,CAAC,CAAC,CACD,cAAc;GACb,UAAU,KAAA;EACZ,CAAC;EACH,OAAO;CACT;AACF;AAIA,MAAM,cAAwB,KAAK,OAAO;CACxC,MAAM,QAAQ,WAAW,KAAK,EAAE;CAChC,MAAM,MAAM;CACZ,aAAa,aAAa,KAAK;AACjC;;;;;;AAYA,eAAsB,YACpB,UACA,SACA,UAKI,CAAC,GACgB;CACrB,MAAM,EAAE,MAAM,KAAK,KAAK,WAAW,eAAe;CAClD,MAAM,MAAM,QAAQ,2BAA2B,KAAK,OAAO;CAC3D,IAAI,CAAC,KAAK,MAAM,IAAI,MAAM,+CAA+C;CACzE,IAAI,UAAU;CACd,IAAI,eAAe,CAAC;CACpB,MAAM,QAAQ,YAAY;EACxB,MAAM,QAAQ,MAAM,SAAS,OAAO;EACpC,IAAI,CAAC,SAAS,MAAM,IAAI,SAAS,gBAAgB,MAAM,KAAK,CAAC;EAC7D,OAAO;CACT;CACA,MAAM,QAAQ,UAAuB;EACnC,IAAI,SAAS;EACb,MAAM,OAAO,KAAK,IAAI,MAAM,YAAY,mBAAmB,IAAI,GAAG,cAAc;EAChF,SAAS,eAAe,KAAK,QAAQ,GAAG,IAAI;CAC9C;CACA,MAAM,UAAU,YAAY;EAC1B,IAAI;GACF,MAAM,QAAQ,MAAM,MAAM;GAC1B,QAAQ,YAAY,KAAK;GACzB,KAAK,KAAK;EACZ,SAAS,OAAO;GACd,QAAQ,UAAU,KAAK;GACvB,IAAI,CAAC,SAAS,SAAS,eAAe,KAAK,QAAQ,GAAG,QAAQ;EAChE;CACF;CACA,KAAK,MAAM,MAAM,CAAC;CAClB,aAAa;EACX,UAAU;EACV,OAAO;CACT;AACF;;;ACtFA,MAAa,qBAAqB;;AAUlC,SAAgB,eACd,QAA+C,CAAC,GAChD,MAAyB,QAAQ,KACjB;CAChB,MAAM,UAAU,MAAM,WAAW,IAAA;CACjC,IAAI,CAAC,SACH,MAAM,IAAI,WACR,+DAA+D,oBACjE;CAEF,OAAO;EAAE;EAAS,UAAU,MAAM,YAAY;EAAU,SAAS,MAAM,WAAW,CAAC;CAAE;AACvF;;AAGA,SAAgB,iBACd,UACA,MAAyB,QAAQ,KACjB;CAChB,MAAM,QAAQ,SAAS,MAAM,MAAM,EAAE,SAAS,QAAQ;CACtD,OAAO,eAAe,OAAO,SAAS,WAAW,QAAQ,CAAC,GAAG,GAAG;AAClE;;AAGA,MAAa,gBAAgB,UAA0B,cACrD,SAAS,QAAQ,cAAc,SAAS;;AAG1C,MAAa,cAAc,WACzB,WAAW,WAAW,8BAA8B,GAAG,OAAO;AAEhE,MAAM,iBAAiB;;AAEvB,MAAM,sBAAsB;;AAG5B,SAAgB,eAAe,MAAM,KAAK,KAAkB;CAC1D,IAAI;CACJ,OAAO,YAAY;EACjB,WAAW,OAAO,sBAAsB,CAAC,MAAM,EAAE,iBAC/C,IAAI,WAAW,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,UAAU,CACzD;EACA,MAAM,OAAO,MAAM;EACnB,MAAM,EAAE,UAAU,MAAM,KAAK,eAAe;EAC5C,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,sDAAsD;EAClF,MAAM,SAAU,KAA2D,aACvE;EACJ,OAAO;GAAE;GAAO,WAAW,OAAO,WAAW,WAAW,SAAS,IAAI,IAAI;EAAoB;CAC/F;AACF;AAEA,IAAI;;AAEJ,MAAa,qBAAkC;CAC7C,WAAW,kBAAkB,eAAe,CAAC;CAC7C,OAAO;AACT;;;;;;AAsCA,MAAa,kBAAkB,UAC7B,MAAM,QAAQ,gBAAgB,EAAE,CAAC,CAAC,QAAQ,aAAa,KAAK;AAE9D,MAAM,eAAe,SAAiB,KAAK,QAAQ,uBAAuB,MAAM;;AAGhF,SAAS,kBACP,UACA,WACA,SACA,WAA8B,CAAC,IAAI,GACpB;CACf,MAAM,SAAS,aAAa,UAAU,SAAS;CAC/C,OAAO;EACL,MAAM,WAAW,MAAM;EACvB,MAAM,EACJ,OAAO,QAAQ,SAAS,KAAK,GAAG,EAAE,aAAa,YAAY,SAAS,OAAO,EAAE,aAAa,YAAY,MAAM,EAAE,cAAc,UAAU,qBAAqB,QAAQ,KAAK,GAAG,EAAE,cAC/K;EACA,QAAQ,CAAC,MAAM;CACjB;AACF;;AAGA,MAAM,YACH,eACA,UACC,UAAU,KAAK,WAAW;CACxB;CACA,WAAW,EAAE,SAAS,EAAE,eAAe,UAAU,QAAQ,EAAE;AAC7D,EAAE;;;;;;AAON,SAAS,WAAW,UAA0B,SAAS,aAAa,GAAgB;CAClF,MAAM,SAAS,aAAa,UAAU,WAAW;CACjD,MAAM,YAAY,CAAC,kBAAkB,UAAU,aAAa,CAAC,oBAAoB,YAAY,CAAC,CAAC;CAC/F,OAAO;EACL,YAAY;EACZ;EACA,OAAO,CAAC,WAAW,MAAM,CAAC;EAC1B,cAAc,EAAE,MAAM,WAAW;GAC/B,IAAI,SAAS,WAAW,MAAM,GAAG,OAAO,KAAA;GACxC,MAAM,QAAQ,6CAA6C,KAAK,IAAI,CAAC,GAAG;GACxE,OAAO,SAAS,UAAU,iBAAiB,QAAQ,KAAA;EACrD;EACA,SAAS;EACT,UAAU;GACR,wBAAwB;GACxB,8BAA8B;GAC9B,6BAA6B,SAAS;GACtC,iBAAiB;EACnB;EACA,OAAO;GAAE;GAAQ,iBAAiB,SAAS,SAAS;EAAE;CACxD;AACF;;AAGA,MAAM,iBAAgC;CACpC,MAAM;CACN,MAAM,EAAE,OAAO,SAAS;CACxB,QAAQ,CAAC,MAAM;AACjB;;AAGA,MAAa,uBAAuB;AAEpC,IAAI;;AAEJ,MAAM,0BAA0B;CAC9B,eAAe,oBAAoB,OAAO;EACxC,eAAe;EACf,oBAAoB;GAAE,MAAM;GAAS,QAAQ;EAAM;EACnD,mBAAmB;GAAE,MAAM;GAAQ,QAAQ;EAAM;CACnD,CAAC,CAAC,CAAC;CACH,OAAO;AACT;AAEA,MAAM,kBAAkB,YAAoB,kBAAkB,QAAQ;;AAGtE,SAAS,WAAW,UAAsC;CACxD,MAAM,UAAU,KAAK,UAAU;EAC7B,MAAM;EACN,YAAY,SAAS;EACrB,gBAAgB;EAChB,aAAa,kBAAkB;EAC/B,cAAc,eAAe,SAAS,OAAO;EAC7C,WAAW;EACX,WAAW,WAAW,eAAe,KAAK;CAC5C,CAAC;CACD,OAAO;EACL,SAAS,SAAS;EAClB,UAAU,aAAa,UAAU,QAAQ;EACzC,MAAM;EACN;CACF;AACF;AAEA,MAAM,QAAQ,SAAkB,KAAK,UAAU,IAAI;;AAGnD,MAAM,gBAA+B;CACnC,OAAO;CACP,QAAQ;CACR,MAAM,KAAK;EAAE,cAAc;EAAsB,YAAY;EAAM,YAAY;CAAS,CAAC;AAC3F;;;;;;AAOA,SAAS,OACP,aAAgC,CAAC,GACjC,uBAA+E,CAAC,GAChF,WAAsC,UAAU,MAAM,QAAQ,aAAa,EAAE,GAC7E;CACA,QAAQ,UAA0B,SAAS,aAAa,MAAmB;EACzE,MAAM,SAAS,aAAa,UAAU,QAAQ;EAC9C,MAAM,YAAY,CAChB,kBACE,UACA,UACA,CAAC,yBAAyB,iBAAiB,GAC3C,CAAC,MAAM,SAAS,CAClB,CACF;EACA,OAAO;GACL,YAAY;GACZ;GACA,OAAO;IAAC,WAAW,MAAM;IAAG,eAAe;IAAM,GAAG;GAAU;GAC9D,cAAc,EAAE,MAAM,WACpB,SAAS,WAAW,MAAM,IACtB,0CAA0C,KAAK,IAAI,CAAC,GAAG,KACvD,KAAA;GACN;GACA,OAAO;IAAE;IAAQ,iBAAiB,SAAS,SAAS;GAAE;GACtD,KAAK,WAAW,QAAQ;GACxB,gBAAgB,CAAC,eAAe,GAAG,eAAe,QAAQ,CAAC;EAC7D;CACF;AACF;;AAGA,MAAM,uBAAuB,aAA8C,CACzE;CACE,OAAO;EAAE,MAAM;EAAsB,MAAM,EAAE,OAAO,sBAAsB;EAAG,QAAQ,CAAC,KAAK;CAAE;CAC7F,QAAQ;CACR,MAAM,KAAK;EAAE,IAAI;EAAK,OAAO,eAAe,SAAS,OAAO;EAAG,gBAAgB;CAAK,CAAC;AACvF,CACF;;AAGA,MAAa,eAAe,UAAmB,MAAM,SAAS,GAAG,IAAI,QAAQ,OAAO;;;;;;AAOpF,SAAS,UAAU,UAA0B,SAAS,aAAa,GAAgB;CACjF,MAAM,SAAS,aAAa,UAAU,KAAK;CAC3C,MAAM,YAA6B,CACjC;EACE,MAAM,WAAW,MAAM;EACvB,MAAM,EAEJ,OAAO,iBAAiB,YAAY,mBAAmB,SAAS,OAAO,CAAC,EAAE,aAAa,YAAY,MAAM,EAAE,gDAC7G;EACA,QAAQ,CAAC,MAAM;CACjB,CACF;CACA,OAAO;EACL,YAAY;EACZ;EACA,gBAAgB;EAChB,OAAO,CAAC,WAAW,MAAM,CAAC;EAC1B,cAAc,EAAE,MAAM,oBACpB,SAAS,WAAW,MAAM,IAAI,gBAAgB,KAAA;EAChD,SAAS;EACT,YAAY;GAAE,SAAS,SAAS;GAAS;EAAO;EAChD,OAAO;GAAE;GAAQ,iBAAiB,SAAS,SAAS;EAAE;CACxD;AACF;;AAGA,MAAa,oBAAoB,UAC/B,iBAAiB,MAAM,QAAQ,+BAA+B,EAAE;;AAGlE,MAAa,cAAc,UAAkB,UAAU,MAAM,QAAQ,aAAa,EAAE;;;;;;;AAQpF,SAAS,GAAG,UAA0B,SAAS,aAAa,GAAgB;CAC1E,MAAM,SAAS,aAAa,UAAU,QAAQ;CAC9C,MAAM,OAAO,WAAW,MAAM;CAC9B,MAAM,YAA6B,CACjC;EACE;EACA,MAAM,EACJ,OAAO,6BAA6B,YAAY,SAAS,OAAO,EAAE,aAAa,YAAY,MAAM,EAAE,gDACrG;EACA,QAAQ,CAAC,MAAM;CACjB,CACF;CACA,OAAO;EACL,YAAY;EACZ;EACA,gBAAgB;EAChB,OAAO,CAAC,IAAI;EACZ,cAAc,YACZ,QAAQ,SAAS,OAAO,QAAQ,eAAe,QAAQ,aAAa,EAAE,IAAI,KAAA;EAC5E,SAAS;EACT,UAAU;GAAE,mBAAmB,SAAS;GAAS,oBAAoB;EAAO;EAC5E,OAAO;GAAE;GAAQ,iBAAiB,SAAS,SAAS;EAAE;CACxD;AACF;;AAGA,MAAa,gBAET;CACF,eAAe;CAEf,aAAa,OAAO,CAAC,oBAAoB,GAAG,sBAAsB,UAAU,KAAK;CAEjF,OAAO,OAAO;CAEd,IAAI,OAAO;CAGX,UAAU,OAAO,CAAC,YAAY,SAAS,CAAC,GAAG,gBAAgB;CAC3D,cAAc;CACd;AACF;;;;;AAMA,MAAM,4BAAyE;CAC7E,eAAe;CACf,aAAa;CACb,OAAO;CACP,IAAI;CACJ,UAAU;CACV,IAAI;AACN;;;;;;AAOA,SAAgB,mBAAmB,MAAiB,OAAmC;CACrF,MAAM,YAAY,0BAA0B;CAC5C,IAAI,CAAC,WAAW,OAAO,KAAA;CACvB,MAAM,KAAK,MAAM,QAAQ,yCAAyC,EAAE;CACpE,OAAO,GAAG,UAAU,GAAG,SAAS,gBAAgB,GAAG,QAAQ,yBAAyB,EAAE,IAAI;AAC5F;;;;;;AAOA,SAAgB,iBACd,OACA,UAC4D;CAC5D,MAAM,YAAY,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CACnD,MAAM,SAAS,aAAa,UAAU,SAAS;CAC/C,MAAM,OAAO,WAAW,WAAW,MAAM,EAAE,eAAe,mBAAmB,SAAS,OAAO,EAAE,aAAa,OAAO,cAAc,UAAU;CAC3I,MAAM,KAAK,MAAM,MAAM,UAAU,SAAS,CAAC;CAC3C,IAAI,cAAc,aAAa;EAC7B,MAAM,SAAS,eAAe,EAAE;EAChC,OAAO;GACL,IAAI;GACJ;GACA,KAAK,GAAG,KAAK;GACb,MAAM;IAAE,OAAO;IAAQ,UAAU,CAAC;KAAE,MAAM;KAAQ,SAAS;IAAO,CAAC;GAAE;EACvE;CACF;CACA,OAAO;EACL;EACA;EACA,KAAK,GAAG,KAAK,GAAG,GAAG;EACnB,MAAM,EAAE,UAAU,CAAC;GAAE,MAAM;GAAQ,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC;EAAE,CAAC,EAAE;CAClE;AACF;;;;;;;;AASA,SAAgB,sBAAsB,GAA+C;CACnF,IAAI,EAAE,SAAS,eAAe,OAAO,6BAA6B,EAAE,KAAK;CACzE,MAAM,WAAW,aAAa,KAAK,EAAE,KAAK,CAAC,GAAG;CAC9C,IAAI,EAAE,SAAS,YACb,OAAO,aAAa,YAAY,aAAa,kBACzC,KAAA,IACA,oGAAoG,KAAK,UAAU,EAAE,KAAK,EAAE;CAElI,IAAI,EAAE,SAAS,MAAM,OAAO,KAAA;CAC5B,IAAI,aAAa,YAAa,CAAC,YAAY,WAAW,KAAK,EAAE,KAAK,GAAI,OAAO,KAAA;CAC7E,OAAO,iGAAiG,KAAK,UAAU,EAAE,KAAK,EAAE;AAClI;;AAGA,MAAa,kBAAsD;CACjE,QAAQ;CACR,OAAO;AACT;;AAGA,MAAa,yBAAyB,CAAC,UAAU,WAAW;;AAG5D,MAAa,sBAAsB,UAChC,uBAA6C,SAAS,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC,CAAC,IACrF,KAAA,IACA,qBAAqB,uBAAuB,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,OAAO,EAAE;;;;;AAMrF,eAAsB,oBACpB,OACA,UACA,SAAsB,aAAa,GACnC;CACA,MAAM,UAAU,mBAAmB,KAAK;CACxC,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,QAAQ,SAAS,MAAM,GAAG;CAC1D,MAAM,WAAW,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CAClD,MAAM,KAAK,MAAM,MAAM,SAAS,SAAS,CAAC;CAC1C,MAAM,oBAAoB,EACxB,YAAY,EAAE,gBAAgB,aAAa,EAAE,QAAQ,MAAM,OAAO,EAAA,CAAG,MAAM,GAAG,EAChF;CAGA,IAAI,aAAa,aAAa;EAC5B,MAAM,EAAE,0BAA0B,MAAM,OAAO;EAC/C,MAAM,WAAW,aAAa,UAAU,WAAW;EACnD,OAAO,sBAAsB;GAAE,SAAS,SAAS;GAAS;GAAU;EAAkB,CAAC,CAAC,CACtF,eAAe,EAAE,CACnB;CACF;CACA,MAAM,EAAE,iBAAiB,MAAM,OAAO;CACtC,MAAM,WAAW,aAAa,UAAU,QAAQ;CAChD,OAAO,aAAa;EAAE,SAAS,SAAS;EAAS;EAAU;CAAkB,CAAC,CAAC,CAAC,EAAE;AACpF;;AAGA,eAAsB,oBACpB,OACA,UACA,SAAsB,aAAa,GACnC;CACA,MAAM,EAAE,uCAAuC,MAAM,OACnD;CAEF,OAAO,IAAI,mCAAmC;EAC5C,OAAO,MAAM,oBAAoB,OAAO,UAAU,MAAM;EACxD,UAAU;CACZ,CAAC;AACH"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { o as adcTokenSource } from "./vertex-BLQMayc1.js";
|
package/dist/view/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { $n as
|
|
1
|
+
import { $n as effortVerdict, An as UpstreamCount, Bn as TOKENS_UNREPORTED, Cn as HistoryCell, Dn as RunInvocation, En as Lift, Fn as EffortContrast, Gn as cellSamples, Hn as bothUnderATenth, In as EffortMetric, Jn as effortChangeLabel, Kn as contrastEffort, Ln as EffortSample, Mn as DIRECTIONAL_MIN, Nn as EFFORT_ALPHA, On as RunRecord, Pn as EFFORT_MIN_RUNS, Qn as effortSummary, Rn as EffortVerdict, Sn as FileRef, Tn as JudgmentRecord, Un as cellContrast, Vn as VERDICT_TEXT, Wn as cellEffort, Xn as effortOf, Yn as effortLabel, Zn as effortSample, _ as CapturePage, _n as FamilyLift, bn as FamilyRecord, en as TokenUsage, er as filesChanged, f as CaptureConsoleMessage, fn as ArtifactRef, gn as EvalEntry, hn as CheckTally, in as TurnStatus, ir as tokenValues, jn as CellEffort, jt as Outcome, kn as StackSummary, mn as CellRecord, nr as formatRatio, p as CaptureFailedRequest, pn as AttemptRecord, q as ErrorKind, qn as directional, qt as SandboxWarning, rr as formatSeconds, tn as TranscriptEvent, tr as formatCount, v as CapturePageFile, vn as FamilyLiftSummary, wn as HistoryEntry, xn as FamilyVerdict, yn as FamilyRate, zn as Spread } from "../types-DGvexkcp.js";
|
|
2
2
|
/** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
|
|
3
|
-
export declare const ANALYSIS_PROMPT_VERSION = "analysis-
|
|
3
|
+
export declare const ANALYSIS_PROMPT_VERSION = "analysis-v13";
|
|
4
4
|
/** Bumped when the digest's shape or selection rules change. */
|
|
5
|
-
export declare const DIGEST_VERSION =
|
|
5
|
+
export declare const DIGEST_VERSION = 15;
|
|
6
6
|
export type Confidence = 'high' | 'medium' | 'low';
|
|
7
7
|
export type AnalysisClaim = {
|
|
8
8
|
text: string;
|
|
@@ -393,10 +393,10 @@ export type JudgeKeyRate = {
|
|
|
393
393
|
question: string;
|
|
394
394
|
pass: number;
|
|
395
395
|
counted: number;
|
|
396
|
-
/** Compact lift vs the paired control,
|
|
396
|
+
/** Compact lift vs the paired control, when both sides counted an attempt. */
|
|
397
397
|
lift: Pick<Lift, 'points' | 'control' | 'variant' | 'withinNoise'> | null;
|
|
398
398
|
};
|
|
399
|
-
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides
|
|
399
|
+
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides counted an attempt. */
|
|
400
400
|
export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
|
|
401
401
|
/**
|
|
402
402
|
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
@@ -500,6 +500,8 @@ export type LiveProgress = {
|
|
|
500
500
|
planned: number;
|
|
501
501
|
counts: LiveCounts;
|
|
502
502
|
attempts: LiveAttempt[];
|
|
503
|
+
/** How the run was started, for a `--resume` after a quit that wrote no record. */
|
|
504
|
+
invocation?: RunInvocation;
|
|
503
505
|
};
|
|
504
506
|
/** A live run as the studio lists it: `stopped` when its process is gone or went quiet. */
|
|
505
507
|
export type LiveRun = Omit<LiveProgress, 'schemaVersion' | 'pid' | 'host' | 'heartbeatAt'> & {
|
|
@@ -513,6 +515,8 @@ export declare const liveTranscriptPath: (runId: string, attemptKey: string) =>
|
|
|
513
515
|
export declare const FINISHED: readonly LiveStatus[];
|
|
514
516
|
export declare const isFinished: (status: LiveStatus) => boolean;
|
|
515
517
|
export declare function liveCounts(attempts: readonly Pick<LiveAttempt, 'status'>[]): LiveCounts;
|
|
518
|
+
/** A saved invocation, or undefined when it isn't one; its numbers are whole and at least 1. */
|
|
519
|
+
export declare function readableInvocation(v: unknown): RunInvocation | undefined;
|
|
516
520
|
/**
|
|
517
521
|
* A progress file as the studio reads it, or null when it isn't one it can read: half-written,
|
|
518
522
|
* corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
|
|
@@ -585,16 +589,15 @@ export declare const shortSha: (sha: string | null | undefined) => string;
|
|
|
585
589
|
/** Reasoning time, or `n/a` when the harness couldn't measure it: never `0s` for unknown. */
|
|
586
590
|
export declare const reasoningLabel: (ms: number | null) => string;
|
|
587
591
|
/**
|
|
588
|
-
* An attempt's effort,
|
|
589
|
-
*
|
|
590
|
-
* With the workspace diff's count, `files changed` follows: an edit made from the shell is in it,
|
|
591
|
-
* never in the file edits.
|
|
592
|
+
* An attempt's effort: tool calls, then the files the workspace diff changed, which counts a file
|
|
593
|
+
* the shell wrote, then the calls to the agent's edit tools, which doesn't.
|
|
592
594
|
*/
|
|
593
|
-
export declare function activityLabel(a: Pick<AttemptRecord['activity'], 'tools' | 'fileEdits'>, changed
|
|
595
|
+
export declare function activityLabel(a: Pick<AttemptRecord['activity'], 'tools' | 'fileEdits'>, changed: number): string;
|
|
594
596
|
/**
|
|
595
597
|
* A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
|
|
596
598
|
* failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
|
|
597
|
-
* default) don't. An interrupted run is `skipped`, whatever ran
|
|
599
|
+
* default) don't. An interrupted run is `skipped`, whatever ran; a resume its filters left
|
|
600
|
+
* `interrupted` exits on its outcome as if it were complete.
|
|
598
601
|
*/
|
|
599
602
|
export declare function runOutcome(run: {
|
|
600
603
|
status: RunRecord['status'];
|
|
@@ -606,6 +609,11 @@ export declare function runOutcome(run: {
|
|
|
606
609
|
}>;
|
|
607
610
|
errors?: readonly string[];
|
|
608
611
|
}): Outcome;
|
|
612
|
+
/**
|
|
613
|
+
* The eval hasn't run yet: its run stopped (interrupted, or out of budget) or a filtered resume
|
|
614
|
+
* left it out, and it's `skipped` without being a todo or declared skipped. A resume runs it.
|
|
615
|
+
*/
|
|
616
|
+
export declare const notRunYet: (run: Pick<RunRecord, 'status'>, e: Pick<EvalEntry, 'outcome' | 'todo' | 'skip'>) => boolean;
|
|
609
617
|
/** Distinct judge labels that failed on a counted attempt, in first-seen order. */
|
|
610
618
|
export declare function failedJudgeLabels(record: Pick<RunRecord, 'evals'>): string[];
|
|
611
619
|
/**
|
|
@@ -852,6 +860,44 @@ export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string)
|
|
|
852
860
|
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
853
861
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
854
862
|
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
863
|
+
/** One observed name over one cell's attempts: its values tallied, or its numbers summarized. Never gates. */
|
|
864
|
+
export type ObservedTally = {
|
|
865
|
+
name: string;
|
|
866
|
+
/** The cell's key. */
|
|
867
|
+
cell: string;
|
|
868
|
+
/** Attempts that observed it. */
|
|
869
|
+
attempts: number;
|
|
870
|
+
/**
|
|
871
|
+
* Each string, boolean or null value, and each such item of an array value once per attempt,
|
|
872
|
+
* by how many attempts gave it, most first (ties in first-seen order); empty when none.
|
|
873
|
+
*/
|
|
874
|
+
counts: Array<{
|
|
875
|
+
value: string;
|
|
876
|
+
n: number;
|
|
877
|
+
}>;
|
|
878
|
+
/** Over the number values; null when none. */
|
|
879
|
+
numbers: {
|
|
880
|
+
median: number;
|
|
881
|
+
min: number;
|
|
882
|
+
max: number;
|
|
883
|
+
n: number;
|
|
884
|
+
} | null;
|
|
885
|
+
/** Values neither tallied nor summarized: objects, and arrays with no scalar item. */
|
|
886
|
+
others: number;
|
|
887
|
+
};
|
|
888
|
+
/**
|
|
889
|
+
* Each name and cell any attempt observed it on, by name in first-observed order and then by
|
|
890
|
+
* cell, tallied over that cell's attempts.
|
|
891
|
+
*/
|
|
892
|
+
export declare function observedTallies(attempts: ReadonlyArray<Pick<AttemptRecord, 'cell' | 'observations'>>): ObservedTally[];
|
|
893
|
+
/** `drizzle 9 · prisma 3`, `median 4 · range 1–12 · 13 attempts`, as the run output shows a tally. */
|
|
894
|
+
export declare function observedLine(t: ObservedTally): string;
|
|
895
|
+
/**
|
|
896
|
+
* How one name's tally on one cell moved from base to head, or null when it didn't:
|
|
897
|
+
* `prisma 6 → 1 · drizzle 2 → 8` (each value whose count changed, head's most first),
|
|
898
|
+
* `median 4 → 9`, or a side's whole line against `none` when only one side observed it.
|
|
899
|
+
*/
|
|
900
|
+
export declare function observedShift(base: ObservedTally | undefined, head: ObservedTally | undefined): string | null;
|
|
855
901
|
export type PatchFile = {
|
|
856
902
|
path: string;
|
|
857
903
|
status: 'added' | 'deleted' | 'modified' | 'renamed' | 'binary';
|
|
@@ -937,7 +983,7 @@ export type Remark = {
|
|
|
937
983
|
*/
|
|
938
984
|
id: string;
|
|
939
985
|
evalId: string;
|
|
940
|
-
kind: 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret' | 'snapshot-too-large';
|
|
986
|
+
kind: 'too-few-tries' | 'cant-tell-apart' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret' | 'snapshot-too-large';
|
|
941
987
|
/** Fixed by kind (`REMARK_KINDS`). */
|
|
942
988
|
severity: 'concern' | 'suggestion';
|
|
943
989
|
/** One plain sentence: what's wrong. */
|
|
@@ -982,19 +1028,67 @@ export declare const readableRemarks: (doc: RunRemarks, what?: string) => RunRem
|
|
|
982
1028
|
/** A capture's `<name>.capture.json` as the studio sees it; one from a newer reeval throws. */
|
|
983
1029
|
export declare const readableCapturePage: (page: CapturePageFile, what?: string) => CapturePageFile;
|
|
984
1030
|
/**
|
|
985
|
-
*
|
|
986
|
-
*
|
|
987
|
-
*
|
|
988
|
-
*
|
|
989
|
-
* the time when nothing changed.
|
|
1031
|
+
* The runs a side a compare defaults to and the plan suggests. It doesn't gate the effect: a lift
|
|
1032
|
+
* always reads as its observed direction, and whether its interval excludes 0 decides whether it
|
|
1033
|
+
* holds. At 4 a side, a 4/4 against 0/4 split happens by chance about 7% of the time when nothing
|
|
1034
|
+
* changed.
|
|
990
1035
|
*/
|
|
991
|
-
export declare const
|
|
1036
|
+
export declare const COMPARE_RUNS = 4;
|
|
1037
|
+
type MeasureName = 'pass' | 'check' | 'judge' | 'question' | 'tools' | 'agentMs' | 'tokens';
|
|
992
1038
|
/**
|
|
993
|
-
*
|
|
994
|
-
*
|
|
995
|
-
*
|
|
1039
|
+
* `better` and `worse` are the measure's own test holding in that direction (for tool calls,
|
|
1040
|
+
* agent time and tokens, fewer is better); `leans` is a difference the test doesn't hold that
|
|
1041
|
+
* passes the lean screen (`screened`); `same` is every other difference, or none.
|
|
996
1042
|
*/
|
|
997
|
-
|
|
1043
|
+
type MeasureVerdict = 'better' | 'worse' | 'same' | 'leans';
|
|
1044
|
+
type Measure = {
|
|
1045
|
+
measure: MeasureName;
|
|
1046
|
+
/** The check's label, the judge's check, the question; null for the rest. */
|
|
1047
|
+
label: string | null;
|
|
1048
|
+
/**
|
|
1049
|
+
* Each side's value: a pass rate (0–1), a mean judge score, a question's decisive pairs won, or
|
|
1050
|
+
* a median count, milliseconds or tokens.
|
|
1051
|
+
*/
|
|
1052
|
+
variant: number;
|
|
1053
|
+
control: number;
|
|
1054
|
+
/**
|
|
1055
|
+
* The difference: pass-rate points (-1–1) for pass rate and checks, the mean score's difference
|
|
1056
|
+
* for a judge, the variant's share of decisive pairs for a question, and variant over control
|
|
1057
|
+
* for effort (null when the control's is 0).
|
|
1058
|
+
*/
|
|
1059
|
+
estimate: number | null;
|
|
1060
|
+
/** The 95% interval on pass-rate points; null for the other measures. */
|
|
1061
|
+
lo: number | null;
|
|
1062
|
+
hi: number | null;
|
|
1063
|
+
/** The test's two-sided p; null where the measure's verdict is an interval's (pass rate, checks). */
|
|
1064
|
+
p: number | null;
|
|
1065
|
+
verdict: MeasureVerdict;
|
|
1066
|
+
/** On a lean, which way it leans. */
|
|
1067
|
+
leaning: 'better' | 'worse' | null;
|
|
1068
|
+
/** `5× faster`, `same pass rate`, `judge "tidy" +0.12`. */
|
|
1069
|
+
text: string;
|
|
1070
|
+
/**
|
|
1071
|
+
* On a lean, the runs a side (a case, on a family) or the cases (`unit`) at which a lean this
|
|
1072
|
+
* size would hold; null past `RUNS_TO_HOLD_MAX`, or on any other verdict.
|
|
1073
|
+
*/
|
|
1074
|
+
needed: number | null;
|
|
1075
|
+
unit: 'runs' | 'cases';
|
|
1076
|
+
/**
|
|
1077
|
+
* On a lean whose test can't reach its lean p at this sample at all (two runs a side, or three,
|
|
1078
|
+
* for a rank test): the runs a side or cases (`unit`) it has (`now`) and the fewest at which it
|
|
1079
|
+
* can (`at`). Null otherwise.
|
|
1080
|
+
*/
|
|
1081
|
+
short: {
|
|
1082
|
+
now: number;
|
|
1083
|
+
at: number;
|
|
1084
|
+
} | null;
|
|
1085
|
+
};
|
|
1086
|
+
/**
|
|
1087
|
+
* A variant against its baseline: the direction of their observed pass rates, at any number of
|
|
1088
|
+
* runs. Whether it holds is the interval's to say (`CellResult.interval`), and the
|
|
1089
|
+
* `too-few-tries` remark says how many runs would settle it.
|
|
1090
|
+
*/
|
|
1091
|
+
export type Effect = 'better' | 'worse' | 'same';
|
|
998
1092
|
/**
|
|
999
1093
|
* What an eval's cells compare: `change` when it has variants (or several environments),
|
|
1000
1094
|
* `harness` when it has several harnesses, `both`, or `none`.
|
|
@@ -1021,7 +1115,8 @@ export type SuiteResult = {
|
|
|
1021
1115
|
status: RunRecord['status'];
|
|
1022
1116
|
/**
|
|
1023
1117
|
* The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
|
|
1024
|
-
* an attempt, and `skipped` on an interrupted run.
|
|
1118
|
+
* an attempt, and `skipped` on an interrupted run. `reeval run --json` gives a resume that its
|
|
1119
|
+
* filters left `interrupted` the outcome of the attempts it ran, as its exit code.
|
|
1025
1120
|
*/
|
|
1026
1121
|
outcome: Outcome;
|
|
1027
1122
|
/**
|
|
@@ -1075,6 +1170,11 @@ export type EvalResult = {
|
|
|
1075
1170
|
family: boolean;
|
|
1076
1171
|
/** A placeholder (`evaluate.todo`) that never ran. */
|
|
1077
1172
|
todo: boolean;
|
|
1173
|
+
/**
|
|
1174
|
+
* Hasn't run yet: skipped, but not a todo or declared skipped, in a run that isn't complete. On
|
|
1175
|
+
* a family, every case.
|
|
1176
|
+
*/
|
|
1177
|
+
notRun: boolean;
|
|
1078
1178
|
/** On a family: `fail` when a case failed or its pooled gate did. */
|
|
1079
1179
|
outcome: ResultOutcome;
|
|
1080
1180
|
cells: CellResult[] | null;
|
|
@@ -1086,6 +1186,8 @@ export type EvalResult = {
|
|
|
1086
1186
|
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
1087
1187
|
*/
|
|
1088
1188
|
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
1189
|
+
/** Each name its attempts observed with `c.observe`, tallied per cell; empty on a family. */
|
|
1190
|
+
observations: ObservedTally[];
|
|
1089
1191
|
};
|
|
1090
1192
|
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
1091
1193
|
export type QuestionResult = {
|
|
@@ -1133,6 +1235,8 @@ export type CaseResult = {
|
|
|
1133
1235
|
/** Its place in the case list. */
|
|
1134
1236
|
order: number;
|
|
1135
1237
|
outcome: ResultOutcome;
|
|
1238
|
+
/** Hasn't run yet, as `EvalResult.notRun`. */
|
|
1239
|
+
notRun: boolean;
|
|
1136
1240
|
cells: CellResult[];
|
|
1137
1241
|
};
|
|
1138
1242
|
export type CellResult = {
|
|
@@ -1172,16 +1276,16 @@ export type CellResult = {
|
|
|
1172
1276
|
* attempt (every one an infra error or skipped) lists each as `not run`.
|
|
1173
1277
|
*/
|
|
1174
1278
|
checks: CheckResult[];
|
|
1175
|
-
/** On a variant, against its baseline; null on a control. */
|
|
1279
|
+
/** On a variant, the direction of its pass rate against its baseline's; null on a control or with nothing counted on a side. */
|
|
1176
1280
|
effect: Effect | null;
|
|
1177
1281
|
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1178
1282
|
interval: Interval | null;
|
|
1179
1283
|
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1180
1284
|
minLift: number | null;
|
|
1181
|
-
/** Medians over the counted attempts; null below `
|
|
1285
|
+
/** Medians over the counted attempts; null below `COMPARE_RUNS` of them. */
|
|
1182
1286
|
effort: EffortCounters | null;
|
|
1183
|
-
/** On a variant,
|
|
1184
|
-
|
|
1287
|
+
/** On a variant, every measure against its baseline (`cellMeasures`); null on a control. */
|
|
1288
|
+
measures: Measure[] | null;
|
|
1185
1289
|
/** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
|
|
1186
1290
|
errors: CellError[];
|
|
1187
1291
|
/**
|
|
@@ -1220,7 +1324,7 @@ export type CheckResult = {
|
|
|
1220
1324
|
order: number;
|
|
1221
1325
|
outcome: CheckOutcome;
|
|
1222
1326
|
/**
|
|
1223
|
-
* On a variant, this check against the baseline's (
|
|
1327
|
+
* On a variant, this check against the baseline's (null when either side never answered it);
|
|
1224
1328
|
* null on a control or without one.
|
|
1225
1329
|
*/
|
|
1226
1330
|
effect: Effect | null;
|
|
@@ -1284,21 +1388,22 @@ export type PooledArm = {
|
|
|
1284
1388
|
/** Cases with counted attempts on both sides. */
|
|
1285
1389
|
pairs: number;
|
|
1286
1390
|
interval: Interval;
|
|
1391
|
+
/** The pooled lift's verdict: `insufficient` below `PAIRED_MIN` discordant pairs or cases. */
|
|
1392
|
+
verdict: FamilyVerdict;
|
|
1287
1393
|
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
1288
1394
|
minLift: number | null;
|
|
1289
1395
|
outcome: 'pass' | 'fail';
|
|
1396
|
+
/** Every measure over the cases (`familyMeasures`). */
|
|
1397
|
+
measures: Measure[];
|
|
1290
1398
|
};
|
|
1291
1399
|
type Side = {
|
|
1292
1400
|
pass: number;
|
|
1293
1401
|
counted: number;
|
|
1294
1402
|
};
|
|
1295
|
-
/**
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
export declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
|
|
1300
|
-
/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
|
|
1301
|
-
export declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
|
|
1403
|
+
/** The direction of `variant`'s pass rate against `control`'s; null when a side counted none. */
|
|
1404
|
+
export declare function effectOf(variant: Side, control: Side): Effect | null;
|
|
1405
|
+
/** The check's direction on the variant against the baseline; null when a side never answered it. */
|
|
1406
|
+
export declare const checkEffect: (variant: CheckTally, control: CheckTally) => Effect | null;
|
|
1302
1407
|
/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
|
|
1303
1408
|
export declare const factSegment: (s: string) => string;
|
|
1304
1409
|
export declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
|