reeval 0.17.2 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/{adapter-DyvDj4ZN.js → adapter-CptK032x.js} +2 -2
- package/dist/{adapter-DyvDj4ZN.js.map → adapter-CptK032x.js.map} +1 -1
- package/dist/{agent-v036GEgG.js → agent-B63OzUX9.js} +6 -6
- package/dist/{agent-v036GEgG.js.map → agent-B63OzUX9.js.map} +1 -1
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-sgqwfjtn.js} +4 -4
- package/dist/app-files-sgqwfjtn.js.map +1 -0
- package/dist/assemble-CxBHkavM.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-Me7Whp-y.js} +58 -194
- package/dist/assemble-Me7Whp-y.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
- package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-Cik0SoIn.js} +17 -9
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
- package/dist/{capture-Bdcx9Gap.js → capture-C3RmjPph.js} +392 -613
- package/dist/capture-C3RmjPph.js.map +1 -0
- package/dist/capture-YPwxp_iK.js +1 -0
- package/dist/{cli-CGd51thd.js → cli-BBOlXKPG.js} +1075 -457
- package/dist/cli-BBOlXKPG.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
- package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
- package/dist/client-tZ2WBDxf.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-CFQ9c1hk.js} +13 -13
- package/dist/{command-TLmY-rYp.js.map → command-CFQ9c1hk.js.map} +1 -1
- package/dist/{command-CtTdCF3B.js → command-DhiwsZl1.js} +9 -9
- package/dist/{command-CtTdCF3B.js.map → command-DhiwsZl1.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-fO0BFsTq.js} +102 -37
- package/dist/command-fO0BFsTq.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-qGS802uX.js} +16 -14
- package/dist/command-qGS802uX.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CC60TOT5.js} +64 -18
- package/dist/commands-CC60TOT5.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-xYTMNTPv.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-C2UtS9vy.js} +718 -76
- package/dist/config-C2UtS9vy.js.map +1 -0
- package/dist/{contrasts-DtSX8E_V.js → contrasts-BjOg7wyr.js} +3 -3
- package/dist/{contrasts-DtSX8E_V.js.map → contrasts-BjOg7wyr.js.map} +1 -1
- package/dist/cost-BJU0MAji.js +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BTK_apL-.js} +2 -2
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
- package/dist/{db-sHGpaLo9.js → db-C5SZ0510.js} +72 -14
- package/dist/db-C5SZ0510.js.map +1 -0
- package/dist/db-puzLzX4k.js +2 -0
- package/dist/{default-effort-BZRa-SQP.js → default-effort-BVmx7r7L.js} +8 -5
- package/dist/default-effort-BVmx7r7L.js.map +1 -0
- package/dist/{default-effort-fpdbYVKm.js → default-effort-FEidNAIi.js} +1 -1
- package/dist/{docker-CWuSRj7a.js → docker-B2F8-PGC.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-B2F8-PGC.js.map} +1 -1
- package/dist/{docker-CIzHzwlC.js → docker-BlSopHpZ.js} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-Dto8ROCq.js} +5 -5
- package/dist/{docker--2ytOHcu.js.map → docker-Dto8ROCq.js.map} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-BcvfKpBX.js} +20 -20
- package/dist/{doctor-eE65YpEv.js.map → doctor-BcvfKpBX.js.map} +1 -1
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-CC1RV21n.js} +3 -3
- package/dist/{environment-Cy3FVLR5.js.map → environment-CC1RV21n.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-CzxdgFBW.js} +18 -10
- package/dist/eval-source-CzxdgFBW.js.map +1 -0
- package/dist/evaluate-DToSB2bt.d.ts +101 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-NTNJ8T9H.js} +129 -51
- package/dist/evaluate-NTNJ8T9H.js.map +1 -0
- package/dist/{files-tnMgu7-X.js → files-CZXrkPja.js} +5 -5
- package/dist/{files-tnMgu7-X.js.map → files-CZXrkPja.js.map} +1 -1
- package/dist/files-DFtuRQJx.js +1 -0
- package/dist/{files-CQU08uh4.js → files-DsXULUi5.js} +3 -3
- package/dist/{files-CQU08uh4.js.map → files-DsXULUi5.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-aAzoGNCB.js} +4 -4
- package/dist/{files-CIX-pLl4.js.map → files-aAzoGNCB.js.map} +1 -1
- package/dist/files-oOYNR7ZH.js +1 -0
- package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
- package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-C1MB5bjr.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-C1MB5bjr.js.map} +1 -1
- package/dist/harness-DGsRZDSo.js +105 -0
- package/dist/harness-DGsRZDSo.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
- package/dist/harness-DYSC4zMh.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-BVZHDf_z.js → images-CNayXCp2.js} +8 -8
- package/dist/{images-BVZHDf_z.js.map → images-CNayXCp2.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5 -5
- package/dist/init-8o6W3_fn.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-D5HtmyMI.js} +5 -5
- package/dist/{init-BWnMbUnc.js.map → init-D5HtmyMI.js.map} +1 -1
- package/dist/ink-list-Ca-5dunO.js +184 -0
- package/dist/ink-list-Ca-5dunO.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-DrGXF9dT.js} +3 -3
- package/dist/{invocations-BGRs1r_C.js.map → invocations-DrGXF9dT.js.map} +1 -1
- package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
- package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-m-mb0zv8.js +246 -0
- package/dist/judgments-m-mb0zv8.js.map +1 -0
- package/dist/list-CM_YSJXq.js +374 -0
- package/dist/list-CM_YSJXq.js.map +1 -0
- package/dist/{live-d_5a5iUv.js → live-B71o2TkX.js} +21 -3
- package/dist/live-B71o2TkX.js.map +1 -0
- package/dist/live-DQc8jr6o.js +1 -0
- package/dist/{live-ColieOai.js → live-uRURFDMZ.js} +7 -7
- package/dist/{live-ColieOai.js.map → live-uRURFDMZ.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
- package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-BxBWFOZn.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-BxBWFOZn.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-CXVVB4RK.js} +1 -1
- package/dist/{model-DOiCiMCD.js → model-Ce9l9_rF.js} +19 -255
- package/dist/model-Ce9l9_rF.js.map +1 -0
- package/dist/model-Cxzo4cjw.js +1 -0
- package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
- package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
- package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
- package/dist/{patch-DvLr8wCZ.js → patch-ug5HpMOj.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-ug5HpMOj.js.map} +1 -1
- package/dist/{plan-CIfOFhth.js → plan-DqnzfDH1.js} +15 -14
- package/dist/plan-DqnzfDH1.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-B2KQ0y3P.js} +4 -4
- package/dist/{plugins-BEs2Am3L.js.map → plugins-B2KQ0y3P.js.map} +1 -1
- package/dist/{preflight-BAIRsTpl.js → preflight-DVXEvzFU.js} +1 -1
- package/dist/{preflight-50Qw5e5K.js → preflight-DyzmvK7i.js} +3 -3
- package/dist/{preflight-50Qw5e5K.js.map → preflight-DyzmvK7i.js.map} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-21WMQZyt.js} +7 -7
- package/dist/{prepare-8XhzhLHX.js.map → prepare-21WMQZyt.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-BcSRa7-3.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
- package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-iRxjp4Pu.js} +4 -4
- package/dist/{provenance-Cc8fFE7Z.js.map → provenance-iRxjp4Pu.js.map} +1 -1
- package/dist/{providers-CP_bBtYm.js → providers-D0ob_tra.js} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-vXMXuwn-.js} +5 -5
- package/dist/{providers-DZSTlC9R.js.map → providers-vXMXuwn-.js.map} +1 -1
- package/dist/{query-DXpxaaTH.js → query-BhaKQ07P.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-BhaKQ07P.js.map} +1 -1
- package/dist/query-CvgUsvep.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-BzdLdV3C.js} +39 -9
- package/dist/questions-BzdLdV3C.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-CauU408t.js} +231 -25
- package/dist/remarks-CauU408t.js.map +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/{remarks-DeZZT8PK.js → remarks-NMgPd30U.js} +4 -3
- package/dist/{remarks-DeZZT8PK.js.map → remarks-NMgPd30U.js.map} +1 -1
- package/dist/{remote-DZ00A8V_.js → remote-Dwy3fyo3.js} +5 -5
- package/dist/{remote-DZ00A8V_.js.map → remote-Dwy3fyo3.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-B1zMnPwK.js} +7 -7
- package/dist/{resolve-B_JlQ0La.js.map → resolve-B1zMnPwK.js.map} +1 -1
- package/dist/resolve-BIIeVT12.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-CfC0id_u.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-CfC0id_u.js.map} +1 -1
- package/dist/{resume-CexsWuGK.js → resume-Dkhrpjow.js} +123 -8
- package/dist/resume-Dkhrpjow.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-DcvQcr36.js} +2 -2
- package/dist/{runtime-CSZ1hjlN.js.map → runtime-DcvQcr36.js.map} +1 -1
- package/dist/{sandbox-B5W8lLL1.js → sandbox-8zCrtuCf.js} +2 -2
- package/dist/{sandbox-B5W8lLL1.js.map → sandbox-8zCrtuCf.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-0tuj8VSa.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Dplq8g0r.js} +7 -7
- package/dist/{server-BfN6q2lH.js.map → server-Dplq8g0r.js.map} +1 -1
- package/dist/{source-BD6Bg0Tr.js → source-NWHqn5-o.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-NWHqn5-o.js.map} +1 -1
- package/dist/source-llwLewOU.js +1 -0
- package/dist/{src-BC2WXULN.js → src-B9oE_mNS.js} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-BQ6BiXoc.js} +19 -11
- package/dist/{src-Cpy9MNS3.js.map → src-BQ6BiXoc.js.map} +1 -1
- package/dist/{src-Dp-fWOfG.js → src-BQXdf-Gw.js} +2 -2
- package/dist/{src-Dp-fWOfG.js.map → src-BQXdf-Gw.js.map} +1 -1
- package/dist/{src-DurP--_D.js → src-BXnv0ViU.js} +1 -1
- package/dist/src-BiS8YNN_.js +3 -0
- package/dist/{src-DVaAKCsH.js → src-BkH4GOxu.js} +1 -1
- package/dist/{src-CM7xdJFA.js → src-C-tH5ebd.js} +1 -1
- package/dist/{src-Cx0HJIJ-.js → src-CUpoVVb8.js} +1 -1
- package/dist/{src-C9elednc.js → src-Cg1bFs9g.js} +1 -1
- package/dist/{src-BKDZO3ra.js → src-DV_UorZv.js} +1 -1
- package/dist/{src-CWEGknZo.js → src-XxUpr2sa.js} +1 -1
- package/dist/{state-CdSjXIfE.js → state-DDCGUhva.js} +5 -5
- package/dist/{state-CdSjXIfE.js.map → state-DDCGUhva.js.map} +1 -1
- package/dist/state-DZbAO95a.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-M8_7xio7.js +1294 -0
- package/dist/suite-M8_7xio7.js.map +1 -0
- package/dist/suite-u2KJnGpY.js +2 -0
- package/dist/{summary-zP3cCXDN.js → summary-Uzc9osfV.js} +47 -13
- package/dist/summary-Uzc9osfV.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
- package/dist/{types-BHR1lGPh.d.ts → types-DGvexkcp.d.ts} +71 -63
- package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
- package/dist/vertex-BLQMayc1.js.map +1 -0
- package/dist/vertex-h_peaZTN.js +1 -0
- package/dist/view/index.d.ts +140 -35
- package/dist/view/index.js +8 -7
- package/dist/{views-CKoN5NAx.js → views-BISrsIjw.js} +6 -6
- package/dist/{views-CKoN5NAx.js.map → views-BISrsIjw.js.map} +1 -1
- package/dist/views-Bfab8fM9.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-DJD_nHkS.js} +5 -5
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
- package/dist/workspace-B8EOCuxl.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-BO1VhUR9.js} +5 -5
- package/dist/{workspace-Cho30qlM.js.map → workspace-BO1VhUR9.js.map} +1 -1
- package/package.json +3 -1
- package/skills/eval-rubric/SKILL.md +22 -3
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,23 +1,24 @@
|
|
|
1
1
|
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
2
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
3
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
|
|
4
|
-
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-
|
|
5
|
-
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-
|
|
6
|
-
import {
|
|
3
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty$1, m as TIMER_LIMIT, n as dataProblem, o as genericProblem, p as MAX_TIMER_MS, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
|
|
4
|
+
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-CC1RV21n.js";
|
|
5
|
+
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-B2KQ0y3P.js";
|
|
6
|
+
import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DYSC4zMh.js";
|
|
7
7
|
import { a as fromCwd, l as sha256 } from "./ids-DuK6xl4w.js";
|
|
8
|
-
import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-
|
|
9
|
-
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-
|
|
10
|
-
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-
|
|
11
|
-
import { r as badCharacterAt } from "./redact-
|
|
12
|
-
import {
|
|
13
|
-
import { t as inCI } from "./env-
|
|
8
|
+
import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-Co7plCea.js";
|
|
9
|
+
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-DYOF69Oe.js";
|
|
10
|
+
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-BLQMayc1.js";
|
|
11
|
+
import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
|
|
12
|
+
import { o as withoutBom } from "./text-B8-W3Cvn.js";
|
|
13
|
+
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
14
14
|
import { u as ownFileURL } from "./root-Co0jhOT5.js";
|
|
15
|
-
import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-
|
|
15
|
+
import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-BQ6BiXoc.js";
|
|
16
16
|
import { t as InfraError } from "./types-C4OkYhZg.js";
|
|
17
|
-
import { n as gateLabel, u as pairFactors } from "./gate-
|
|
18
|
-
import "./
|
|
19
|
-
import
|
|
20
|
-
import {
|
|
17
|
+
import { n as gateLabel, u as pairFactors } from "./gate-Cp_pQoEe.js";
|
|
18
|
+
import { u as opencodeModelError } from "./harness-DGsRZDSo.js";
|
|
19
|
+
import "./stats-C18W4xPX.js";
|
|
20
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CptK032x.js";
|
|
21
|
+
import { t as docker } from "./sandbox-8zCrtuCf.js";
|
|
21
22
|
import { existsSync, readFileSync } from "node:fs";
|
|
22
23
|
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
23
24
|
import { dirname, join, resolve, win32 } from "node:path";
|
|
@@ -88,20 +89,22 @@ const sharedInputFields = {
|
|
|
88
89
|
task: z.boolean().optional(),
|
|
89
90
|
output: z.boolean().optional(),
|
|
90
91
|
files: z.boolean().optional(),
|
|
91
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
92
|
+
diff: z.union([z.boolean(), z.array(nonEmpty$1()).min(1)]).optional(),
|
|
92
93
|
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
93
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
94
|
+
checks: z.union([z.boolean(), z.array(nonEmpty$1())]).optional(),
|
|
95
|
+
observations: z.union([z.boolean(), z.array(nonEmpty$1())]).optional()
|
|
94
96
|
};
|
|
95
97
|
const turnsSchema = z.enum(["last", "all"]);
|
|
96
|
-
const { task, output, files, diff, transcript, checks } = sharedInputFields;
|
|
98
|
+
const { task, output, files, diff, transcript, checks, observations } = sharedInputFields;
|
|
97
99
|
const judgeInputsSchema = z.strictObject({
|
|
98
100
|
task,
|
|
99
101
|
output,
|
|
100
102
|
files,
|
|
101
103
|
diff,
|
|
102
|
-
contents: z.array(nonEmpty()).optional(),
|
|
104
|
+
contents: z.array(nonEmpty$1()).optional(),
|
|
103
105
|
transcript,
|
|
104
106
|
checks,
|
|
107
|
+
observations,
|
|
105
108
|
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
106
109
|
});
|
|
107
110
|
const judgeSubjectSchema = z.strictObject({
|
|
@@ -149,7 +152,8 @@ const VALUES$1 = {
|
|
|
149
152
|
contents: "a list of paths or patterns",
|
|
150
153
|
transcript: "true, false, 'tools' or 'full'",
|
|
151
154
|
turns: "'last' or 'all'",
|
|
152
|
-
checks: "true, false or a list of commands"
|
|
155
|
+
checks: "true, false or a list of commands",
|
|
156
|
+
observations: "true, false or a list of names"
|
|
153
157
|
};
|
|
154
158
|
function distance(a, b) {
|
|
155
159
|
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
@@ -239,7 +243,7 @@ function judgeProblem(judge, site) {
|
|
|
239
243
|
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
240
244
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
241
245
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
242
|
-
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
246
|
+
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty$1()).min(1)]);
|
|
243
247
|
const questionInputFields = {
|
|
244
248
|
...sharedInputFields,
|
|
245
249
|
artifacts: questionArtifactsSchema.optional()
|
|
@@ -257,9 +261,9 @@ const unique = (list, ctx) => {
|
|
|
257
261
|
});
|
|
258
262
|
};
|
|
259
263
|
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
260
|
-
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
264
|
+
const choicesSchema = z.array(nonEmpty$1()).min(2).superRefine(unique);
|
|
261
265
|
const questionSchema = z.strictObject({
|
|
262
|
-
question: nonEmpty(),
|
|
266
|
+
question: nonEmpty$1(),
|
|
263
267
|
model: questionModelSchema.optional(),
|
|
264
268
|
across: acrossSchema.optional(),
|
|
265
269
|
choices: choicesSchema.optional(),
|
|
@@ -296,7 +300,8 @@ const VALUES = {
|
|
|
296
300
|
diff: "true, false or a non-empty list of patterns",
|
|
297
301
|
transcript: "true, false, 'tools' or 'full'",
|
|
298
302
|
turns: "'last' or 'all'",
|
|
299
|
-
checks: "true, false or a list of commands"
|
|
303
|
+
checks: "true, false or a list of commands",
|
|
304
|
+
observations: "true, false or a list of names"
|
|
300
305
|
};
|
|
301
306
|
const describe$2 = (value) => {
|
|
302
307
|
if (Array.isArray(value)) return `a list of ${value.length}`;
|
|
@@ -344,8 +349,8 @@ function acrossProblem(question, across, where) {
|
|
|
344
349
|
function choicesProblem(question, choices, where) {
|
|
345
350
|
if (choicesSchema.safeParse(choices).success) return void 0;
|
|
346
351
|
const at = `questions "${question}" choices in ${where}`;
|
|
347
|
-
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
348
|
-
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty().safeParse(c).success);
|
|
352
|
+
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty$1().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
353
|
+
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty$1().safeParse(c).success);
|
|
349
354
|
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
350
355
|
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
351
356
|
}
|
|
@@ -359,7 +364,7 @@ function questionProblem(item, where) {
|
|
|
359
364
|
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
360
365
|
}
|
|
361
366
|
const { question, model, across, choices } = item;
|
|
362
|
-
if (!nonEmpty().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
367
|
+
if (!nonEmpty$1().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
363
368
|
const text = question;
|
|
364
369
|
if (across !== void 0) {
|
|
365
370
|
const problem = acrossProblem(text, across, where);
|
|
@@ -1137,7 +1142,7 @@ const AGENTS = {
|
|
|
1137
1142
|
},
|
|
1138
1143
|
unreported: cursorUnreported,
|
|
1139
1144
|
mcpServers: cursorMcpServers,
|
|
1140
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1145
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-BXnv0ViU.js")).createCursorSdk({
|
|
1141
1146
|
auth,
|
|
1142
1147
|
...mcpServers ? { mcpServers } : {},
|
|
1143
1148
|
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
@@ -1209,7 +1214,7 @@ const AGENTS = {
|
|
|
1209
1214
|
},
|
|
1210
1215
|
servedModel: geminiServedModel,
|
|
1211
1216
|
mcpServers: antigravityMcpServers,
|
|
1212
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-
|
|
1217
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-XxUpr2sa.js")).createAntigravity({
|
|
1213
1218
|
auth,
|
|
1214
1219
|
...vertex ? { vertex } : {},
|
|
1215
1220
|
...mcpServers ? { mcpServers } : {},
|
|
@@ -1239,7 +1244,7 @@ const AGENTS = {
|
|
|
1239
1244
|
servedModel: geminiServedModel,
|
|
1240
1245
|
forModel: opencodeForModel,
|
|
1241
1246
|
mcpServers: opencodeMcpServers,
|
|
1242
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1247
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BiS8YNN_.js")).createOpenCode({
|
|
1243
1248
|
auth,
|
|
1244
1249
|
model,
|
|
1245
1250
|
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
@@ -1264,14 +1269,14 @@ const AGENTS = {
|
|
|
1264
1269
|
},
|
|
1265
1270
|
servedModel: geminiServedModel,
|
|
1266
1271
|
mcpServers: gooseMcpServers,
|
|
1267
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1272
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C-tH5ebd.js")).createGoose({
|
|
1268
1273
|
...vertex ? { vertex } : {},
|
|
1269
1274
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1270
1275
|
auth,
|
|
1271
|
-
provider: (await import("./src-
|
|
1276
|
+
provider: (await import("./src-C-tH5ebd.js")).AI_GATEWAY_PROVIDER
|
|
1272
1277
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1273
1278
|
auth,
|
|
1274
|
-
provider: (await import("./src-
|
|
1279
|
+
provider: (await import("./src-C-tH5ebd.js")).OPENROUTER_PROVIDER
|
|
1275
1280
|
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1276
1281
|
model,
|
|
1277
1282
|
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1294,13 +1299,13 @@ const AGENTS = {
|
|
|
1294
1299
|
endpoints: GEMINI_ENDPOINTS,
|
|
1295
1300
|
servedModel: geminiServedModel,
|
|
1296
1301
|
mcpServers: piMcpServers,
|
|
1297
|
-
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-
|
|
1302
|
+
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-B9oE_mNS.js")).createPi({
|
|
1298
1303
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1299
1304
|
auth,
|
|
1300
|
-
gateway: (await import("./src-
|
|
1305
|
+
gateway: (await import("./src-B9oE_mNS.js")).PI_AI_GATEWAY
|
|
1301
1306
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1302
1307
|
auth,
|
|
1303
|
-
gateway: (await import("./src-
|
|
1308
|
+
gateway: (await import("./src-B9oE_mNS.js")).PI_OPENROUTER
|
|
1304
1309
|
} : {
|
|
1305
1310
|
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1306
1311
|
...vertex ? { vertex } : {}
|
|
@@ -1335,9 +1340,9 @@ const AGENTS = {
|
|
|
1335
1340
|
},
|
|
1336
1341
|
servedModel: openaiServedModel,
|
|
1337
1342
|
mcpServers: codexMcpServers,
|
|
1338
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1343
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-Cg1bFs9g.js")).createCodex({
|
|
1339
1344
|
auth,
|
|
1340
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-
|
|
1345
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).OPENROUTER_PROVIDER } : {},
|
|
1341
1346
|
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1342
1347
|
...mcpServers ? { mcpServers } : {},
|
|
1343
1348
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
@@ -1358,7 +1363,7 @@ const AGENTS = {
|
|
|
1358
1363
|
...fxForModel("provider/model"),
|
|
1359
1364
|
forModel: fxForModel,
|
|
1360
1365
|
mcpServers: acpMcpServers,
|
|
1361
|
-
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-
|
|
1366
|
+
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-DV_UorZv.js")).createFx({
|
|
1362
1367
|
auth,
|
|
1363
1368
|
model,
|
|
1364
1369
|
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
@@ -1386,7 +1391,7 @@ const AGENTS = {
|
|
|
1386
1391
|
servedModel: xaiServedModel,
|
|
1387
1392
|
mcpServers: acpMcpServers,
|
|
1388
1393
|
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1389
|
-
const grok = await import("./src-
|
|
1394
|
+
const grok = await import("./src-CUpoVVb8.js");
|
|
1390
1395
|
return grok.createGrokBuild({
|
|
1391
1396
|
...vertexChat ? {
|
|
1392
1397
|
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
@@ -1742,6 +1747,644 @@ function resolvedEffortOf(models) {
|
|
|
1742
1747
|
}
|
|
1743
1748
|
return out;
|
|
1744
1749
|
}
|
|
1750
|
+
// src/contract/attempt.ts
|
|
1751
|
+
const nullableNumber = z.number().nullable();
|
|
1752
|
+
const tokenUsageSchema = z.strictObject({
|
|
1753
|
+
input: z.number(),
|
|
1754
|
+
output: z.number(),
|
|
1755
|
+
cacheRead: z.number(),
|
|
1756
|
+
cacheWrite: z.number(),
|
|
1757
|
+
reasoning: nullableNumber
|
|
1758
|
+
});
|
|
1759
|
+
const fileChangeSchema = z.strictObject({
|
|
1760
|
+
path: z.string(),
|
|
1761
|
+
status: z.enum([
|
|
1762
|
+
"added",
|
|
1763
|
+
"modified",
|
|
1764
|
+
"deleted"
|
|
1765
|
+
])
|
|
1766
|
+
});
|
|
1767
|
+
const diffSchema = z.strictObject({
|
|
1768
|
+
patch: z.string(),
|
|
1769
|
+
files: z.array(fileChangeSchema)
|
|
1770
|
+
});
|
|
1771
|
+
const turnStatusSchema = z.enum([
|
|
1772
|
+
"completed",
|
|
1773
|
+
"aborted",
|
|
1774
|
+
"error"
|
|
1775
|
+
]);
|
|
1776
|
+
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
|
|
1777
|
+
const at$1 = {
|
|
1778
|
+
ts: z.string(),
|
|
1779
|
+
atMs: z.number()
|
|
1780
|
+
};
|
|
1781
|
+
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
1782
|
+
z.strictObject({
|
|
1783
|
+
...at$1,
|
|
1784
|
+
type: z.enum(["text", "reasoning"]),
|
|
1785
|
+
text: z.string(),
|
|
1786
|
+
durationMs: nullableNumber
|
|
1787
|
+
}),
|
|
1788
|
+
z.strictObject({
|
|
1789
|
+
...at$1,
|
|
1790
|
+
type: z.literal("file-edit"),
|
|
1791
|
+
toolCallId: z.string(),
|
|
1792
|
+
path: z.string(),
|
|
1793
|
+
kind: z.enum([
|
|
1794
|
+
"create",
|
|
1795
|
+
"modify",
|
|
1796
|
+
"write",
|
|
1797
|
+
"delete"
|
|
1798
|
+
]),
|
|
1799
|
+
added: nullableNumber,
|
|
1800
|
+
removed: nullableNumber,
|
|
1801
|
+
durationMs: nullableNumber,
|
|
1802
|
+
stalled: z.boolean().optional(),
|
|
1803
|
+
subagent: z.string().optional()
|
|
1804
|
+
}),
|
|
1805
|
+
z.strictObject({
|
|
1806
|
+
...at$1,
|
|
1807
|
+
type: z.literal("web-search"),
|
|
1808
|
+
toolCallId: z.string(),
|
|
1809
|
+
query: z.string(),
|
|
1810
|
+
references: nullableNumber,
|
|
1811
|
+
ok: z.boolean().nullable().optional(),
|
|
1812
|
+
durationMs: nullableNumber,
|
|
1813
|
+
stalled: z.boolean().optional(),
|
|
1814
|
+
subagent: z.string().optional()
|
|
1815
|
+
}),
|
|
1816
|
+
z.strictObject({
|
|
1817
|
+
...at$1,
|
|
1818
|
+
type: z.literal("web-fetch"),
|
|
1819
|
+
toolCallId: z.string(),
|
|
1820
|
+
url: z.string(),
|
|
1821
|
+
ok: z.boolean().nullable(),
|
|
1822
|
+
durationMs: nullableNumber,
|
|
1823
|
+
stalled: z.boolean().optional(),
|
|
1824
|
+
subagent: z.string().optional()
|
|
1825
|
+
}),
|
|
1826
|
+
z.strictObject({
|
|
1827
|
+
...at$1,
|
|
1828
|
+
type: z.literal("tool"),
|
|
1829
|
+
toolCallId: z.string(),
|
|
1830
|
+
tool: z.string(),
|
|
1831
|
+
input: z.unknown(),
|
|
1832
|
+
output: z.unknown(),
|
|
1833
|
+
ok: z.boolean().nullable(),
|
|
1834
|
+
durationMs: nullableNumber,
|
|
1835
|
+
stalled: z.boolean().optional(),
|
|
1836
|
+
subagent: z.string().optional()
|
|
1837
|
+
}),
|
|
1838
|
+
z.strictObject({
|
|
1839
|
+
...at$1,
|
|
1840
|
+
type: z.literal("error"),
|
|
1841
|
+
message: z.string()
|
|
1842
|
+
}),
|
|
1843
|
+
z.strictObject({
|
|
1844
|
+
...at$1,
|
|
1845
|
+
type: z.literal("retry"),
|
|
1846
|
+
message: z.string()
|
|
1847
|
+
})
|
|
1848
|
+
]);
|
|
1849
|
+
const turnDataSchema = z.strictObject({
|
|
1850
|
+
prompt: z.string(),
|
|
1851
|
+
text: z.string(),
|
|
1852
|
+
status: turnStatusSchema,
|
|
1853
|
+
startedAt: z.string(),
|
|
1854
|
+
durationMs: z.number(),
|
|
1855
|
+
transcript: z.array(transcriptEventSchema),
|
|
1856
|
+
usage: tokenUsageSchema.nullable(),
|
|
1857
|
+
model: z.string().nullable(),
|
|
1858
|
+
unreported: z.array(webEventTypeSchema),
|
|
1859
|
+
diff: diffSchema
|
|
1860
|
+
});
|
|
1861
|
+
const execResultSchema = z.strictObject({
|
|
1862
|
+
exitCode: z.number().int(),
|
|
1863
|
+
stdout: z.string(),
|
|
1864
|
+
stderr: z.string()
|
|
1865
|
+
});
|
|
1866
|
+
const execRetrySchema = z.strictObject({
|
|
1867
|
+
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
1868
|
+
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
1869
|
+
});
|
|
1870
|
+
/** A background command has no exit for `retries` to wait on. */
|
|
1871
|
+
const noBackgroundRetry = (options, ctx) => {
|
|
1872
|
+
if (!options.background) return;
|
|
1873
|
+
for (const key of ["retries", "interval"]) {
|
|
1874
|
+
if (options[key] === void 0) continue;
|
|
1875
|
+
ctx.addIssue({
|
|
1876
|
+
code: "custom",
|
|
1877
|
+
path: [key],
|
|
1878
|
+
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
1879
|
+
});
|
|
1880
|
+
}
|
|
1881
|
+
};
|
|
1882
|
+
const execOptionsSchema = z.strictObject({
|
|
1883
|
+
background: z.boolean().optional(),
|
|
1884
|
+
...execRetrySchema.shape
|
|
1885
|
+
}).superRefine(noBackgroundRetry);
|
|
1886
|
+
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
1887
|
+
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
1888
|
+
const assertionRecordSchema = z.strictObject({
|
|
1889
|
+
label: z.string(),
|
|
1890
|
+
labelled: z.boolean(),
|
|
1891
|
+
matcher: z.string(),
|
|
1892
|
+
status: z.enum([
|
|
1893
|
+
"pass",
|
|
1894
|
+
"fail",
|
|
1895
|
+
"error"
|
|
1896
|
+
]),
|
|
1897
|
+
soft: z.boolean(),
|
|
1898
|
+
at: z.string().optional(),
|
|
1899
|
+
message: z.string().optional(),
|
|
1900
|
+
durationMs: z.number().optional()
|
|
1901
|
+
});
|
|
1902
|
+
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
1903
|
+
const captureConsoleMessageSchema = z.strictObject({
|
|
1904
|
+
level: z.enum([
|
|
1905
|
+
"error",
|
|
1906
|
+
"warning",
|
|
1907
|
+
"info",
|
|
1908
|
+
"log",
|
|
1909
|
+
"debug"
|
|
1910
|
+
]),
|
|
1911
|
+
text: z.string(),
|
|
1912
|
+
source: z.string(),
|
|
1913
|
+
url: z.string().optional()
|
|
1914
|
+
});
|
|
1915
|
+
const captureFailedRequestSchema = z.strictObject({
|
|
1916
|
+
method: z.string(),
|
|
1917
|
+
url: z.string(),
|
|
1918
|
+
status: z.number().optional(),
|
|
1919
|
+
error: z.string().optional()
|
|
1920
|
+
});
|
|
1921
|
+
const capturePageFields = {
|
|
1922
|
+
status: z.number().optional(),
|
|
1923
|
+
finalUrl: z.string(),
|
|
1924
|
+
console: z.array(captureConsoleMessageSchema),
|
|
1925
|
+
errors: z.array(z.string()),
|
|
1926
|
+
failedRequests: z.array(captureFailedRequestSchema),
|
|
1927
|
+
dropped: z.number().optional()
|
|
1928
|
+
};
|
|
1929
|
+
z.strictObject(capturePageFields);
|
|
1930
|
+
const captureSchema = z.strictObject({
|
|
1931
|
+
name: z.string(),
|
|
1932
|
+
port: z.number(),
|
|
1933
|
+
path: z.string(),
|
|
1934
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]),
|
|
1935
|
+
file: z.strictObject({
|
|
1936
|
+
path: z.string(),
|
|
1937
|
+
sha256: z.string(),
|
|
1938
|
+
bytes: z.number()
|
|
1939
|
+
}),
|
|
1940
|
+
...capturePageFields
|
|
1941
|
+
});
|
|
1942
|
+
const judgeArtifactRefSchema = z.strictObject({
|
|
1943
|
+
name: z.string(),
|
|
1944
|
+
path: z.string(),
|
|
1945
|
+
sha256: z.string(),
|
|
1946
|
+
bytes: z.number(),
|
|
1947
|
+
mediaType: z.literal("image/png")
|
|
1948
|
+
});
|
|
1949
|
+
const verdictFields = {
|
|
1950
|
+
threshold: z.number(),
|
|
1951
|
+
question: z.string(),
|
|
1952
|
+
model: z.string(),
|
|
1953
|
+
gateway: gatewayNameSchema.optional(),
|
|
1954
|
+
artifacts: z.array(judgeArtifactRefSchema).optional()
|
|
1955
|
+
};
|
|
1956
|
+
const verdictSchema = z.strictObject({
|
|
1957
|
+
verdict: z.enum(["pass", "fail"]),
|
|
1958
|
+
score: z.number(),
|
|
1959
|
+
...verdictFields,
|
|
1960
|
+
resolvedModel: z.string().optional(),
|
|
1961
|
+
upstream: z.string().optional(),
|
|
1962
|
+
costUsd: z.number().optional()
|
|
1963
|
+
});
|
|
1964
|
+
const judgeFailureSchema = z.strictObject({
|
|
1965
|
+
verdict: z.literal("error"),
|
|
1966
|
+
score: z.null(),
|
|
1967
|
+
...verdictFields,
|
|
1968
|
+
error: z.string()
|
|
1969
|
+
});
|
|
1970
|
+
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
1971
|
+
const patternSchema = z.strictObject({
|
|
1972
|
+
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
1973
|
+
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
1974
|
+
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
1975
|
+
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
1976
|
+
function hasAnsweredPattern(value) {
|
|
1977
|
+
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
1978
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1979
|
+
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
1980
|
+
}
|
|
1981
|
+
const answeredPatternSchema = z.strictObject({
|
|
1982
|
+
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
1983
|
+
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
1984
|
+
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
1985
|
+
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
1986
|
+
const matcherArgSchema = z.union([
|
|
1987
|
+
patternSchema,
|
|
1988
|
+
answeredPatternSchema,
|
|
1989
|
+
jsonValueSchema
|
|
1990
|
+
]).superRefine((value, ctx) => {
|
|
1991
|
+
if (!isPatternLike(value)) return;
|
|
1992
|
+
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
1993
|
+
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
1994
|
+
const matcherSubjectSchema = z.union([
|
|
1995
|
+
z.strictObject({ turn: turnIndex }),
|
|
1996
|
+
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
1997
|
+
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
1998
|
+
z.strictObject({
|
|
1999
|
+
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
2000
|
+
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
2001
|
+
}),
|
|
2002
|
+
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
2003
|
+
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
2004
|
+
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
2005
|
+
const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
|
|
2006
|
+
const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
|
|
2007
|
+
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
2008
|
+
const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
|
|
2009
|
+
const captureWaitSchema = z.enum(["load", "idle"]);
|
|
2010
|
+
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
|
|
2011
|
+
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
2012
|
+
const captureCallDataSchema = z.strictObject({
|
|
2013
|
+
name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
|
|
2014
|
+
path: z.string().optional(),
|
|
2015
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
2016
|
+
timeout: durationSchema.optional(),
|
|
2017
|
+
wait: waitSchema.optional(),
|
|
2018
|
+
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
2019
|
+
model: z.string().optional()
|
|
2020
|
+
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
2021
|
+
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
2022
|
+
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
2023
|
+
const judgeCallDataSchema = z.strictObject({
|
|
2024
|
+
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
2025
|
+
...judgeFields,
|
|
2026
|
+
...sharedInputFields,
|
|
2027
|
+
contents: z.array(nonEmpty).optional(),
|
|
2028
|
+
turns: turnsSchema.optional(),
|
|
2029
|
+
on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
|
|
2030
|
+
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
2031
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
|
|
2032
|
+
/** An observation's name: one a query, a tally line and a judge's input can carry as it is. */
|
|
2033
|
+
const OBSERVE_NAME = /^[A-Za-z0-9_.:/-]{1,64}$/;
|
|
2034
|
+
const callSiteSchema = z.strictObject({
|
|
2035
|
+
file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
|
|
2036
|
+
line: z.number().int().min(1).describe("From 1.")
|
|
2037
|
+
}).describe("Where the eval made the call, which the record names beside the check's outcome.");
|
|
2038
|
+
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
2039
|
+
z.strictObject({
|
|
2040
|
+
op: z.literal("send"),
|
|
2041
|
+
prompt: z.string().describe("The turn's prompt.")
|
|
2042
|
+
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
2043
|
+
z.strictObject({
|
|
2044
|
+
op: z.literal("exec"),
|
|
2045
|
+
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
2046
|
+
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
2047
|
+
...execRetrySchema.shape
|
|
2048
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
2049
|
+
z.strictObject({
|
|
2050
|
+
op: z.literal("serve"),
|
|
2051
|
+
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
2052
|
+
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
2053
|
+
}).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
|
|
2054
|
+
z.strictObject({
|
|
2055
|
+
op: z.literal("kill"),
|
|
2056
|
+
handle: handleId
|
|
2057
|
+
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
2058
|
+
z.strictObject({
|
|
2059
|
+
op: z.literal("capture"),
|
|
2060
|
+
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
2061
|
+
options: captureCallDataSchema.optional()
|
|
2062
|
+
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
2063
|
+
z.strictObject({
|
|
2064
|
+
op: z.literal("readFile"),
|
|
2065
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
2066
|
+
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
2067
|
+
z.strictObject({
|
|
2068
|
+
op: z.literal("writeFile"),
|
|
2069
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
2070
|
+
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
2071
|
+
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
2072
|
+
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
2073
|
+
z.strictObject({
|
|
2074
|
+
op: z.literal("observe"),
|
|
2075
|
+
name: z.string().regex(OBSERVE_NAME, "must be 1 to 64 letters, digits, _ . : / or -").describe("What the value is of, e.g. `installed`."),
|
|
2076
|
+
value: jsonValueSchema.describe("Any JSON, at most 4 KB once serialized.")
|
|
2077
|
+
}).describe("`c.observe(name, value)`: records a fact about the attempt that never affects its outcome. A name observed again keeps the last value. Fails after the body's `done`."),
|
|
2078
|
+
z.strictObject({
|
|
2079
|
+
op: z.literal("expect"),
|
|
2080
|
+
subject: matcherSubjectSchema,
|
|
2081
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
2082
|
+
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
2083
|
+
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
2084
|
+
soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
|
|
2085
|
+
label: z.string().optional().describe("`expect`'s second argument."),
|
|
2086
|
+
at: callSiteSchema.optional(),
|
|
2087
|
+
poll: z.strictObject({
|
|
2088
|
+
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
2089
|
+
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
2090
|
+
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
2091
|
+
}).superRefine((call, ctx) => {
|
|
2092
|
+
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
2093
|
+
code: "custom",
|
|
2094
|
+
path: ["subject"],
|
|
2095
|
+
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
2096
|
+
});
|
|
2097
|
+
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
2098
|
+
code: "custom",
|
|
2099
|
+
path: ["subject", "result"],
|
|
2100
|
+
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
2101
|
+
});
|
|
2102
|
+
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
2103
|
+
code: "custom",
|
|
2104
|
+
path: ["args"],
|
|
2105
|
+
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
2106
|
+
});
|
|
2107
|
+
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
2108
|
+
z.strictObject({
|
|
2109
|
+
op: z.literal("record"),
|
|
2110
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
2111
|
+
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
2112
|
+
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
2113
|
+
status: z.enum(["pass", "fail"]),
|
|
2114
|
+
message: z.string().optional().describe("Why it failed."),
|
|
2115
|
+
soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
|
|
2116
|
+
at: callSiteSchema.optional()
|
|
2117
|
+
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
2118
|
+
z.strictObject({
|
|
2119
|
+
op: z.literal("judge"),
|
|
2120
|
+
question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
|
|
2121
|
+
options: judgeCallDataSchema
|
|
2122
|
+
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
2123
|
+
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
2124
|
+
const attemptErrorSchema = z.looseObject({
|
|
2125
|
+
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
2126
|
+
message: z.string()
|
|
2127
|
+
}).describe("Why a call failed, in reeval's words.");
|
|
2128
|
+
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
2129
|
+
const attemptRequestSchema = z.strictObject({
|
|
2130
|
+
id,
|
|
2131
|
+
call: attemptCallSchema
|
|
2132
|
+
}).describe("A call, numbered so its reply can find it.");
|
|
2133
|
+
z.union([
|
|
2134
|
+
z.discriminatedUnion("op", [
|
|
2135
|
+
z.strictObject({
|
|
2136
|
+
id,
|
|
2137
|
+
ok: z.literal(true),
|
|
2138
|
+
op: z.literal("send"),
|
|
2139
|
+
turn: turnDataSchema
|
|
2140
|
+
}),
|
|
2141
|
+
z.strictObject({
|
|
2142
|
+
id,
|
|
2143
|
+
ok: z.literal(true),
|
|
2144
|
+
op: z.literal("exec"),
|
|
2145
|
+
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
2146
|
+
}),
|
|
2147
|
+
z.strictObject({
|
|
2148
|
+
id,
|
|
2149
|
+
ok: z.literal(true),
|
|
2150
|
+
op: z.literal("serve"),
|
|
2151
|
+
handle: handleId,
|
|
2152
|
+
port: z.number().describe("The container port it listens on.")
|
|
2153
|
+
}),
|
|
2154
|
+
z.strictObject({
|
|
2155
|
+
id,
|
|
2156
|
+
ok: z.literal(true),
|
|
2157
|
+
op: z.literal("kill")
|
|
2158
|
+
}),
|
|
2159
|
+
z.strictObject({
|
|
2160
|
+
id,
|
|
2161
|
+
ok: z.literal(true),
|
|
2162
|
+
op: z.literal("capture"),
|
|
2163
|
+
capture: captureSchema
|
|
2164
|
+
}),
|
|
2165
|
+
z.strictObject({
|
|
2166
|
+
id,
|
|
2167
|
+
ok: z.literal(true),
|
|
2168
|
+
op: z.literal("readFile"),
|
|
2169
|
+
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
2170
|
+
}),
|
|
2171
|
+
z.strictObject({
|
|
2172
|
+
id,
|
|
2173
|
+
ok: z.literal(true),
|
|
2174
|
+
op: z.literal("writeFile")
|
|
2175
|
+
}),
|
|
2176
|
+
z.strictObject({
|
|
2177
|
+
id,
|
|
2178
|
+
ok: z.literal(true),
|
|
2179
|
+
op: z.literal("observe")
|
|
2180
|
+
}),
|
|
2181
|
+
z.strictObject({
|
|
2182
|
+
id,
|
|
2183
|
+
ok: z.literal(true),
|
|
2184
|
+
op: z.literal("expect"),
|
|
2185
|
+
assertion: assertionRecordSchema
|
|
2186
|
+
}),
|
|
2187
|
+
z.strictObject({
|
|
2188
|
+
id,
|
|
2189
|
+
ok: z.literal(true),
|
|
2190
|
+
op: z.literal("record"),
|
|
2191
|
+
assertion: assertionRecordSchema
|
|
2192
|
+
}),
|
|
2193
|
+
z.strictObject({
|
|
2194
|
+
id,
|
|
2195
|
+
ok: z.literal(true),
|
|
2196
|
+
op: z.literal("judge"),
|
|
2197
|
+
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
2198
|
+
})
|
|
2199
|
+
]),
|
|
2200
|
+
z.strictObject({
|
|
2201
|
+
id,
|
|
2202
|
+
ok: z.literal(true),
|
|
2203
|
+
op: z.literal("expect"),
|
|
2204
|
+
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
2205
|
+
}),
|
|
2206
|
+
z.strictObject({
|
|
2207
|
+
id,
|
|
2208
|
+
ok: z.literal(false),
|
|
2209
|
+
op: z.enum([
|
|
2210
|
+
"send",
|
|
2211
|
+
"exec",
|
|
2212
|
+
"readFile",
|
|
2213
|
+
"writeFile",
|
|
2214
|
+
"observe",
|
|
2215
|
+
"expect",
|
|
2216
|
+
"record",
|
|
2217
|
+
"judge",
|
|
2218
|
+
"serve",
|
|
2219
|
+
"kill",
|
|
2220
|
+
"capture"
|
|
2221
|
+
]),
|
|
2222
|
+
error: attemptErrorSchema
|
|
2223
|
+
})
|
|
2224
|
+
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
2225
|
+
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
2226
|
+
const cellSchema = z.strictObject({
|
|
2227
|
+
key: z.string(),
|
|
2228
|
+
harness: z.string(),
|
|
2229
|
+
environment: z.string(),
|
|
2230
|
+
sandbox: z.string(),
|
|
2231
|
+
role: cellRoleSchema,
|
|
2232
|
+
variant: z.string().optional()
|
|
2233
|
+
});
|
|
2234
|
+
z.strictObject({
|
|
2235
|
+
type: z.literal("start"),
|
|
2236
|
+
attempt: z.strictObject({
|
|
2237
|
+
id: nonEmpty,
|
|
2238
|
+
cell: cellSchema,
|
|
2239
|
+
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
2240
|
+
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
2241
|
+
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
2242
|
+
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
2243
|
+
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
2244
|
+
z.strictObject({
|
|
2245
|
+
type: z.literal("abort"),
|
|
2246
|
+
reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
|
|
2247
|
+
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
2248
|
+
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
2249
|
+
z.strictObject({
|
|
2250
|
+
type: z.literal("done"),
|
|
2251
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
|
|
2252
|
+
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
2253
|
+
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
2254
|
+
const OTHER_KEYS = [
|
|
2255
|
+
"id",
|
|
2256
|
+
"call",
|
|
2257
|
+
"turn",
|
|
2258
|
+
"attempt",
|
|
2259
|
+
"value",
|
|
2260
|
+
"regexp",
|
|
2261
|
+
"flags",
|
|
2262
|
+
"matchedBy",
|
|
2263
|
+
"answers",
|
|
2264
|
+
...Object.keys(judgeCallDataSchema.shape),
|
|
2265
|
+
...Object.keys(captureSchema.shape),
|
|
2266
|
+
...Object.keys(captureCallDataSchema.shape),
|
|
2267
|
+
"port",
|
|
2268
|
+
"handle",
|
|
2269
|
+
"exec",
|
|
2270
|
+
"result",
|
|
2271
|
+
"readFile",
|
|
2272
|
+
"interval",
|
|
2273
|
+
"timeout",
|
|
2274
|
+
"ms"
|
|
2275
|
+
];
|
|
2276
|
+
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
2277
|
+
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
2278
|
+
// src/attempt/observe.ts
|
|
2279
|
+
/** The most JSON one value may serialize to, masked. */
|
|
2280
|
+
const MAX_OBSERVED_BYTES = 4096;
|
|
2281
|
+
/** The deepest a value may nest arrays and objects, so a check of it can't run out of stack. */
|
|
2282
|
+
const MAX_DEPTH = 64;
|
|
2283
|
+
const at = (path) => path ? ` at ${path}` : "";
|
|
2284
|
+
/** Why `value` isn't JSON, naming where, or undefined. */
|
|
2285
|
+
function notJson(value, path, seen, depth = 0) {
|
|
2286
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return void 0;
|
|
2287
|
+
if (typeof value === "number") return Number.isFinite(value) ? void 0 : `${value}${at(path)} isn't a finite number`;
|
|
2288
|
+
if (typeof value !== "object") return `a ${typeof value}${at(path)} isn't JSON`;
|
|
2289
|
+
if (seen.has(value)) return `the value refers to itself${at(path)}`;
|
|
2290
|
+
if (depth >= MAX_DEPTH) return `it nests deeper than ${MAX_DEPTH} levels${at(path)}`;
|
|
2291
|
+
const proto = Object.getPrototypeOf(value);
|
|
2292
|
+
if (!Array.isArray(value) && proto !== Object.prototype && proto !== null) return `a ${value.constructor?.name ?? "class instance"}${at(path)} isn't plain JSON`;
|
|
2293
|
+
seen.add(value);
|
|
2294
|
+
const entries = Array.isArray(value) ? Array.from(value, (v, i) => [`${path}[${i}]`, v]) : Object.entries(value).map(([k, v]) => [path ? `${path}.${k}` : k, v]);
|
|
2295
|
+
for (const [p, v] of entries) {
|
|
2296
|
+
const problem = notJson(v, p, seen, depth + 1);
|
|
2297
|
+
if (problem) return problem;
|
|
2298
|
+
}
|
|
2299
|
+
seen.delete(value);
|
|
2300
|
+
}
|
|
2301
|
+
/**
|
|
2302
|
+
* `value` with each number whose digits are a secret's masked too: `redactValue` leaves numbers
|
|
2303
|
+
* as they are, but an observed value goes to a judge's or question's model as it's recorded.
|
|
2304
|
+
*/
|
|
2305
|
+
function maskNumbers(value) {
|
|
2306
|
+
if (typeof value === "number") return redactSecrets(String(value)) === String(value) ? value : REDACTED;
|
|
2307
|
+
if (Array.isArray(value)) return value.map(maskNumbers);
|
|
2308
|
+
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, maskNumbers(v)]));
|
|
2309
|
+
return value;
|
|
2310
|
+
}
|
|
2311
|
+
/**
|
|
2312
|
+
* An observation's name as it's recorded and shown: masked as its value is. A name that holds a
|
|
2313
|
+
* secret is a value too, and a judge's or question's list of names is matched against this.
|
|
2314
|
+
*/
|
|
2315
|
+
const observedName = (name) => redactSecrets(name);
|
|
2316
|
+
/** Each attempt's names as its body gave them, by the name they're recorded as. */
|
|
2317
|
+
const givenNames = /* @__PURE__ */ new WeakMap();
|
|
2318
|
+
/**
|
|
2319
|
+
* Whether this attempt observed `name` as given: its masked name alone may be another secret's.
|
|
2320
|
+
* An attempt that never called `c.observe` has observed nothing.
|
|
2321
|
+
*/
|
|
2322
|
+
const observedHere = (a, name) => givenNames.get(a)?.get(observedName(name)) === name;
|
|
2323
|
+
/**
|
|
2324
|
+
* `c.observe`: keeps the last value per name, masked, with the turn it came after. Two names that
|
|
2325
|
+
* mask to the same name are refused rather than renamed: a name is the key that tallies, compare
|
|
2326
|
+
* and `attempt_values` group by across attempts, and a numbered stand-in would depend on the
|
|
2327
|
+
* order of each attempt's calls, joining values that don't belong together.
|
|
2328
|
+
*/
|
|
2329
|
+
function observer(a) {
|
|
2330
|
+
const raw = /* @__PURE__ */ new Map();
|
|
2331
|
+
givenNames.set(a, raw);
|
|
2332
|
+
return (given, value) => {
|
|
2333
|
+
if (typeof given !== "string" || !OBSERVE_NAME.test(given)) throw new UsageError(`c.observe() needs a name of 1 to 64 letters, digits, _ . : / or - (got ${JSON.stringify(typeof given === "string" ? observedName(given) : given)})`);
|
|
2334
|
+
const name = observedName(given);
|
|
2335
|
+
const earlier = raw.get(name);
|
|
2336
|
+
if (earlier !== void 0 && earlier !== given) throw new UsageError(`c.observe("${name}") names a secret, and its masked name is another observation's: name the value without the secret`);
|
|
2337
|
+
if (a.ended) throw new UsageError(`c.observe("${name}") came after the eval's body returned`);
|
|
2338
|
+
if (a.signal.aborted) throw new UsageError(`c.observe("${name}") came after the attempt timed out or was canceled`);
|
|
2339
|
+
const problem = notJson(value, "", /* @__PURE__ */ new Set());
|
|
2340
|
+
if (problem) throw new UsageError(`c.observe("${name}") takes JSON: ${problem}`);
|
|
2341
|
+
const masked = maskNumbers(redactValue(value));
|
|
2342
|
+
const bytes = Buffer.byteLength(JSON.stringify(masked));
|
|
2343
|
+
if (bytes > 4096) throw new UsageError(`c.observe("${name}") takes at most ${MAX_OBSERVED_BYTES} bytes of JSON (got ${bytes}): observe a count or a summary instead`);
|
|
2344
|
+
a.record.observations ??= [];
|
|
2345
|
+
const list = a.record.observations;
|
|
2346
|
+
const same = list.find((o) => o.name === name);
|
|
2347
|
+
if (!same && list.length >= 50) throw new UsageError(`c.observe("${name}") is one name too many: an attempt observes at most 50`);
|
|
2348
|
+
raw.set(name, given);
|
|
2349
|
+
const observation = {
|
|
2350
|
+
name,
|
|
2351
|
+
value: masked,
|
|
2352
|
+
turn: a.turns.length
|
|
2353
|
+
};
|
|
2354
|
+
if (same) Object.assign(same, observation);
|
|
2355
|
+
else list.push(observation);
|
|
2356
|
+
};
|
|
2357
|
+
}
|
|
2358
|
+
// src/judge/state-options.ts
|
|
2359
|
+
/**
|
|
2360
|
+
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2361
|
+
* selections serialize the same.
|
|
2362
|
+
*/
|
|
2363
|
+
function resolveStateOptions(call) {
|
|
2364
|
+
const observations = observedNames(call.observations);
|
|
2365
|
+
return {
|
|
2366
|
+
task: call.task ?? false,
|
|
2367
|
+
output: call.output ?? false,
|
|
2368
|
+
files: call.files ?? false,
|
|
2369
|
+
diff: call.diff ?? false,
|
|
2370
|
+
contents: call.contents ?? [],
|
|
2371
|
+
transcript: call.transcript ?? false,
|
|
2372
|
+
turns: call.turns ?? "last",
|
|
2373
|
+
checks: call.checks ?? false,
|
|
2374
|
+
...observationsOn(observations) ? { observations } : {}
|
|
2375
|
+
};
|
|
2376
|
+
}
|
|
2377
|
+
/** A list of observation names as they're recorded, so a secret in one is masked. */
|
|
2378
|
+
const observedNames = (value) => Array.isArray(value) ? value.map(observedName) : value;
|
|
2379
|
+
/** `observations` when it turns the input on: `true` or a non-empty list. */
|
|
2380
|
+
const observationsOn = (value) => value === true || Array.isArray(value) && value.length > 0;
|
|
2381
|
+
/** `inner` over `outer`, field by field. */
|
|
2382
|
+
function mergeJudge(outer, inner) {
|
|
2383
|
+
return inner ? {
|
|
2384
|
+
...outer,
|
|
2385
|
+
...inner
|
|
2386
|
+
} : outer;
|
|
2387
|
+
}
|
|
1745
2388
|
// src/compare/state.ts
|
|
1746
2389
|
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1747
2390
|
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
@@ -1764,6 +2407,7 @@ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
|
|
|
1764
2407
|
* question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
|
|
1765
2408
|
*/
|
|
1766
2409
|
function resolveQuestionState(state) {
|
|
2410
|
+
const observations = observedNames(state?.observations);
|
|
1767
2411
|
return {
|
|
1768
2412
|
task: state?.task ?? false,
|
|
1769
2413
|
output: state?.output ?? false,
|
|
@@ -1771,7 +2415,8 @@ function resolveQuestionState(state) {
|
|
|
1771
2415
|
diff: state?.diff ?? false,
|
|
1772
2416
|
transcript: state?.transcript ?? false,
|
|
1773
2417
|
turns: state?.turns ?? "all",
|
|
1774
|
-
checks: state?.checks ?? false
|
|
2418
|
+
checks: state?.checks ?? false,
|
|
2419
|
+
...observationsOn(observations) ? { observations } : {}
|
|
1775
2420
|
};
|
|
1776
2421
|
}
|
|
1777
2422
|
/** `inner` over `outer`, field by field. */
|
|
@@ -1792,7 +2437,7 @@ function withQuestionDefaults(q, defaults) {
|
|
|
1792
2437
|
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1793
2438
|
/** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1794
2439
|
function questionShows(state, artifacts) {
|
|
1795
|
-
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
2440
|
+
const on = [...KEYS.filter((k) => k !== "turns" && !isOff(state[k])), ...state.observations ? ["observations"] : []];
|
|
1796
2441
|
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
1797
2442
|
}
|
|
1798
2443
|
// src/contract/compare.ts
|
|
@@ -1905,7 +2550,7 @@ const addressFor = (listeners, port) => {
|
|
|
1905
2550
|
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
1906
2551
|
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
1907
2552
|
};
|
|
1908
|
-
/** `want` is taken whoever listens on it:
|
|
2553
|
+
/** `want` is taken whoever listens on it: `servePort` found it free when the command started. */
|
|
1909
2554
|
function decide(s) {
|
|
1910
2555
|
const { listeners, alive } = s.probe;
|
|
1911
2556
|
const ports = newPorts(listeners, s.before);
|
|
@@ -1962,6 +2607,24 @@ async function hostListening(port) {
|
|
|
1962
2607
|
socket.once("error", () => resolve(false));
|
|
1963
2608
|
})) return host;
|
|
1964
2609
|
}
|
|
2610
|
+
/** How many times `c.serve` allocates `PORT` again when the sandbox already listens on it. */
|
|
2611
|
+
const ALLOCATE_TRIES = 5;
|
|
2612
|
+
/**
|
|
2613
|
+
* The port `c.serve` sets as `PORT` and waits on: `port`, or one from `allocate` that nothing
|
|
2614
|
+
* listens on yet. Throws when something already listens on `port`, since that server, not the
|
|
2615
|
+
* command's, would answer the checks: c.serve only stops what it started.
|
|
2616
|
+
*/
|
|
2617
|
+
async function servePort(command, port, allocate, taken) {
|
|
2618
|
+
if (port === void 0) {
|
|
2619
|
+
for (let i = 0; i < ALLOCATE_TRIES; i++) {
|
|
2620
|
+
const want = await allocate();
|
|
2621
|
+
if (!await taken(want)) return want;
|
|
2622
|
+
}
|
|
2623
|
+
throw new Error(`c.serve(${JSON.stringify(command)}) found every port it allocated for PORT already in use in the sandbox`);
|
|
2624
|
+
}
|
|
2625
|
+
if (await taken(port)) throw new Error(`c.serve(${JSON.stringify(command)}, ${port}): port ${port} was already in use before c.serve started (a server the agent left running?), so the checks would reach that server, not this command's. Stop it first, or drop the port to serve on PORT`);
|
|
2626
|
+
return port;
|
|
2627
|
+
}
|
|
1965
2628
|
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
1966
2629
|
const done = () => {
|
|
1967
2630
|
clearTimeout(timer);
|
|
@@ -2261,16 +2924,16 @@ function resolveOutput(input) {
|
|
|
2261
2924
|
"errors-only",
|
|
2262
2925
|
"none"
|
|
2263
2926
|
]);
|
|
2264
|
-
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "
|
|
2927
|
+
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "none";
|
|
2265
2928
|
const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
|
|
2266
2929
|
return {
|
|
2267
2930
|
mode,
|
|
2268
2931
|
statusLine,
|
|
2932
|
+
interactive: statusLine && !!input.stdinTTY && outputLogs === "none",
|
|
2269
2933
|
logOrder,
|
|
2270
2934
|
logPrefix,
|
|
2271
2935
|
outputLogs,
|
|
2272
2936
|
color,
|
|
2273
|
-
rich: statusLine && color,
|
|
2274
2937
|
githubActions,
|
|
2275
2938
|
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
2276
2939
|
};
|
|
@@ -2398,30 +3061,6 @@ function egressLabel({ policy, auto }) {
|
|
|
2398
3061
|
const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
|
|
2399
3062
|
return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
|
|
2400
3063
|
}
|
|
2401
|
-
// src/judge/state-options.ts
|
|
2402
|
-
/**
|
|
2403
|
-
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2404
|
-
* selections serialize the same.
|
|
2405
|
-
*/
|
|
2406
|
-
function resolveStateOptions(call) {
|
|
2407
|
-
return {
|
|
2408
|
-
task: call.task ?? false,
|
|
2409
|
-
output: call.output ?? false,
|
|
2410
|
-
files: call.files ?? false,
|
|
2411
|
-
diff: call.diff ?? false,
|
|
2412
|
-
contents: call.contents ?? [],
|
|
2413
|
-
transcript: call.transcript ?? false,
|
|
2414
|
-
turns: call.turns ?? "last",
|
|
2415
|
-
checks: call.checks ?? false
|
|
2416
|
-
};
|
|
2417
|
-
}
|
|
2418
|
-
/** `inner` over `outer`, field by field. */
|
|
2419
|
-
function mergeJudge(outer, inner) {
|
|
2420
|
-
return inner ? {
|
|
2421
|
-
...outer,
|
|
2422
|
-
...inner
|
|
2423
|
-
} : outer;
|
|
2424
|
-
}
|
|
2425
3064
|
const KNOWN = [...[
|
|
2426
3065
|
"harness",
|
|
2427
3066
|
"sandbox",
|
|
@@ -2754,9 +3393,8 @@ function normalizeCompare(value) {
|
|
|
2754
3393
|
};
|
|
2755
3394
|
}
|
|
2756
3395
|
/**
|
|
2757
|
-
* Runs per cell when no level sets `runs`: a compare with variants runs `
|
|
2758
|
-
*
|
|
2759
|
-
* included, runs once.
|
|
3396
|
+
* Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
|
|
3397
|
+
* anything else, a pooled family's cases included, runs once.
|
|
2760
3398
|
*/
|
|
2761
3399
|
const COMPARE_DEFAULT_RUNS = 4;
|
|
2762
3400
|
function applyLevel(out, options, where, egress = "own") {
|
|
@@ -3257,7 +3895,11 @@ const GATEWAYS = {
|
|
|
3257
3895
|
vertex: { cantServe: (h) => {
|
|
3258
3896
|
if (h.kind === "fake") return void 0;
|
|
3259
3897
|
if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
|
|
3260
|
-
|
|
3898
|
+
const effort = harnessEffort(h);
|
|
3899
|
+
const refused = vertexModelRefusalFor(h);
|
|
3900
|
+
if (refused) return refused;
|
|
3901
|
+
if (h.kind === "claude-code" && effort !== void 0) return claudeCodeTakesEffort(h.model, "vertex") ? void 0 : `Claude Code sends no effort to ${claudeCodeModelName(h.model, "vertex")} on Vertex AI, where it sends one only to the models its catalog lists: drop effort, or run one of them, e.g. claude-sonnet-5-5`;
|
|
3902
|
+
return h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", effort) : void 0;
|
|
3261
3903
|
} }
|
|
3262
3904
|
};
|
|
3263
3905
|
/** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
|
|
@@ -3886,7 +4528,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3886
4528
|
}
|
|
3887
4529
|
const ANALYSIS_DEFAULTS = {
|
|
3888
4530
|
maxInputTokens: 2e4,
|
|
3889
|
-
maxOutputTokens:
|
|
4531
|
+
maxOutputTokens: 32e3
|
|
3890
4532
|
};
|
|
3891
4533
|
function checkAnalysis(analysis, file) {
|
|
3892
4534
|
if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
|
|
@@ -4054,6 +4696,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
4054
4696
|
return at ? [`${name} (character ${at})`] : [];
|
|
4055
4697
|
});
|
|
4056
4698
|
}
|
|
4057
|
-
export {
|
|
4699
|
+
export { directOnCloud as $, evalPlan as A, agentMcp as At, profileJudge as B, refusedPlugins as Bt, openCodeModelOn as C, execOptionsProblem as Ct, cascade as D, AGENTS as Dt, brokeredIn as E, servedEffortOf as Et, matchesTags as F, expectedBlockedFor as Ft, sandboxSetBy as G, judgeProblem as Gt, runsOf as H, INPUT_KEYS as Ht, plainHeaderError as I, isAgentKind as It, unbrokeredError as J, decideState as Jt, secretVarsIn as K, artifactRefsOf as Kt, plainHeaders as L, keyRejected as Lt, harnessId as M, brokeringFor as Mt, levelOf as N, claudeMcpServers as Nt, checkLimitHarness as O, agentAuth as Ot, majorityOf as P, credentialOf as Pt, ciRunner as Q, plannedSandboxIds as R, layoutOf as Rt, fallbackHint as S, attemptRequestProblem as St, routedHarness as T, resolvedEffortOf as Tt, sandboxId as U, NO_JUDGE_INPUT as Ut, refuseIllegalCompare as V, writePluginFiles as Vt, sandboxProvider as W, inputsOn as Wt, DEFAULT_EGRESS_ALLOW as X, resolveJudgeArtifacts as Xt, DEFAULT_EGRESS as Y, filePartsOf as Yt, allowedHosts as Z, fellBackToDirect as _, observedHere as _t, configPathFor as a, createFakeSandbox as at, resolveModelGateway as b, CAPTURE_NAME as bt, findConfig as c, parseProbe as ct, malformedCredentials as d, STOPPED_FIRST as dt, egressLabel as et, missingCredentials as f, UNRECORDED_QUESTION_STATE as ft, selectProfile as g, resolveStateOptions as gt, runJudgeModel as h, mergeJudge as ht, configLabel as i, resolveOutput as it, expand as j, authHint as jt, checkOptions as k, agentFor as kt, hostCapacity as l, probeScript as lt, resolveConcurrency as m, resolveQuestionState as mt, analysisModelOf as n, isDirect as nt, defaultConcurrency as o, awaitListening as ot, requireConfig as p, questionShows as pt, unbrokered as q, badJudgeArtifacts as qt, configFilesPhrase as r, unenforceable as rt, defineConfig as s, hostListening as st, CONFIG_FILES as t, gatewayPolicy as tt, loadConfig as u, servePort as ut, gatewayScope as v, observedName as vt, resolveGateway as w, harnessEffort as wt, effortRefusedOn as x, CAPTURE_NAME_RULE as xt, modelGatewayOrNone as y, observer as yt, plannedSandboxes as z, refusedMcp as zt };
|
|
4058
4700
|
|
|
4059
|
-
//# sourceMappingURL=config-
|
|
4701
|
+
//# sourceMappingURL=config-C2UtS9vy.js.map
|