reeval 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-BIlJWXff.js → adapter-CptK032x.js} +2 -2
- package/dist/{adapter-BIlJWXff.js.map → adapter-CptK032x.js.map} +1 -1
- package/dist/{agent-CE-qdvmh.js → agent-B63OzUX9.js} +124 -11
- package/dist/agent-B63OzUX9.js.map +1 -0
- package/dist/{analysis-CTBb_C0Z.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-CTBb_C0Z.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-YMmDAHA8.js → app-files-sgqwfjtn.js} +4 -4
- package/dist/app-files-sgqwfjtn.js.map +1 -0
- package/dist/assemble-CxBHkavM.js +1 -0
- package/dist/{assemble-7RI7Nuhz.js → assemble-Me7Whp-y.js} +62 -196
- package/dist/assemble-Me7Whp-y.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
- package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
- package/dist/{attempt-host-CJM_wEA4.js → attempt-host-Cik0SoIn.js} +17 -9
- package/dist/{attempt-host-CJM_wEA4.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
- package/dist/bridge/index.mjs +9 -5
- package/dist/{capture-DF8cHsJh.js → capture-C3RmjPph.js} +392 -609
- package/dist/capture-C3RmjPph.js.map +1 -0
- package/dist/capture-YPwxp_iK.js +1 -0
- package/dist/{cli-Bm9ntSxz.js → cli-BBOlXKPG.js} +1078 -459
- package/dist/cli-BBOlXKPG.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
- package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
- package/dist/client-tZ2WBDxf.js.map +1 -0
- package/dist/{command-Dk6wggV-.js → command-CFQ9c1hk.js} +13 -13
- package/dist/{command-Dk6wggV-.js.map → command-CFQ9c1hk.js.map} +1 -1
- package/dist/{command-DPF8hnWS.js → command-DhiwsZl1.js} +9 -9
- package/dist/{command-DPF8hnWS.js.map → command-DhiwsZl1.js.map} +1 -1
- package/dist/{command-DDkz6jsD.js → command-fO0BFsTq.js} +105 -37
- package/dist/command-fO0BFsTq.js.map +1 -0
- package/dist/{command-B7MslY_w.js → command-qGS802uX.js} +17 -15
- package/dist/command-qGS802uX.js.map +1 -0
- package/dist/{commands-DHT-nhEJ.js → commands-CC60TOT5.js} +64 -18
- package/dist/commands-CC60TOT5.js.map +1 -0
- package/dist/{commands-DQBayPCV.js → commands-xYTMNTPv.js} +1 -1
- package/dist/{config-BLL5JqeX.js → config-C2UtS9vy.js} +745 -80
- package/dist/config-C2UtS9vy.js.map +1 -0
- package/dist/{context-I6nq3uS7.js → context-BajfiH0S.js} +3 -3
- package/dist/{context-I6nq3uS7.js.map → context-BajfiH0S.js.map} +1 -1
- package/dist/{contrasts-CjOU_pfD.js → contrasts-BjOg7wyr.js} +3 -3
- package/dist/{contrasts-CjOU_pfD.js.map → contrasts-BjOg7wyr.js.map} +1 -1
- package/dist/cost-BJU0MAji.js +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
- package/dist/{cursor-catalog-BeBJTk6A.js → cursor-catalog-BTK_apL-.js} +2 -2
- package/dist/{cursor-catalog-BeBJTk6A.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
- package/dist/{db-DPiWC7Br.js → db-C5SZ0510.js} +79 -18
- package/dist/db-C5SZ0510.js.map +1 -0
- package/dist/db-puzLzX4k.js +2 -0
- package/dist/{default-effort-ciIXMA7h.js → default-effort-BVmx7r7L.js} +8 -5
- package/dist/default-effort-BVmx7r7L.js.map +1 -0
- package/dist/{default-effort-B7c4V9Qd.js → default-effort-FEidNAIi.js} +1 -1
- package/dist/{dist-CGwueB0T.js → dist-ByYk_wAs.js} +58 -290
- package/dist/dist-ByYk_wAs.js.map +1 -0
- package/dist/{docker-Bcrc39Z8.js → docker-B2F8-PGC.js} +4 -4
- package/dist/{docker-Bcrc39Z8.js.map → docker-B2F8-PGC.js.map} +1 -1
- package/dist/{docker-D83s_ADn.js → docker-BlSopHpZ.js} +1 -1
- package/dist/{docker-BUZOxh62.js → docker-Dto8ROCq.js} +5 -5
- package/dist/{docker-BUZOxh62.js.map → docker-Dto8ROCq.js.map} +1 -1
- package/dist/{doctor-CIhEAzbw.js → doctor-BcvfKpBX.js} +20 -20
- package/dist/{doctor-CIhEAzbw.js.map → doctor-BcvfKpBX.js.map} +1 -1
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Ccytlpgf.js → environment-CC1RV21n.js} +3 -3
- package/dist/{environment-Ccytlpgf.js.map → environment-CC1RV21n.js.map} +1 -1
- package/dist/{eval-source-D4xqc0iO.js → eval-source-CzxdgFBW.js} +18 -10
- package/dist/eval-source-CzxdgFBW.js.map +1 -0
- package/dist/evaluate-DToSB2bt.d.ts +101 -0
- package/dist/{evaluate-DNzyRQFe.js → evaluate-NTNJ8T9H.js} +135 -54
- package/dist/evaluate-NTNJ8T9H.js.map +1 -0
- package/dist/{files-D_CBK9Zo.js → files-CZXrkPja.js} +6 -6
- package/dist/{files-D_CBK9Zo.js.map → files-CZXrkPja.js.map} +1 -1
- package/dist/files-DFtuRQJx.js +1 -0
- package/dist/{files-3ZqUBbwg.js → files-DsXULUi5.js} +4 -4
- package/dist/{files-3ZqUBbwg.js.map → files-DsXULUi5.js.map} +1 -1
- package/dist/{files-B2U5RezT.js → files-aAzoGNCB.js} +5 -5
- package/dist/{files-B2U5RezT.js.map → files-aAzoGNCB.js.map} +1 -1
- package/dist/files-oOYNR7ZH.js +1 -0
- package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
- package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-s-qf-vww.js → git-C1MB5bjr.js} +3 -3
- package/dist/{git-s-qf-vww.js.map → git-C1MB5bjr.js.map} +1 -1
- package/dist/harness-DGsRZDSo.js +105 -0
- package/dist/harness-DGsRZDSo.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
- package/dist/harness-DYSC4zMh.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-h1VBM7TP.js → images-CNayXCp2.js} +8 -8
- package/dist/{images-h1VBM7TP.js.map → images-CNayXCp2.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5 -5
- package/dist/init-8o6W3_fn.js +1 -0
- package/dist/{init-Z1cRnE3E.js → init-D5HtmyMI.js} +5 -5
- package/dist/{init-Z1cRnE3E.js.map → init-D5HtmyMI.js.map} +1 -1
- package/dist/ink-list-Ca-5dunO.js +184 -0
- package/dist/ink-list-Ca-5dunO.js.map +1 -0
- package/dist/{invocations-z6hUD15n.js → invocations-DrGXF9dT.js} +4 -4
- package/dist/{invocations-z6hUD15n.js.map → invocations-DrGXF9dT.js.map} +1 -1
- package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
- package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-m-mb0zv8.js +246 -0
- package/dist/judgments-m-mb0zv8.js.map +1 -0
- package/dist/list-CM_YSJXq.js +374 -0
- package/dist/list-CM_YSJXq.js.map +1 -0
- package/dist/{live-D22qY6S_.js → live-B71o2TkX.js} +22 -4
- package/dist/live-B71o2TkX.js.map +1 -0
- package/dist/live-DQc8jr6o.js +1 -0
- package/dist/{live-DpfJfc33.js → live-uRURFDMZ.js} +7 -7
- package/dist/{live-DpfJfc33.js.map → live-uRURFDMZ.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
- package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
- package/dist/{model-BL_10Neu.js → model-BxBWFOZn.js} +44 -20
- package/dist/{model-BL_10Neu.js.map → model-BxBWFOZn.js.map} +1 -1
- package/dist/{model-D1TBKeZj.js → model-CXVVB4RK.js} +1 -1
- package/dist/{model-Bn7ndUQZ.js → model-Ce9l9_rF.js} +27 -254
- package/dist/model-Ce9l9_rF.js.map +1 -0
- package/dist/model-Cxzo4cjw.js +1 -0
- package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
- package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
- package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
- package/dist/{patch-LFunkZ-L.js → patch-ug5HpMOj.js} +3 -3
- package/dist/{patch-LFunkZ-L.js.map → patch-ug5HpMOj.js.map} +1 -1
- package/dist/{plan-BrfiMcHh.js → plan-DqnzfDH1.js} +15 -14
- package/dist/plan-DqnzfDH1.js.map +1 -0
- package/dist/{plugins-CW-sQMg3.js → plugins-B2KQ0y3P.js} +4 -4
- package/dist/{plugins-CW-sQMg3.js.map → plugins-B2KQ0y3P.js.map} +1 -1
- package/dist/{preflight-CH-7KyhS.js → preflight-DVXEvzFU.js} +1 -1
- package/dist/{preflight-BnFpd3Kv.js → preflight-DyzmvK7i.js} +3 -3
- package/dist/{preflight-BnFpd3Kv.js.map → preflight-DyzmvK7i.js.map} +1 -1
- package/dist/{prepare-0cA77eFD.js → prepare-21WMQZyt.js} +7 -7
- package/dist/{prepare-0cA77eFD.js.map → prepare-21WMQZyt.js.map} +1 -1
- package/dist/{prepare-CRyEwb5q.js → prepare-BcSRa7-3.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
- package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
- package/dist/{provenance-CKIG_1LB.js → provenance-iRxjp4Pu.js} +4 -4
- package/dist/{provenance-CKIG_1LB.js.map → provenance-iRxjp4Pu.js.map} +1 -1
- package/dist/{providers-CCl8ElNW.js → providers-D0ob_tra.js} +1 -1
- package/dist/{providers-Dt6BEN_1.js → providers-vXMXuwn-.js} +5 -5
- package/dist/{providers-Dt6BEN_1.js.map → providers-vXMXuwn-.js.map} +1 -1
- package/dist/{query-BV-zcw2t.js → query-BhaKQ07P.js} +3 -3
- package/dist/{query-BV-zcw2t.js.map → query-BhaKQ07P.js.map} +1 -1
- package/dist/query-CvgUsvep.js +1 -0
- package/dist/{questions-AyLUMHYX.js → questions-BzdLdV3C.js} +39 -9
- package/dist/questions-BzdLdV3C.js.map +1 -0
- package/dist/{record-J45gJise.js → record-DhzwkSXc.js} +22 -2
- package/dist/{record-J45gJise.js.map → record-DhzwkSXc.js.map} +1 -1
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-UIZTSIhd.js → remarks-CauU408t.js} +231 -25
- package/dist/remarks-CauU408t.js.map +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/{remarks-CgO1nIoG.js → remarks-NMgPd30U.js} +4 -3
- package/dist/{remarks-CgO1nIoG.js.map → remarks-NMgPd30U.js.map} +1 -1
- package/dist/{remote-BA9-hhgB.js → remote-Dwy3fyo3.js} +5 -5
- package/dist/{remote-BA9-hhgB.js.map → remote-Dwy3fyo3.js.map} +1 -1
- package/dist/{resolve-FHjE-dN3.js → resolve-B1zMnPwK.js} +7 -7
- package/dist/{resolve-FHjE-dN3.js.map → resolve-B1zMnPwK.js.map} +1 -1
- package/dist/resolve-BIIeVT12.js +1 -0
- package/dist/{results-CFE6Nwb0.js → results-CfC0id_u.js} +7 -7
- package/dist/{results-CFE6Nwb0.js.map → results-CfC0id_u.js.map} +1 -1
- package/dist/{resume-iaTxWRew.js → resume-Dkhrpjow.js} +123 -8
- package/dist/resume-Dkhrpjow.js.map +1 -0
- package/dist/{runtime-SEkGS1Rx.js → runtime-DcvQcr36.js} +2 -2
- package/dist/{runtime-SEkGS1Rx.js.map → runtime-DcvQcr36.js.map} +1 -1
- package/dist/{sandbox-CAsXlGro.js → sandbox-8zCrtuCf.js} +2 -2
- package/dist/{sandbox-CAsXlGro.js.map → sandbox-8zCrtuCf.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-0tuj8VSa.js +1 -0
- package/dist/{server-C9_kV1IP.js → server-Dplq8g0r.js} +8 -8
- package/dist/{server-C9_kV1IP.js.map → server-Dplq8g0r.js.map} +1 -1
- package/dist/{source-BszY6yAC.js → source-NWHqn5-o.js} +2 -2
- package/dist/{source-BszY6yAC.js.map → source-NWHqn5-o.js.map} +1 -1
- package/dist/source-llwLewOU.js +1 -0
- package/dist/{src-Dt6BWDcJ.js → src-B9oE_mNS.js} +1 -1
- package/dist/{src-DaYQ1O9Y.js → src-BQ6BiXoc.js} +86 -31
- package/dist/{src-DaYQ1O9Y.js.map → src-BQ6BiXoc.js.map} +1 -1
- package/dist/src-BQXdf-Gw.js +1448 -0
- package/dist/src-BQXdf-Gw.js.map +1 -0
- package/dist/{src-DVGVWOAB.js → src-BXnv0ViU.js} +1 -1
- package/dist/src-BiS8YNN_.js +3 -0
- package/dist/{src-CZANB4pb.js → src-BkH4GOxu.js} +1 -1
- package/dist/{src-8fy4nSWU.js → src-BvM0jWuZ.js} +28 -2
- package/dist/{src-8fy4nSWU.js.map → src-BvM0jWuZ.js.map} +1 -1
- package/dist/src-C-tH5ebd.js +2 -0
- package/dist/{src-umdk2alV.js → src-CUpoVVb8.js} +2 -2
- package/dist/src-Cg1bFs9g.js +2 -0
- package/dist/{src-CWrKJO92.js → src-D5t4OiRt.js} +2 -2
- package/dist/{src-CWrKJO92.js.map → src-D5t4OiRt.js.map} +1 -1
- package/dist/{src-DoxUxkU3.js → src-DV_UorZv.js} +1 -1
- package/dist/{src-DMiwtprP.js → src-KlDOGD0B.js} +4 -4
- package/dist/{src-DMiwtprP.js.map → src-KlDOGD0B.js.map} +1 -1
- package/dist/{src-BYpY_vXK.js → src-XxUpr2sa.js} +1 -1
- package/dist/{state-DfcikRK3.js → state-DDCGUhva.js} +16 -16
- package/dist/{state-DfcikRK3.js.map → state-DDCGUhva.js.map} +1 -1
- package/dist/state-DZbAO95a.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-M8_7xio7.js +1294 -0
- package/dist/suite-M8_7xio7.js.map +1 -0
- package/dist/suite-u2KJnGpY.js +2 -0
- package/dist/{summary-BmooCwQv.js → summary-Uzc9osfV.js} +47 -13
- package/dist/summary-Uzc9osfV.js.map +1 -0
- package/dist/testing.d.ts +51 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
- package/dist/{types-BlM-aN9_.d.ts → types-DGvexkcp.d.ts} +116 -66
- package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
- package/dist/vertex-BLQMayc1.js.map +1 -0
- package/dist/vertex-h_peaZTN.js +1 -0
- package/dist/view/index.d.ts +143 -34
- package/dist/view/index.js +9 -8
- package/dist/{views-DPng-2-z.js → views-BISrsIjw.js} +6 -6
- package/dist/{views-DPng-2-z.js.map → views-BISrsIjw.js.map} +1 -1
- package/dist/views-Bfab8fM9.js +2 -0
- package/dist/{worker-entry-lt5otEb5.js → worker-entry-DJD_nHkS.js} +5 -5
- package/dist/{worker-entry-lt5otEb5.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
- package/dist/workspace-B8EOCuxl.js +1 -0
- package/dist/{workspace-1O8x4oDc.js → workspace-BO1VhUR9.js} +5 -5
- package/dist/{workspace-1O8x4oDc.js.map → workspace-BO1VhUR9.js.map} +1 -1
- package/docker/Dockerfile +4 -4
- package/package.json +14 -12
- package/skills/eval-rubric/SKILL.md +22 -3
- package/dist/agent-CE-qdvmh.js.map +0 -1
- package/dist/app-files-YMmDAHA8.js.map +0 -1
- package/dist/assemble-7RI7Nuhz.js.map +0 -1
- package/dist/assemble-CANqwtwD.js +0 -1
- package/dist/capture-DF8cHsJh.js.map +0 -1
- package/dist/capture-DagM1npp.js +0 -1
- package/dist/cli-Bm9ntSxz.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-B7MslY_w.js.map +0 -1
- package/dist/command-DDkz6jsD.js.map +0 -1
- package/dist/commands-DHT-nhEJ.js.map +0 -1
- package/dist/config-BLL5JqeX.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-DPiWC7Br.js.map +0 -1
- package/dist/db-DysoTexX.js +0 -2
- package/dist/default-effort-ciIXMA7h.js.map +0 -1
- package/dist/dist-CGwueB0T.js.map +0 -1
- package/dist/eval-source-D4xqc0iO.js.map +0 -1
- package/dist/evaluate-BEGAQQuo.d.ts +0 -43
- package/dist/evaluate-DNzyRQFe.js.map +0 -1
- package/dist/files-Bw2NQecs.js +0 -1
- package/dist/files-Cp1jp0Nq.js +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/init-DJ3WFUt8.js +0 -1
- package/dist/live-D22qY6S_.js.map +0 -1
- package/dist/live-DUpitP_9.js +0 -1
- package/dist/model-Bn7ndUQZ.js.map +0 -1
- package/dist/model-CR3-qY-z.js +0 -1
- package/dist/plan-BrfiMcHh.js.map +0 -1
- package/dist/query-DSSMxkmw.js +0 -1
- package/dist/questions-AyLUMHYX.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-UIZTSIhd.js.map +0 -1
- package/dist/resolve-C56s8_EF.js +0 -1
- package/dist/resume-iaTxWRew.js.map +0 -1
- package/dist/server-BYfqpMD0.js +0 -1
- package/dist/source-C4Q5psjD.js +0 -1
- package/dist/src-BBmCEE-M.js +0 -3
- package/dist/src-BWcgyARO.js +0 -640
- package/dist/src-BWcgyARO.js.map +0 -1
- package/dist/src-CnrLy_r_.js +0 -2
- package/dist/src-DqIQQTu1.js +0 -2
- package/dist/state-haxTOljN.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BVcBo_ua.js +0 -544
- package/dist/suite-BVcBo_ua.js.map +0 -1
- package/dist/suite-CmeXyn66.js +0 -2
- package/dist/summary-BmooCwQv.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CBAsDkCB.js +0 -2
- package/dist/workspace-It7dvpbg.js +0 -1
- /package/dist/bridge/{pnpm-lock.yaml → pnpm-lockfile.yaml} +0 -0
|
@@ -1,23 +1,24 @@
|
|
|
1
1
|
import { n as isOneOf, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
2
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
3
|
-
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
|
|
4
|
-
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-
|
|
5
|
-
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-
|
|
6
|
-
import {
|
|
3
|
+
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty$1, m as TIMER_LIMIT, n as dataProblem, o as genericProblem, p as MAX_TIMER_MS, s as got, t as closest, u as oneOf } from "./shared-ogqPJ0uo.js";
|
|
4
|
+
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-CC1RV21n.js";
|
|
5
|
+
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-B2KQ0y3P.js";
|
|
6
|
+
import { c as GATEWAY_NAMES, d as gatewayEntryDataSchema, f as gatewayNameSchema, i as claudeCodeTakesEffort, l as directOffPin, o as harnessDescriptorSchema, r as claudeCodeModelName, s as harnessProblem, t as FACTORY, u as gatewayDescriptorSchema } from "./harness-DYSC4zMh.js";
|
|
7
7
|
import { a as fromCwd, l as sha256 } from "./ids-DuK6xl4w.js";
|
|
8
|
-
import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-
|
|
9
|
-
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-
|
|
10
|
-
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-
|
|
11
|
-
import { r as badCharacterAt } from "./redact-
|
|
12
|
-
import {
|
|
13
|
-
import { t as inCI } from "./env-
|
|
8
|
+
import { a as providerEndpoints, c as providerServedModel, n as openCodeEffortBody, o as providerHosts, r as openCodeEffortRefusal, s as providerModelResponses, u as resolveOpenCodeProvider } from "./opencode-providers-Co7plCea.js";
|
|
9
|
+
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-DYOF69Oe.js";
|
|
10
|
+
import { a as VERTEX_ROUTES, c as openCodeVertexId, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-BLQMayc1.js";
|
|
11
|
+
import { _ as redactValue, h as redactSecrets, r as badCharacterAt, t as REDACTED } from "./redact-BhjtnjrU.js";
|
|
12
|
+
import { o as withoutBom } from "./text-B8-W3Cvn.js";
|
|
13
|
+
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
14
14
|
import { u as ownFileURL } from "./root-Co0jhOT5.js";
|
|
15
|
-
import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-
|
|
15
|
+
import { F as HARNESS_HOSTS, I as PACKAGE_HOSTS, P as DOCKER_EGRESS, mt as shellQuote, pt as run } from "./src-BQ6BiXoc.js";
|
|
16
16
|
import { t as InfraError } from "./types-C4OkYhZg.js";
|
|
17
|
-
import { n as gateLabel, u as pairFactors } from "./gate-
|
|
18
|
-
import "./
|
|
19
|
-
import
|
|
20
|
-
import {
|
|
17
|
+
import { n as gateLabel, u as pairFactors } from "./gate-Cp_pQoEe.js";
|
|
18
|
+
import { u as opencodeModelError } from "./harness-DGsRZDSo.js";
|
|
19
|
+
import "./stats-C18W4xPX.js";
|
|
20
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CptK032x.js";
|
|
21
|
+
import { t as docker } from "./sandbox-8zCrtuCf.js";
|
|
21
22
|
import { existsSync, readFileSync } from "node:fs";
|
|
22
23
|
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
23
24
|
import { dirname, join, resolve, win32 } from "node:path";
|
|
@@ -88,20 +89,22 @@ const sharedInputFields = {
|
|
|
88
89
|
task: z.boolean().optional(),
|
|
89
90
|
output: z.boolean().optional(),
|
|
90
91
|
files: z.boolean().optional(),
|
|
91
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
92
|
+
diff: z.union([z.boolean(), z.array(nonEmpty$1()).min(1)]).optional(),
|
|
92
93
|
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
93
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
94
|
+
checks: z.union([z.boolean(), z.array(nonEmpty$1())]).optional(),
|
|
95
|
+
observations: z.union([z.boolean(), z.array(nonEmpty$1())]).optional()
|
|
94
96
|
};
|
|
95
97
|
const turnsSchema = z.enum(["last", "all"]);
|
|
96
|
-
const { task, output, files, diff, transcript, checks } = sharedInputFields;
|
|
98
|
+
const { task, output, files, diff, transcript, checks, observations } = sharedInputFields;
|
|
97
99
|
const judgeInputsSchema = z.strictObject({
|
|
98
100
|
task,
|
|
99
101
|
output,
|
|
100
102
|
files,
|
|
101
103
|
diff,
|
|
102
|
-
contents: z.array(nonEmpty()).optional(),
|
|
104
|
+
contents: z.array(nonEmpty$1()).optional(),
|
|
103
105
|
transcript,
|
|
104
106
|
checks,
|
|
107
|
+
observations,
|
|
105
108
|
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
106
109
|
});
|
|
107
110
|
const judgeSubjectSchema = z.strictObject({
|
|
@@ -149,7 +152,8 @@ const VALUES$1 = {
|
|
|
149
152
|
contents: "a list of paths or patterns",
|
|
150
153
|
transcript: "true, false, 'tools' or 'full'",
|
|
151
154
|
turns: "'last' or 'all'",
|
|
152
|
-
checks: "true, false or a list of commands"
|
|
155
|
+
checks: "true, false or a list of commands",
|
|
156
|
+
observations: "true, false or a list of names"
|
|
153
157
|
};
|
|
154
158
|
function distance(a, b) {
|
|
155
159
|
const row = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
@@ -239,7 +243,7 @@ function judgeProblem(judge, site) {
|
|
|
239
243
|
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
240
244
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
241
245
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
242
|
-
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
246
|
+
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty$1()).min(1)]);
|
|
243
247
|
const questionInputFields = {
|
|
244
248
|
...sharedInputFields,
|
|
245
249
|
artifacts: questionArtifactsSchema.optional()
|
|
@@ -257,9 +261,9 @@ const unique = (list, ctx) => {
|
|
|
257
261
|
});
|
|
258
262
|
};
|
|
259
263
|
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
260
|
-
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
264
|
+
const choicesSchema = z.array(nonEmpty$1()).min(2).superRefine(unique);
|
|
261
265
|
const questionSchema = z.strictObject({
|
|
262
|
-
question: nonEmpty(),
|
|
266
|
+
question: nonEmpty$1(),
|
|
263
267
|
model: questionModelSchema.optional(),
|
|
264
268
|
across: acrossSchema.optional(),
|
|
265
269
|
choices: choicesSchema.optional(),
|
|
@@ -296,7 +300,8 @@ const VALUES = {
|
|
|
296
300
|
diff: "true, false or a non-empty list of patterns",
|
|
297
301
|
transcript: "true, false, 'tools' or 'full'",
|
|
298
302
|
turns: "'last' or 'all'",
|
|
299
|
-
checks: "true, false or a list of commands"
|
|
303
|
+
checks: "true, false or a list of commands",
|
|
304
|
+
observations: "true, false or a list of names"
|
|
300
305
|
};
|
|
301
306
|
const describe$2 = (value) => {
|
|
302
307
|
if (Array.isArray(value)) return `a list of ${value.length}`;
|
|
@@ -344,8 +349,8 @@ function acrossProblem(question, across, where) {
|
|
|
344
349
|
function choicesProblem(question, choices, where) {
|
|
345
350
|
if (choicesSchema.safeParse(choices).success) return void 0;
|
|
346
351
|
const at = `questions "${question}" choices in ${where}`;
|
|
347
|
-
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
348
|
-
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty().safeParse(c).success);
|
|
352
|
+
if (Array.isArray(choices) && choices.length >= 2 && choices.every((c) => nonEmpty$1().safeParse(c).success)) return `${at} lists "${twice(choices)}" twice. List each answer once.`;
|
|
353
|
+
const blank = Array.isArray(choices) && choices.find((c) => !nonEmpty$1().safeParse(c).success);
|
|
349
354
|
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
350
355
|
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
351
356
|
}
|
|
@@ -359,7 +364,7 @@ function questionProblem(item, where) {
|
|
|
359
364
|
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
360
365
|
}
|
|
361
366
|
const { question, model, across, choices } = item;
|
|
362
|
-
if (!nonEmpty().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
367
|
+
if (!nonEmpty$1().safeParse(question).success) return `questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$2(question)})`}, e.g. { question: 'Which is cleaner?', across: 'environment' }.`;
|
|
363
368
|
const text = question;
|
|
364
369
|
if (across !== void 0) {
|
|
365
370
|
const problem = acrossProblem(text, across, where);
|
|
@@ -1090,7 +1095,7 @@ const LAYOUTS = {
|
|
|
1090
1095
|
goose: {
|
|
1091
1096
|
rulesFile: "AGENTS.md",
|
|
1092
1097
|
workspaceSkillDir: ".goose/skills",
|
|
1093
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/
|
|
1098
|
+
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/goose/home/.config/goose/skills",
|
|
1094
1099
|
pluginPlacement: "flat"
|
|
1095
1100
|
},
|
|
1096
1101
|
pi: {
|
|
@@ -1109,13 +1114,13 @@ const LAYOUTS = {
|
|
|
1109
1114
|
fx: {
|
|
1110
1115
|
rulesFile: "AGENTS.md",
|
|
1111
1116
|
workspaceSkillDir: ".fx/skills",
|
|
1112
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/
|
|
1117
|
+
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/fx/home/.fx/skills",
|
|
1113
1118
|
pluginPlacement: "flat"
|
|
1114
1119
|
},
|
|
1115
1120
|
"grok-build": {
|
|
1116
1121
|
rulesFile: "AGENTS.md",
|
|
1117
1122
|
workspaceSkillDir: ".grok/skills",
|
|
1118
|
-
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/
|
|
1123
|
+
harnessSkillDir: ".ai-sdk-harness/.harness-bootstrap/grok-build/home/.grok/skills",
|
|
1119
1124
|
pluginPlacement: "namespaced"
|
|
1120
1125
|
}
|
|
1121
1126
|
};
|
|
@@ -1126,6 +1131,8 @@ const AGENTS = {
|
|
|
1126
1131
|
pluginParts: PLUGIN_PARTS,
|
|
1127
1132
|
name: "Cursor",
|
|
1128
1133
|
factory: "cursor",
|
|
1134
|
+
reasoning: "streamed",
|
|
1135
|
+
reasoningTokens: false,
|
|
1129
1136
|
credential: "CURSOR_API_KEY",
|
|
1130
1137
|
endpoints: [CURSOR_EXCHANGE],
|
|
1131
1138
|
tokens: {
|
|
@@ -1135,7 +1142,7 @@ const AGENTS = {
|
|
|
1135
1142
|
},
|
|
1136
1143
|
unreported: cursorUnreported,
|
|
1137
1144
|
mcpServers: cursorMcpServers,
|
|
1138
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1145
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-BXnv0ViU.js")).createCursorSdk({
|
|
1139
1146
|
auth,
|
|
1140
1147
|
...mcpServers ? { mcpServers } : {},
|
|
1141
1148
|
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
@@ -1148,6 +1155,9 @@ const AGENTS = {
|
|
|
1148
1155
|
pluginParts: CLAUDE_FORMAT_PARTS,
|
|
1149
1156
|
name: "Claude Code",
|
|
1150
1157
|
factory: "claudeCode",
|
|
1158
|
+
reasoning: "streamed",
|
|
1159
|
+
reasoningTokens: false,
|
|
1160
|
+
subagentCalls: true,
|
|
1151
1161
|
credential: "ANTHROPIC_API_KEY",
|
|
1152
1162
|
endpoints: [
|
|
1153
1163
|
{
|
|
@@ -1172,7 +1182,7 @@ const AGENTS = {
|
|
|
1172
1182
|
}
|
|
1173
1183
|
],
|
|
1174
1184
|
mcpServers: claudeMcpServers,
|
|
1175
|
-
adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-
|
|
1185
|
+
adapter: async ({ auth, env, mcpServers, descriptor }) => (await import("./dist-ByYk_wAs.js")).createClaudeCode({
|
|
1176
1186
|
auth,
|
|
1177
1187
|
...mcpServers ? { mcpServers } : {},
|
|
1178
1188
|
...descriptor.kind === "claude-code" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1194,6 +1204,8 @@ const AGENTS = {
|
|
|
1194
1204
|
],
|
|
1195
1205
|
name: "Antigravity",
|
|
1196
1206
|
factory: "antigravity",
|
|
1207
|
+
reasoning: "none",
|
|
1208
|
+
reasoningTokens: true,
|
|
1197
1209
|
credential: "GEMINI_API_KEY",
|
|
1198
1210
|
endpoints: ANTIGRAVITY_ENDPOINTS,
|
|
1199
1211
|
expectedBlocked: {
|
|
@@ -1202,7 +1214,7 @@ const AGENTS = {
|
|
|
1202
1214
|
},
|
|
1203
1215
|
servedModel: geminiServedModel,
|
|
1204
1216
|
mcpServers: antigravityMcpServers,
|
|
1205
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-
|
|
1217
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-XxUpr2sa.js")).createAntigravity({
|
|
1206
1218
|
auth,
|
|
1207
1219
|
...vertex ? { vertex } : {},
|
|
1208
1220
|
...mcpServers ? { mcpServers } : {},
|
|
@@ -1224,12 +1236,15 @@ const AGENTS = {
|
|
|
1224
1236
|
],
|
|
1225
1237
|
name: "OpenCode",
|
|
1226
1238
|
factory: "opencode",
|
|
1239
|
+
configuredModelOnStart: true,
|
|
1240
|
+
reasoning: "streamed",
|
|
1241
|
+
reasoningTokens: "provider",
|
|
1227
1242
|
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1228
1243
|
endpoints: GEMINI_ENDPOINTS,
|
|
1229
1244
|
servedModel: geminiServedModel,
|
|
1230
1245
|
forModel: opencodeForModel,
|
|
1231
1246
|
mcpServers: opencodeMcpServers,
|
|
1232
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1247
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BiS8YNN_.js")).createOpenCode({
|
|
1233
1248
|
auth,
|
|
1234
1249
|
model,
|
|
1235
1250
|
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
@@ -1243,6 +1258,9 @@ const AGENTS = {
|
|
|
1243
1258
|
pluginParts: ["skills", "mcp"],
|
|
1244
1259
|
name: "Goose",
|
|
1245
1260
|
factory: "goose",
|
|
1261
|
+
configuredModelOnStart: true,
|
|
1262
|
+
reasoning: "streamed",
|
|
1263
|
+
reasoningTokens: false,
|
|
1246
1264
|
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1247
1265
|
endpoints: GOOSE_ENDPOINTS,
|
|
1248
1266
|
expectedBlocked: {
|
|
@@ -1251,14 +1269,14 @@ const AGENTS = {
|
|
|
1251
1269
|
},
|
|
1252
1270
|
servedModel: geminiServedModel,
|
|
1253
1271
|
mcpServers: gooseMcpServers,
|
|
1254
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1272
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-C-tH5ebd.js")).createGoose({
|
|
1255
1273
|
...vertex ? { vertex } : {},
|
|
1256
1274
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1257
1275
|
auth,
|
|
1258
|
-
provider: (await import("./src-
|
|
1276
|
+
provider: (await import("./src-C-tH5ebd.js")).AI_GATEWAY_PROVIDER
|
|
1259
1277
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1260
1278
|
auth,
|
|
1261
|
-
provider: (await import("./src-
|
|
1279
|
+
provider: (await import("./src-C-tH5ebd.js")).OPENROUTER_PROVIDER
|
|
1262
1280
|
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1263
1281
|
model,
|
|
1264
1282
|
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1275,17 +1293,19 @@ const AGENTS = {
|
|
|
1275
1293
|
],
|
|
1276
1294
|
name: "pi",
|
|
1277
1295
|
factory: "pi",
|
|
1296
|
+
reasoning: "streamed",
|
|
1297
|
+
reasoningTokens: true,
|
|
1278
1298
|
credential: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
1279
1299
|
endpoints: GEMINI_ENDPOINTS,
|
|
1280
1300
|
servedModel: geminiServedModel,
|
|
1281
1301
|
mcpServers: piMcpServers,
|
|
1282
|
-
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-
|
|
1302
|
+
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-B9oE_mNS.js")).createPi({
|
|
1283
1303
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1284
1304
|
auth,
|
|
1285
|
-
gateway: (await import("./src-
|
|
1305
|
+
gateway: (await import("./src-B9oE_mNS.js")).PI_AI_GATEWAY
|
|
1286
1306
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1287
1307
|
auth,
|
|
1288
|
-
gateway: (await import("./src-
|
|
1308
|
+
gateway: (await import("./src-B9oE_mNS.js")).PI_OPENROUTER
|
|
1289
1309
|
} : {
|
|
1290
1310
|
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1291
1311
|
...vertex ? { vertex } : {}
|
|
@@ -1304,6 +1324,8 @@ const AGENTS = {
|
|
|
1304
1324
|
pluginServersByName: true,
|
|
1305
1325
|
name: "Codex",
|
|
1306
1326
|
factory: "codex",
|
|
1327
|
+
reasoning: "whole",
|
|
1328
|
+
reasoningTokens: true,
|
|
1307
1329
|
credential: "OPENAI_API_KEY",
|
|
1308
1330
|
endpoints: OPENAI_ENDPOINTS,
|
|
1309
1331
|
modelResponses: OPENAI_MODEL_ENDPOINTS,
|
|
@@ -1318,9 +1340,9 @@ const AGENTS = {
|
|
|
1318
1340
|
},
|
|
1319
1341
|
servedModel: openaiServedModel,
|
|
1320
1342
|
mcpServers: codexMcpServers,
|
|
1321
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1343
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-Cg1bFs9g.js")).createCodex({
|
|
1322
1344
|
auth,
|
|
1323
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-
|
|
1345
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-Cg1bFs9g.js")).OPENROUTER_PROVIDER } : {},
|
|
1324
1346
|
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1325
1347
|
...mcpServers ? { mcpServers } : {},
|
|
1326
1348
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
@@ -1335,10 +1357,13 @@ const AGENTS = {
|
|
|
1335
1357
|
pluginParts: ["skills", "mcp"],
|
|
1336
1358
|
name: "fx",
|
|
1337
1359
|
factory: "fx",
|
|
1360
|
+
configuredModelOnStart: true,
|
|
1361
|
+
reasoning: "streamed",
|
|
1362
|
+
reasoningTokens: "provider",
|
|
1338
1363
|
...fxForModel("provider/model"),
|
|
1339
1364
|
forModel: fxForModel,
|
|
1340
1365
|
mcpServers: acpMcpServers,
|
|
1341
|
-
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-
|
|
1366
|
+
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-DV_UorZv.js")).createFx({
|
|
1342
1367
|
auth,
|
|
1343
1368
|
model,
|
|
1344
1369
|
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
@@ -1357,13 +1382,16 @@ const AGENTS = {
|
|
|
1357
1382
|
pluginServersByName: true,
|
|
1358
1383
|
name: "Grok Build",
|
|
1359
1384
|
factory: "grokBuild",
|
|
1385
|
+
configuredModelOnStart: true,
|
|
1386
|
+
reasoning: "streamed",
|
|
1387
|
+
reasoningTokens: true,
|
|
1360
1388
|
credential: "XAI_API_KEY",
|
|
1361
1389
|
endpoints: XAI_ENDPOINTS,
|
|
1362
1390
|
modelResponses: XAI_MODEL_RESPONSES,
|
|
1363
1391
|
servedModel: xaiServedModel,
|
|
1364
1392
|
mcpServers: acpMcpServers,
|
|
1365
1393
|
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1366
|
-
const grok = await import("./src-
|
|
1394
|
+
const grok = await import("./src-CUpoVVb8.js");
|
|
1367
1395
|
return grok.createGrokBuild({
|
|
1368
1396
|
...vertexChat ? {
|
|
1369
1397
|
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
@@ -1719,6 +1747,644 @@ function resolvedEffortOf(models) {
|
|
|
1719
1747
|
}
|
|
1720
1748
|
return out;
|
|
1721
1749
|
}
|
|
1750
|
+
// src/contract/attempt.ts
|
|
1751
|
+
const nullableNumber = z.number().nullable();
|
|
1752
|
+
const tokenUsageSchema = z.strictObject({
|
|
1753
|
+
input: z.number(),
|
|
1754
|
+
output: z.number(),
|
|
1755
|
+
cacheRead: z.number(),
|
|
1756
|
+
cacheWrite: z.number(),
|
|
1757
|
+
reasoning: nullableNumber
|
|
1758
|
+
});
|
|
1759
|
+
const fileChangeSchema = z.strictObject({
|
|
1760
|
+
path: z.string(),
|
|
1761
|
+
status: z.enum([
|
|
1762
|
+
"added",
|
|
1763
|
+
"modified",
|
|
1764
|
+
"deleted"
|
|
1765
|
+
])
|
|
1766
|
+
});
|
|
1767
|
+
const diffSchema = z.strictObject({
|
|
1768
|
+
patch: z.string(),
|
|
1769
|
+
files: z.array(fileChangeSchema)
|
|
1770
|
+
});
|
|
1771
|
+
const turnStatusSchema = z.enum([
|
|
1772
|
+
"completed",
|
|
1773
|
+
"aborted",
|
|
1774
|
+
"error"
|
|
1775
|
+
]);
|
|
1776
|
+
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
|
|
1777
|
+
const at$1 = {
|
|
1778
|
+
ts: z.string(),
|
|
1779
|
+
atMs: z.number()
|
|
1780
|
+
};
|
|
1781
|
+
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
1782
|
+
z.strictObject({
|
|
1783
|
+
...at$1,
|
|
1784
|
+
type: z.enum(["text", "reasoning"]),
|
|
1785
|
+
text: z.string(),
|
|
1786
|
+
durationMs: nullableNumber
|
|
1787
|
+
}),
|
|
1788
|
+
z.strictObject({
|
|
1789
|
+
...at$1,
|
|
1790
|
+
type: z.literal("file-edit"),
|
|
1791
|
+
toolCallId: z.string(),
|
|
1792
|
+
path: z.string(),
|
|
1793
|
+
kind: z.enum([
|
|
1794
|
+
"create",
|
|
1795
|
+
"modify",
|
|
1796
|
+
"write",
|
|
1797
|
+
"delete"
|
|
1798
|
+
]),
|
|
1799
|
+
added: nullableNumber,
|
|
1800
|
+
removed: nullableNumber,
|
|
1801
|
+
durationMs: nullableNumber,
|
|
1802
|
+
stalled: z.boolean().optional(),
|
|
1803
|
+
subagent: z.string().optional()
|
|
1804
|
+
}),
|
|
1805
|
+
z.strictObject({
|
|
1806
|
+
...at$1,
|
|
1807
|
+
type: z.literal("web-search"),
|
|
1808
|
+
toolCallId: z.string(),
|
|
1809
|
+
query: z.string(),
|
|
1810
|
+
references: nullableNumber,
|
|
1811
|
+
ok: z.boolean().nullable().optional(),
|
|
1812
|
+
durationMs: nullableNumber,
|
|
1813
|
+
stalled: z.boolean().optional(),
|
|
1814
|
+
subagent: z.string().optional()
|
|
1815
|
+
}),
|
|
1816
|
+
z.strictObject({
|
|
1817
|
+
...at$1,
|
|
1818
|
+
type: z.literal("web-fetch"),
|
|
1819
|
+
toolCallId: z.string(),
|
|
1820
|
+
url: z.string(),
|
|
1821
|
+
ok: z.boolean().nullable(),
|
|
1822
|
+
durationMs: nullableNumber,
|
|
1823
|
+
stalled: z.boolean().optional(),
|
|
1824
|
+
subagent: z.string().optional()
|
|
1825
|
+
}),
|
|
1826
|
+
z.strictObject({
|
|
1827
|
+
...at$1,
|
|
1828
|
+
type: z.literal("tool"),
|
|
1829
|
+
toolCallId: z.string(),
|
|
1830
|
+
tool: z.string(),
|
|
1831
|
+
input: z.unknown(),
|
|
1832
|
+
output: z.unknown(),
|
|
1833
|
+
ok: z.boolean().nullable(),
|
|
1834
|
+
durationMs: nullableNumber,
|
|
1835
|
+
stalled: z.boolean().optional(),
|
|
1836
|
+
subagent: z.string().optional()
|
|
1837
|
+
}),
|
|
1838
|
+
z.strictObject({
|
|
1839
|
+
...at$1,
|
|
1840
|
+
type: z.literal("error"),
|
|
1841
|
+
message: z.string()
|
|
1842
|
+
}),
|
|
1843
|
+
z.strictObject({
|
|
1844
|
+
...at$1,
|
|
1845
|
+
type: z.literal("retry"),
|
|
1846
|
+
message: z.string()
|
|
1847
|
+
})
|
|
1848
|
+
]);
|
|
1849
|
+
const turnDataSchema = z.strictObject({
|
|
1850
|
+
prompt: z.string(),
|
|
1851
|
+
text: z.string(),
|
|
1852
|
+
status: turnStatusSchema,
|
|
1853
|
+
startedAt: z.string(),
|
|
1854
|
+
durationMs: z.number(),
|
|
1855
|
+
transcript: z.array(transcriptEventSchema),
|
|
1856
|
+
usage: tokenUsageSchema.nullable(),
|
|
1857
|
+
model: z.string().nullable(),
|
|
1858
|
+
unreported: z.array(webEventTypeSchema),
|
|
1859
|
+
diff: diffSchema
|
|
1860
|
+
});
|
|
1861
|
+
const execResultSchema = z.strictObject({
|
|
1862
|
+
exitCode: z.number().int(),
|
|
1863
|
+
stdout: z.string(),
|
|
1864
|
+
stderr: z.string()
|
|
1865
|
+
});
|
|
1866
|
+
const execRetrySchema = z.strictObject({
|
|
1867
|
+
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
1868
|
+
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
1869
|
+
});
|
|
1870
|
+
/** A background command has no exit for `retries` to wait on. */
|
|
1871
|
+
const noBackgroundRetry = (options, ctx) => {
|
|
1872
|
+
if (!options.background) return;
|
|
1873
|
+
for (const key of ["retries", "interval"]) {
|
|
1874
|
+
if (options[key] === void 0) continue;
|
|
1875
|
+
ctx.addIssue({
|
|
1876
|
+
code: "custom",
|
|
1877
|
+
path: [key],
|
|
1878
|
+
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
1879
|
+
});
|
|
1880
|
+
}
|
|
1881
|
+
};
|
|
1882
|
+
const execOptionsSchema = z.strictObject({
|
|
1883
|
+
background: z.boolean().optional(),
|
|
1884
|
+
...execRetrySchema.shape
|
|
1885
|
+
}).superRefine(noBackgroundRetry);
|
|
1886
|
+
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
1887
|
+
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
1888
|
+
const assertionRecordSchema = z.strictObject({
|
|
1889
|
+
label: z.string(),
|
|
1890
|
+
labelled: z.boolean(),
|
|
1891
|
+
matcher: z.string(),
|
|
1892
|
+
status: z.enum([
|
|
1893
|
+
"pass",
|
|
1894
|
+
"fail",
|
|
1895
|
+
"error"
|
|
1896
|
+
]),
|
|
1897
|
+
soft: z.boolean(),
|
|
1898
|
+
at: z.string().optional(),
|
|
1899
|
+
message: z.string().optional(),
|
|
1900
|
+
durationMs: z.number().optional()
|
|
1901
|
+
});
|
|
1902
|
+
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
1903
|
+
const captureConsoleMessageSchema = z.strictObject({
|
|
1904
|
+
level: z.enum([
|
|
1905
|
+
"error",
|
|
1906
|
+
"warning",
|
|
1907
|
+
"info",
|
|
1908
|
+
"log",
|
|
1909
|
+
"debug"
|
|
1910
|
+
]),
|
|
1911
|
+
text: z.string(),
|
|
1912
|
+
source: z.string(),
|
|
1913
|
+
url: z.string().optional()
|
|
1914
|
+
});
|
|
1915
|
+
const captureFailedRequestSchema = z.strictObject({
|
|
1916
|
+
method: z.string(),
|
|
1917
|
+
url: z.string(),
|
|
1918
|
+
status: z.number().optional(),
|
|
1919
|
+
error: z.string().optional()
|
|
1920
|
+
});
|
|
1921
|
+
const capturePageFields = {
|
|
1922
|
+
status: z.number().optional(),
|
|
1923
|
+
finalUrl: z.string(),
|
|
1924
|
+
console: z.array(captureConsoleMessageSchema),
|
|
1925
|
+
errors: z.array(z.string()),
|
|
1926
|
+
failedRequests: z.array(captureFailedRequestSchema),
|
|
1927
|
+
dropped: z.number().optional()
|
|
1928
|
+
};
|
|
1929
|
+
z.strictObject(capturePageFields);
|
|
1930
|
+
const captureSchema = z.strictObject({
|
|
1931
|
+
name: z.string(),
|
|
1932
|
+
port: z.number(),
|
|
1933
|
+
path: z.string(),
|
|
1934
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]),
|
|
1935
|
+
file: z.strictObject({
|
|
1936
|
+
path: z.string(),
|
|
1937
|
+
sha256: z.string(),
|
|
1938
|
+
bytes: z.number()
|
|
1939
|
+
}),
|
|
1940
|
+
...capturePageFields
|
|
1941
|
+
});
|
|
1942
|
+
const judgeArtifactRefSchema = z.strictObject({
|
|
1943
|
+
name: z.string(),
|
|
1944
|
+
path: z.string(),
|
|
1945
|
+
sha256: z.string(),
|
|
1946
|
+
bytes: z.number(),
|
|
1947
|
+
mediaType: z.literal("image/png")
|
|
1948
|
+
});
|
|
1949
|
+
const verdictFields = {
|
|
1950
|
+
threshold: z.number(),
|
|
1951
|
+
question: z.string(),
|
|
1952
|
+
model: z.string(),
|
|
1953
|
+
gateway: gatewayNameSchema.optional(),
|
|
1954
|
+
artifacts: z.array(judgeArtifactRefSchema).optional()
|
|
1955
|
+
};
|
|
1956
|
+
const verdictSchema = z.strictObject({
|
|
1957
|
+
verdict: z.enum(["pass", "fail"]),
|
|
1958
|
+
score: z.number(),
|
|
1959
|
+
...verdictFields,
|
|
1960
|
+
resolvedModel: z.string().optional(),
|
|
1961
|
+
upstream: z.string().optional(),
|
|
1962
|
+
costUsd: z.number().optional()
|
|
1963
|
+
});
|
|
1964
|
+
const judgeFailureSchema = z.strictObject({
|
|
1965
|
+
verdict: z.literal("error"),
|
|
1966
|
+
score: z.null(),
|
|
1967
|
+
...verdictFields,
|
|
1968
|
+
error: z.string()
|
|
1969
|
+
});
|
|
1970
|
+
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
1971
|
+
const patternSchema = z.strictObject({
|
|
1972
|
+
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
1973
|
+
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
1974
|
+
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
1975
|
+
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
1976
|
+
function hasAnsweredPattern(value) {
|
|
1977
|
+
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
1978
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1979
|
+
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
1980
|
+
}
|
|
1981
|
+
const answeredPatternSchema = z.strictObject({
|
|
1982
|
+
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
1983
|
+
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
1984
|
+
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
1985
|
+
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
1986
|
+
const matcherArgSchema = z.union([
|
|
1987
|
+
patternSchema,
|
|
1988
|
+
answeredPatternSchema,
|
|
1989
|
+
jsonValueSchema
|
|
1990
|
+
]).superRefine((value, ctx) => {
|
|
1991
|
+
if (!isPatternLike(value)) return;
|
|
1992
|
+
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
1993
|
+
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
1994
|
+
const matcherSubjectSchema = z.union([
|
|
1995
|
+
z.strictObject({ turn: turnIndex }),
|
|
1996
|
+
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
1997
|
+
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
1998
|
+
z.strictObject({
|
|
1999
|
+
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
2000
|
+
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
2001
|
+
}),
|
|
2002
|
+
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
2003
|
+
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
2004
|
+
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
2005
|
+
const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
|
|
2006
|
+
const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
|
|
2007
|
+
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
2008
|
+
const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
|
|
2009
|
+
const captureWaitSchema = z.enum(["load", "idle"]);
|
|
2010
|
+
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
|
|
2011
|
+
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
2012
|
+
const captureCallDataSchema = z.strictObject({
|
|
2013
|
+
name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
|
|
2014
|
+
path: z.string().optional(),
|
|
2015
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
2016
|
+
timeout: durationSchema.optional(),
|
|
2017
|
+
wait: waitSchema.optional(),
|
|
2018
|
+
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
2019
|
+
model: z.string().optional()
|
|
2020
|
+
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
2021
|
+
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
2022
|
+
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
2023
|
+
const judgeCallDataSchema = z.strictObject({
|
|
2024
|
+
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
2025
|
+
...judgeFields,
|
|
2026
|
+
...sharedInputFields,
|
|
2027
|
+
contents: z.array(nonEmpty).optional(),
|
|
2028
|
+
turns: turnsSchema.optional(),
|
|
2029
|
+
on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
|
|
2030
|
+
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
2031
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
|
|
2032
|
+
/** An observation's name: one a query, a tally line and a judge's input can carry as it is. */
|
|
2033
|
+
const OBSERVE_NAME = /^[A-Za-z0-9_.:/-]{1,64}$/;
|
|
2034
|
+
const callSiteSchema = z.strictObject({
|
|
2035
|
+
file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
|
|
2036
|
+
line: z.number().int().min(1).describe("From 1.")
|
|
2037
|
+
}).describe("Where the eval made the call, which the record names beside the check's outcome.");
|
|
2038
|
+
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
2039
|
+
z.strictObject({
|
|
2040
|
+
op: z.literal("send"),
|
|
2041
|
+
prompt: z.string().describe("The turn's prompt.")
|
|
2042
|
+
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
2043
|
+
z.strictObject({
|
|
2044
|
+
op: z.literal("exec"),
|
|
2045
|
+
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
2046
|
+
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
2047
|
+
...execRetrySchema.shape
|
|
2048
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
2049
|
+
z.strictObject({
|
|
2050
|
+
op: z.literal("serve"),
|
|
2051
|
+
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
2052
|
+
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
2053
|
+
}).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
|
|
2054
|
+
z.strictObject({
|
|
2055
|
+
op: z.literal("kill"),
|
|
2056
|
+
handle: handleId
|
|
2057
|
+
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
2058
|
+
z.strictObject({
|
|
2059
|
+
op: z.literal("capture"),
|
|
2060
|
+
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
2061
|
+
options: captureCallDataSchema.optional()
|
|
2062
|
+
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
2063
|
+
z.strictObject({
|
|
2064
|
+
op: z.literal("readFile"),
|
|
2065
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
2066
|
+
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
2067
|
+
z.strictObject({
|
|
2068
|
+
op: z.literal("writeFile"),
|
|
2069
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
2070
|
+
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
2071
|
+
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
2072
|
+
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
2073
|
+
z.strictObject({
|
|
2074
|
+
op: z.literal("observe"),
|
|
2075
|
+
name: z.string().regex(OBSERVE_NAME, "must be 1 to 64 letters, digits, _ . : / or -").describe("What the value is of, e.g. `installed`."),
|
|
2076
|
+
value: jsonValueSchema.describe("Any JSON, at most 4 KB once serialized.")
|
|
2077
|
+
}).describe("`c.observe(name, value)`: records a fact about the attempt that never affects its outcome. A name observed again keeps the last value. Fails after the body's `done`."),
|
|
2078
|
+
z.strictObject({
|
|
2079
|
+
op: z.literal("expect"),
|
|
2080
|
+
subject: matcherSubjectSchema,
|
|
2081
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
2082
|
+
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
2083
|
+
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
2084
|
+
soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
|
|
2085
|
+
label: z.string().optional().describe("`expect`'s second argument."),
|
|
2086
|
+
at: callSiteSchema.optional(),
|
|
2087
|
+
poll: z.strictObject({
|
|
2088
|
+
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
2089
|
+
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
2090
|
+
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
2091
|
+
}).superRefine((call, ctx) => {
|
|
2092
|
+
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
2093
|
+
code: "custom",
|
|
2094
|
+
path: ["subject"],
|
|
2095
|
+
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
2096
|
+
});
|
|
2097
|
+
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
2098
|
+
code: "custom",
|
|
2099
|
+
path: ["subject", "result"],
|
|
2100
|
+
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
2101
|
+
});
|
|
2102
|
+
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
2103
|
+
code: "custom",
|
|
2104
|
+
path: ["args"],
|
|
2105
|
+
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
2106
|
+
});
|
|
2107
|
+
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
2108
|
+
z.strictObject({
|
|
2109
|
+
op: z.literal("record"),
|
|
2110
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
2111
|
+
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
2112
|
+
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
2113
|
+
status: z.enum(["pass", "fail"]),
|
|
2114
|
+
message: z.string().optional().describe("Why it failed."),
|
|
2115
|
+
soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
|
|
2116
|
+
at: callSiteSchema.optional()
|
|
2117
|
+
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
2118
|
+
z.strictObject({
|
|
2119
|
+
op: z.literal("judge"),
|
|
2120
|
+
question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
|
|
2121
|
+
options: judgeCallDataSchema
|
|
2122
|
+
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
2123
|
+
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
2124
|
+
const attemptErrorSchema = z.looseObject({
|
|
2125
|
+
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
2126
|
+
message: z.string()
|
|
2127
|
+
}).describe("Why a call failed, in reeval's words.");
|
|
2128
|
+
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
2129
|
+
const attemptRequestSchema = z.strictObject({
|
|
2130
|
+
id,
|
|
2131
|
+
call: attemptCallSchema
|
|
2132
|
+
}).describe("A call, numbered so its reply can find it.");
|
|
2133
|
+
z.union([
|
|
2134
|
+
z.discriminatedUnion("op", [
|
|
2135
|
+
z.strictObject({
|
|
2136
|
+
id,
|
|
2137
|
+
ok: z.literal(true),
|
|
2138
|
+
op: z.literal("send"),
|
|
2139
|
+
turn: turnDataSchema
|
|
2140
|
+
}),
|
|
2141
|
+
z.strictObject({
|
|
2142
|
+
id,
|
|
2143
|
+
ok: z.literal(true),
|
|
2144
|
+
op: z.literal("exec"),
|
|
2145
|
+
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
2146
|
+
}),
|
|
2147
|
+
z.strictObject({
|
|
2148
|
+
id,
|
|
2149
|
+
ok: z.literal(true),
|
|
2150
|
+
op: z.literal("serve"),
|
|
2151
|
+
handle: handleId,
|
|
2152
|
+
port: z.number().describe("The container port it listens on.")
|
|
2153
|
+
}),
|
|
2154
|
+
z.strictObject({
|
|
2155
|
+
id,
|
|
2156
|
+
ok: z.literal(true),
|
|
2157
|
+
op: z.literal("kill")
|
|
2158
|
+
}),
|
|
2159
|
+
z.strictObject({
|
|
2160
|
+
id,
|
|
2161
|
+
ok: z.literal(true),
|
|
2162
|
+
op: z.literal("capture"),
|
|
2163
|
+
capture: captureSchema
|
|
2164
|
+
}),
|
|
2165
|
+
z.strictObject({
|
|
2166
|
+
id,
|
|
2167
|
+
ok: z.literal(true),
|
|
2168
|
+
op: z.literal("readFile"),
|
|
2169
|
+
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
2170
|
+
}),
|
|
2171
|
+
z.strictObject({
|
|
2172
|
+
id,
|
|
2173
|
+
ok: z.literal(true),
|
|
2174
|
+
op: z.literal("writeFile")
|
|
2175
|
+
}),
|
|
2176
|
+
z.strictObject({
|
|
2177
|
+
id,
|
|
2178
|
+
ok: z.literal(true),
|
|
2179
|
+
op: z.literal("observe")
|
|
2180
|
+
}),
|
|
2181
|
+
z.strictObject({
|
|
2182
|
+
id,
|
|
2183
|
+
ok: z.literal(true),
|
|
2184
|
+
op: z.literal("expect"),
|
|
2185
|
+
assertion: assertionRecordSchema
|
|
2186
|
+
}),
|
|
2187
|
+
z.strictObject({
|
|
2188
|
+
id,
|
|
2189
|
+
ok: z.literal(true),
|
|
2190
|
+
op: z.literal("record"),
|
|
2191
|
+
assertion: assertionRecordSchema
|
|
2192
|
+
}),
|
|
2193
|
+
z.strictObject({
|
|
2194
|
+
id,
|
|
2195
|
+
ok: z.literal(true),
|
|
2196
|
+
op: z.literal("judge"),
|
|
2197
|
+
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
2198
|
+
})
|
|
2199
|
+
]),
|
|
2200
|
+
z.strictObject({
|
|
2201
|
+
id,
|
|
2202
|
+
ok: z.literal(true),
|
|
2203
|
+
op: z.literal("expect"),
|
|
2204
|
+
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
2205
|
+
}),
|
|
2206
|
+
z.strictObject({
|
|
2207
|
+
id,
|
|
2208
|
+
ok: z.literal(false),
|
|
2209
|
+
op: z.enum([
|
|
2210
|
+
"send",
|
|
2211
|
+
"exec",
|
|
2212
|
+
"readFile",
|
|
2213
|
+
"writeFile",
|
|
2214
|
+
"observe",
|
|
2215
|
+
"expect",
|
|
2216
|
+
"record",
|
|
2217
|
+
"judge",
|
|
2218
|
+
"serve",
|
|
2219
|
+
"kill",
|
|
2220
|
+
"capture"
|
|
2221
|
+
]),
|
|
2222
|
+
error: attemptErrorSchema
|
|
2223
|
+
})
|
|
2224
|
+
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
2225
|
+
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
2226
|
+
const cellSchema = z.strictObject({
|
|
2227
|
+
key: z.string(),
|
|
2228
|
+
harness: z.string(),
|
|
2229
|
+
environment: z.string(),
|
|
2230
|
+
sandbox: z.string(),
|
|
2231
|
+
role: cellRoleSchema,
|
|
2232
|
+
variant: z.string().optional()
|
|
2233
|
+
});
|
|
2234
|
+
z.strictObject({
|
|
2235
|
+
type: z.literal("start"),
|
|
2236
|
+
attempt: z.strictObject({
|
|
2237
|
+
id: nonEmpty,
|
|
2238
|
+
cell: cellSchema,
|
|
2239
|
+
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
2240
|
+
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
2241
|
+
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
2242
|
+
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
2243
|
+
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
2244
|
+
z.strictObject({
|
|
2245
|
+
type: z.literal("abort"),
|
|
2246
|
+
reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
|
|
2247
|
+
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
2248
|
+
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
2249
|
+
z.strictObject({
|
|
2250
|
+
type: z.literal("done"),
|
|
2251
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
|
|
2252
|
+
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
2253
|
+
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
2254
|
+
const OTHER_KEYS = [
|
|
2255
|
+
"id",
|
|
2256
|
+
"call",
|
|
2257
|
+
"turn",
|
|
2258
|
+
"attempt",
|
|
2259
|
+
"value",
|
|
2260
|
+
"regexp",
|
|
2261
|
+
"flags",
|
|
2262
|
+
"matchedBy",
|
|
2263
|
+
"answers",
|
|
2264
|
+
...Object.keys(judgeCallDataSchema.shape),
|
|
2265
|
+
...Object.keys(captureSchema.shape),
|
|
2266
|
+
...Object.keys(captureCallDataSchema.shape),
|
|
2267
|
+
"port",
|
|
2268
|
+
"handle",
|
|
2269
|
+
"exec",
|
|
2270
|
+
"result",
|
|
2271
|
+
"readFile",
|
|
2272
|
+
"interval",
|
|
2273
|
+
"timeout",
|
|
2274
|
+
"ms"
|
|
2275
|
+
];
|
|
2276
|
+
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
2277
|
+
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
2278
|
+
// src/attempt/observe.ts
|
|
2279
|
+
/** The most JSON one value may serialize to, masked. */
|
|
2280
|
+
const MAX_OBSERVED_BYTES = 4096;
|
|
2281
|
+
/** The deepest a value may nest arrays and objects, so a check of it can't run out of stack. */
|
|
2282
|
+
const MAX_DEPTH = 64;
|
|
2283
|
+
const at = (path) => path ? ` at ${path}` : "";
|
|
2284
|
+
/** Why `value` isn't JSON, naming where, or undefined. */
|
|
2285
|
+
function notJson(value, path, seen, depth = 0) {
|
|
2286
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") return void 0;
|
|
2287
|
+
if (typeof value === "number") return Number.isFinite(value) ? void 0 : `${value}${at(path)} isn't a finite number`;
|
|
2288
|
+
if (typeof value !== "object") return `a ${typeof value}${at(path)} isn't JSON`;
|
|
2289
|
+
if (seen.has(value)) return `the value refers to itself${at(path)}`;
|
|
2290
|
+
if (depth >= MAX_DEPTH) return `it nests deeper than ${MAX_DEPTH} levels${at(path)}`;
|
|
2291
|
+
const proto = Object.getPrototypeOf(value);
|
|
2292
|
+
if (!Array.isArray(value) && proto !== Object.prototype && proto !== null) return `a ${value.constructor?.name ?? "class instance"}${at(path)} isn't plain JSON`;
|
|
2293
|
+
seen.add(value);
|
|
2294
|
+
const entries = Array.isArray(value) ? Array.from(value, (v, i) => [`${path}[${i}]`, v]) : Object.entries(value).map(([k, v]) => [path ? `${path}.${k}` : k, v]);
|
|
2295
|
+
for (const [p, v] of entries) {
|
|
2296
|
+
const problem = notJson(v, p, seen, depth + 1);
|
|
2297
|
+
if (problem) return problem;
|
|
2298
|
+
}
|
|
2299
|
+
seen.delete(value);
|
|
2300
|
+
}
|
|
2301
|
+
/**
|
|
2302
|
+
* `value` with each number whose digits are a secret's masked too: `redactValue` leaves numbers
|
|
2303
|
+
* as they are, but an observed value goes to a judge's or question's model as it's recorded.
|
|
2304
|
+
*/
|
|
2305
|
+
function maskNumbers(value) {
|
|
2306
|
+
if (typeof value === "number") return redactSecrets(String(value)) === String(value) ? value : REDACTED;
|
|
2307
|
+
if (Array.isArray(value)) return value.map(maskNumbers);
|
|
2308
|
+
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, maskNumbers(v)]));
|
|
2309
|
+
return value;
|
|
2310
|
+
}
|
|
2311
|
+
/**
|
|
2312
|
+
* An observation's name as it's recorded and shown: masked as its value is. A name that holds a
|
|
2313
|
+
* secret is a value too, and a judge's or question's list of names is matched against this.
|
|
2314
|
+
*/
|
|
2315
|
+
const observedName = (name) => redactSecrets(name);
|
|
2316
|
+
/** Each attempt's names as its body gave them, by the name they're recorded as. */
|
|
2317
|
+
const givenNames = /* @__PURE__ */ new WeakMap();
|
|
2318
|
+
/**
|
|
2319
|
+
* Whether this attempt observed `name` as given: its masked name alone may be another secret's.
|
|
2320
|
+
* An attempt that never called `c.observe` has observed nothing.
|
|
2321
|
+
*/
|
|
2322
|
+
const observedHere = (a, name) => givenNames.get(a)?.get(observedName(name)) === name;
|
|
2323
|
+
/**
|
|
2324
|
+
* `c.observe`: keeps the last value per name, masked, with the turn it came after. Two names that
|
|
2325
|
+
* mask to the same name are refused rather than renamed: a name is the key that tallies, compare
|
|
2326
|
+
* and `attempt_values` group by across attempts, and a numbered stand-in would depend on the
|
|
2327
|
+
* order of each attempt's calls, joining values that don't belong together.
|
|
2328
|
+
*/
|
|
2329
|
+
function observer(a) {
|
|
2330
|
+
const raw = /* @__PURE__ */ new Map();
|
|
2331
|
+
givenNames.set(a, raw);
|
|
2332
|
+
return (given, value) => {
|
|
2333
|
+
if (typeof given !== "string" || !OBSERVE_NAME.test(given)) throw new UsageError(`c.observe() needs a name of 1 to 64 letters, digits, _ . : / or - (got ${JSON.stringify(typeof given === "string" ? observedName(given) : given)})`);
|
|
2334
|
+
const name = observedName(given);
|
|
2335
|
+
const earlier = raw.get(name);
|
|
2336
|
+
if (earlier !== void 0 && earlier !== given) throw new UsageError(`c.observe("${name}") names a secret, and its masked name is another observation's: name the value without the secret`);
|
|
2337
|
+
if (a.ended) throw new UsageError(`c.observe("${name}") came after the eval's body returned`);
|
|
2338
|
+
if (a.signal.aborted) throw new UsageError(`c.observe("${name}") came after the attempt timed out or was canceled`);
|
|
2339
|
+
const problem = notJson(value, "", /* @__PURE__ */ new Set());
|
|
2340
|
+
if (problem) throw new UsageError(`c.observe("${name}") takes JSON: ${problem}`);
|
|
2341
|
+
const masked = maskNumbers(redactValue(value));
|
|
2342
|
+
const bytes = Buffer.byteLength(JSON.stringify(masked));
|
|
2343
|
+
if (bytes > 4096) throw new UsageError(`c.observe("${name}") takes at most ${MAX_OBSERVED_BYTES} bytes of JSON (got ${bytes}): observe a count or a summary instead`);
|
|
2344
|
+
a.record.observations ??= [];
|
|
2345
|
+
const list = a.record.observations;
|
|
2346
|
+
const same = list.find((o) => o.name === name);
|
|
2347
|
+
if (!same && list.length >= 50) throw new UsageError(`c.observe("${name}") is one name too many: an attempt observes at most 50`);
|
|
2348
|
+
raw.set(name, given);
|
|
2349
|
+
const observation = {
|
|
2350
|
+
name,
|
|
2351
|
+
value: masked,
|
|
2352
|
+
turn: a.turns.length
|
|
2353
|
+
};
|
|
2354
|
+
if (same) Object.assign(same, observation);
|
|
2355
|
+
else list.push(observation);
|
|
2356
|
+
};
|
|
2357
|
+
}
|
|
2358
|
+
// src/judge/state-options.ts
|
|
2359
|
+
/**
|
|
2360
|
+
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2361
|
+
* selections serialize the same.
|
|
2362
|
+
*/
|
|
2363
|
+
function resolveStateOptions(call) {
|
|
2364
|
+
const observations = observedNames(call.observations);
|
|
2365
|
+
return {
|
|
2366
|
+
task: call.task ?? false,
|
|
2367
|
+
output: call.output ?? false,
|
|
2368
|
+
files: call.files ?? false,
|
|
2369
|
+
diff: call.diff ?? false,
|
|
2370
|
+
contents: call.contents ?? [],
|
|
2371
|
+
transcript: call.transcript ?? false,
|
|
2372
|
+
turns: call.turns ?? "last",
|
|
2373
|
+
checks: call.checks ?? false,
|
|
2374
|
+
...observationsOn(observations) ? { observations } : {}
|
|
2375
|
+
};
|
|
2376
|
+
}
|
|
2377
|
+
/** A list of observation names as they're recorded, so a secret in one is masked. */
|
|
2378
|
+
const observedNames = (value) => Array.isArray(value) ? value.map(observedName) : value;
|
|
2379
|
+
/** `observations` when it turns the input on: `true` or a non-empty list. */
|
|
2380
|
+
const observationsOn = (value) => value === true || Array.isArray(value) && value.length > 0;
|
|
2381
|
+
/** `inner` over `outer`, field by field. */
|
|
2382
|
+
function mergeJudge(outer, inner) {
|
|
2383
|
+
return inner ? {
|
|
2384
|
+
...outer,
|
|
2385
|
+
...inner
|
|
2386
|
+
} : outer;
|
|
2387
|
+
}
|
|
1722
2388
|
// src/compare/state.ts
|
|
1723
2389
|
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1724
2390
|
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
@@ -1741,6 +2407,7 @@ const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
|
|
|
1741
2407
|
* question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
|
|
1742
2408
|
*/
|
|
1743
2409
|
function resolveQuestionState(state) {
|
|
2410
|
+
const observations = observedNames(state?.observations);
|
|
1744
2411
|
return {
|
|
1745
2412
|
task: state?.task ?? false,
|
|
1746
2413
|
output: state?.output ?? false,
|
|
@@ -1748,7 +2415,8 @@ function resolveQuestionState(state) {
|
|
|
1748
2415
|
diff: state?.diff ?? false,
|
|
1749
2416
|
transcript: state?.transcript ?? false,
|
|
1750
2417
|
turns: state?.turns ?? "all",
|
|
1751
|
-
checks: state?.checks ?? false
|
|
2418
|
+
checks: state?.checks ?? false,
|
|
2419
|
+
...observationsOn(observations) ? { observations } : {}
|
|
1752
2420
|
};
|
|
1753
2421
|
}
|
|
1754
2422
|
/** `inner` over `outer`, field by field. */
|
|
@@ -1769,7 +2437,7 @@ function withQuestionDefaults(q, defaults) {
|
|
|
1769
2437
|
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1770
2438
|
/** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1771
2439
|
function questionShows(state, artifacts) {
|
|
1772
|
-
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
2440
|
+
const on = [...KEYS.filter((k) => k !== "turns" && !isOff(state[k])), ...state.observations ? ["observations"] : []];
|
|
1773
2441
|
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
1774
2442
|
}
|
|
1775
2443
|
// src/contract/compare.ts
|
|
@@ -1882,7 +2550,7 @@ const addressFor = (listeners, port) => {
|
|
|
1882
2550
|
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
1883
2551
|
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
1884
2552
|
};
|
|
1885
|
-
/** `want` is taken whoever listens on it:
|
|
2553
|
+
/** `want` is taken whoever listens on it: `servePort` found it free when the command started. */
|
|
1886
2554
|
function decide(s) {
|
|
1887
2555
|
const { listeners, alive } = s.probe;
|
|
1888
2556
|
const ports = newPorts(listeners, s.before);
|
|
@@ -1939,6 +2607,24 @@ async function hostListening(port) {
|
|
|
1939
2607
|
socket.once("error", () => resolve(false));
|
|
1940
2608
|
})) return host;
|
|
1941
2609
|
}
|
|
2610
|
+
/** How many times `c.serve` allocates `PORT` again when the sandbox already listens on it. */
|
|
2611
|
+
const ALLOCATE_TRIES = 5;
|
|
2612
|
+
/**
|
|
2613
|
+
* The port `c.serve` sets as `PORT` and waits on: `port`, or one from `allocate` that nothing
|
|
2614
|
+
* listens on yet. Throws when something already listens on `port`, since that server, not the
|
|
2615
|
+
* command's, would answer the checks: c.serve only stops what it started.
|
|
2616
|
+
*/
|
|
2617
|
+
async function servePort(command, port, allocate, taken) {
|
|
2618
|
+
if (port === void 0) {
|
|
2619
|
+
for (let i = 0; i < ALLOCATE_TRIES; i++) {
|
|
2620
|
+
const want = await allocate();
|
|
2621
|
+
if (!await taken(want)) return want;
|
|
2622
|
+
}
|
|
2623
|
+
throw new Error(`c.serve(${JSON.stringify(command)}) found every port it allocated for PORT already in use in the sandbox`);
|
|
2624
|
+
}
|
|
2625
|
+
if (await taken(port)) throw new Error(`c.serve(${JSON.stringify(command)}, ${port}): port ${port} was already in use before c.serve started (a server the agent left running?), so the checks would reach that server, not this command's. Stop it first, or drop the port to serve on PORT`);
|
|
2626
|
+
return port;
|
|
2627
|
+
}
|
|
1942
2628
|
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
1943
2629
|
const done = () => {
|
|
1944
2630
|
clearTimeout(timer);
|
|
@@ -2238,16 +2924,16 @@ function resolveOutput(input) {
|
|
|
2238
2924
|
"errors-only",
|
|
2239
2925
|
"none"
|
|
2240
2926
|
]);
|
|
2241
|
-
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "
|
|
2927
|
+
const outputLogs = logs && logs !== "auto" ? logs : flags.verbose ? "full" : "none";
|
|
2242
2928
|
const color = env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions;
|
|
2243
2929
|
return {
|
|
2244
2930
|
mode,
|
|
2245
2931
|
statusLine,
|
|
2932
|
+
interactive: statusLine && !!input.stdinTTY && outputLogs === "none",
|
|
2246
2933
|
logOrder,
|
|
2247
2934
|
logPrefix,
|
|
2248
2935
|
outputLogs,
|
|
2249
2936
|
color,
|
|
2250
|
-
rich: statusLine && color,
|
|
2251
2937
|
githubActions,
|
|
2252
2938
|
heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
|
|
2253
2939
|
};
|
|
@@ -2375,30 +3061,6 @@ function egressLabel({ policy, auto }) {
|
|
|
2375
3061
|
const own = policy === "deny-all" ? "deny-all" : policy.allow.length ? `allow ${policy.allow.join(", ")}` : "allow none";
|
|
2376
3062
|
return auto.length ? `${own} + auto: ${auto.join(", ")}` : own;
|
|
2377
3063
|
}
|
|
2378
|
-
// src/judge/state-options.ts
|
|
2379
|
-
/**
|
|
2380
|
-
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2381
|
-
* selections serialize the same.
|
|
2382
|
-
*/
|
|
2383
|
-
function resolveStateOptions(call) {
|
|
2384
|
-
return {
|
|
2385
|
-
task: call.task ?? false,
|
|
2386
|
-
output: call.output ?? false,
|
|
2387
|
-
files: call.files ?? false,
|
|
2388
|
-
diff: call.diff ?? false,
|
|
2389
|
-
contents: call.contents ?? [],
|
|
2390
|
-
transcript: call.transcript ?? false,
|
|
2391
|
-
turns: call.turns ?? "last",
|
|
2392
|
-
checks: call.checks ?? false
|
|
2393
|
-
};
|
|
2394
|
-
}
|
|
2395
|
-
/** `inner` over `outer`, field by field. */
|
|
2396
|
-
function mergeJudge(outer, inner) {
|
|
2397
|
-
return inner ? {
|
|
2398
|
-
...outer,
|
|
2399
|
-
...inner
|
|
2400
|
-
} : outer;
|
|
2401
|
-
}
|
|
2402
3064
|
const KNOWN = [...[
|
|
2403
3065
|
"harness",
|
|
2404
3066
|
"sandbox",
|
|
@@ -2731,9 +3393,8 @@ function normalizeCompare(value) {
|
|
|
2731
3393
|
};
|
|
2732
3394
|
}
|
|
2733
3395
|
/**
|
|
2734
|
-
* Runs per cell when no level sets `runs`: a compare with variants runs `
|
|
2735
|
-
*
|
|
2736
|
-
* included, runs once.
|
|
3396
|
+
* Runs per cell when no level sets `runs`: a compare with variants runs `COMPARE_RUNS` a side;
|
|
3397
|
+
* anything else, a pooled family's cases included, runs once.
|
|
2737
3398
|
*/
|
|
2738
3399
|
const COMPARE_DEFAULT_RUNS = 4;
|
|
2739
3400
|
function applyLevel(out, options, where, egress = "own") {
|
|
@@ -3234,7 +3895,11 @@ const GATEWAYS = {
|
|
|
3234
3895
|
vertex: { cantServe: (h) => {
|
|
3235
3896
|
if (h.kind === "fake") return void 0;
|
|
3236
3897
|
if (!VERTEX_ROUTES[h.kind]) return VERTEX_REFUSALS[h.kind] ?? `reeval doesn't run ${agentName(h)} on Vertex AI yet`;
|
|
3237
|
-
|
|
3898
|
+
const effort = harnessEffort(h);
|
|
3899
|
+
const refused = vertexModelRefusalFor(h);
|
|
3900
|
+
if (refused) return refused;
|
|
3901
|
+
if (h.kind === "claude-code" && effort !== void 0) return claudeCodeTakesEffort(h.model, "vertex") ? void 0 : `Claude Code sends no effort to ${claudeCodeModelName(h.model, "vertex")} on Vertex AI, where it sends one only to the models its catalog lists: drop effort, or run one of them, e.g. claude-sonnet-5-5`;
|
|
3902
|
+
return h.kind === "opencode" ? openCodeEffortOn(h.model, "vertex", effort) : void 0;
|
|
3238
3903
|
} }
|
|
3239
3904
|
};
|
|
3240
3905
|
/** OpenCode's id for `model` on `gateway`, as that gateway's route writes it. */
|
|
@@ -3863,7 +4528,7 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3863
4528
|
}
|
|
3864
4529
|
const ANALYSIS_DEFAULTS = {
|
|
3865
4530
|
maxInputTokens: 2e4,
|
|
3866
|
-
maxOutputTokens:
|
|
4531
|
+
maxOutputTokens: 32e3
|
|
3867
4532
|
};
|
|
3868
4533
|
function checkAnalysis(analysis, file) {
|
|
3869
4534
|
if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
|
|
@@ -4031,6 +4696,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
4031
4696
|
return at ? [`${name} (character ${at})`] : [];
|
|
4032
4697
|
});
|
|
4033
4698
|
}
|
|
4034
|
-
export {
|
|
4699
|
+
export { directOnCloud as $, evalPlan as A, agentMcp as At, profileJudge as B, refusedPlugins as Bt, openCodeModelOn as C, execOptionsProblem as Ct, cascade as D, AGENTS as Dt, brokeredIn as E, servedEffortOf as Et, matchesTags as F, expectedBlockedFor as Ft, sandboxSetBy as G, judgeProblem as Gt, runsOf as H, INPUT_KEYS as Ht, plainHeaderError as I, isAgentKind as It, unbrokeredError as J, decideState as Jt, secretVarsIn as K, artifactRefsOf as Kt, plainHeaders as L, keyRejected as Lt, harnessId as M, brokeringFor as Mt, levelOf as N, claudeMcpServers as Nt, checkLimitHarness as O, agentAuth as Ot, majorityOf as P, credentialOf as Pt, ciRunner as Q, plannedSandboxIds as R, layoutOf as Rt, fallbackHint as S, attemptRequestProblem as St, routedHarness as T, resolvedEffortOf as Tt, sandboxId as U, NO_JUDGE_INPUT as Ut, refuseIllegalCompare as V, writePluginFiles as Vt, sandboxProvider as W, inputsOn as Wt, DEFAULT_EGRESS_ALLOW as X, resolveJudgeArtifacts as Xt, DEFAULT_EGRESS as Y, filePartsOf as Yt, allowedHosts as Z, fellBackToDirect as _, observedHere as _t, configPathFor as a, createFakeSandbox as at, resolveModelGateway as b, CAPTURE_NAME as bt, findConfig as c, parseProbe as ct, malformedCredentials as d, STOPPED_FIRST as dt, egressLabel as et, missingCredentials as f, UNRECORDED_QUESTION_STATE as ft, selectProfile as g, resolveStateOptions as gt, runJudgeModel as h, mergeJudge as ht, configLabel as i, resolveOutput as it, expand as j, authHint as jt, checkOptions as k, agentFor as kt, hostCapacity as l, probeScript as lt, resolveConcurrency as m, resolveQuestionState as mt, analysisModelOf as n, isDirect as nt, defaultConcurrency as o, awaitListening as ot, requireConfig as p, questionShows as pt, unbrokered as q, badJudgeArtifacts as qt, configFilesPhrase as r, unenforceable as rt, defineConfig as s, hostListening as st, CONFIG_FILES as t, gatewayPolicy as tt, loadConfig as u, servePort as ut, gatewayScope as v, observedName as vt, resolveGateway as w, harnessEffort as wt, effortRefusedOn as x, CAPTURE_NAME_RULE as xt, modelGatewayOrNone as y, observer as yt, plannedSandboxes as z, refusedMcp as zt };
|
|
4035
4700
|
|
|
4036
|
-
//# sourceMappingURL=config-
|
|
4701
|
+
//# sourceMappingURL=config-C2UtS9vy.js.map
|