reeval 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-CptK032x.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-B63OzUX9.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-B63OzUX9.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Me7Whp-y.js → assemble-t6h018An.js} +152 -129
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
- package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-Cik0SoIn.js → attempt-host-CIm1-11m.js} +6 -6
- package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-C3RmjPph.js → capture-iBzFiC0j.js} +27 -25
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-BBOlXKPG.js → cli-20Fpvs5H.js} +442 -707
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-qGS802uX.js → command-B5OyHpOu.js} +21 -80
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-DhiwsZl1.js → command-BLNVDiIg.js} +9 -9
- package/dist/{command-DhiwsZl1.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-fO0BFsTq.js → command-CRh-cb6Z.js} +37 -37
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-CFQ9c1hk.js → command-aUJvENwy.js} +16 -16
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-CC60TOT5.js → commands-CWy-WndG.js} +186 -85
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-xYTMNTPv.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-C2UtS9vy.js → config-B9mB1kF5.js} +261 -1708
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
- package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-C5SZ0510.js → db-zgC1tvKB.js} +130 -84
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-FEidNAIi.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BVmx7r7L.js → default-effort-DGhQmu95.js} +4 -4
- package/dist/{default-effort-BVmx7r7L.js.map → default-effort-DGhQmu95.js.map} +1 -1
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-B2F8-PGC.js → docker-Cqw-CyWz.js} +3 -3
- package/dist/{docker-B2F8-PGC.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker-Dto8ROCq.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-BlSopHpZ.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-BcvfKpBX.js → doctor-B2PohneB.js} +84 -27
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{environment-CC1RV21n.js → environment-DB8o3hGi.js} +79 -39
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-CzxdgFBW.js → eval-source-D3sbAzt-.js} +10 -10
- package/dist/{eval-source-CzxdgFBW.js.map → eval-source-D3sbAzt-.js.map} +1 -1
- package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
- package/dist/{evaluate-NTNJ8T9H.js → evaluate-CwKeN_fo.js} +377 -142
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-DsXULUi5.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-DsXULUi5.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-aAzoGNCB.js → files-BiFU16BW.js} +5 -5
- package/dist/{files-aAzoGNCB.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-CZXrkPja.js → files-Cl7BQ5gE.js} +54 -13
- package/dist/{files-CZXrkPja.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-C1MB5bjr.js → git-fDiJ8UIV.js} +2 -2
- package/dist/{git-C1MB5bjr.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
- package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
- package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-CNayXCp2.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-D5HtmyMI.js → init-DbuZVRrW.js} +7 -6
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
- package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
- package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
- package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
- package/dist/{live-uRURFDMZ.js → live-CnGrRM6j.js} +7 -7
- package/dist/{live-uRURFDMZ.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
- package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-CXVVB4RK.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-BxBWFOZn.js → model-DJjLw-0r.js} +5 -5
- package/dist/{model-BxBWFOZn.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-ug5HpMOj.js → patch-DH4e_Gkd.js} +2 -2
- package/dist/{patch-ug5HpMOj.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-DqnzfDH1.js → plan-Cgq9NCEX.js} +110 -39
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-B2KQ0y3P.js → plugins-BkmPxDiY.js} +10 -267
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-DyzmvK7i.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-DVXEvzFU.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-21WMQZyt.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-21WMQZyt.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-BcSRa7-3.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
- package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-iRxjp4Pu.js → provenance-zsc4--dP.js} +4 -3
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-vXMXuwn-.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-BhaKQ07P.js → query-Bb3SYICk.js} +2 -2
- package/dist/{query-BhaKQ07P.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-BzdLdV3C.js → questions-CWlR9ffu.js} +9 -9
- package/dist/{questions-BzdLdV3C.js.map → questions-CWlR9ffu.js.map} +1 -1
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-CauU408t.js → remarks-vtkPmQb9.js} +115 -35
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-Dwy3fyo3.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-Dwy3fyo3.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B1zMnPwK.js → resolve-Do0ZaYH6.js} +10 -9
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-CfC0id_u.js → results-JyuM64gy.js} +6 -6
- package/dist/{results-CfC0id_u.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/{resume-Dkhrpjow.js → resume-AW5oXzUc.js} +169 -27
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-DcvQcr36.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-8zCrtuCf.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-Dplq8g0r.js → server-Cn8VqO9S.js} +9 -9
- package/dist/{server-Dplq8g0r.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-NWHqn5-o.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-NWHqn5-o.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-B9oE_mNS.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-BQ6BiXoc.js → src-Ck2r_EMT.js} +607 -72
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-CUpoVVb8.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-DDCGUhva.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-DDCGUhva.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +4 -4
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
- package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
- package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
- package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +101 -58
- package/dist/view/index.js +9 -8
- package/dist/{views-BISrsIjw.js → views-DAIiLiV3.js} +7 -6
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-DJD_nHkS.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-BO1VhUR9.js → workspace-uuE51x35.js} +6 -6
- package/dist/{workspace-BO1VhUR9.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +24 -11
- package/skills/eval-rubric/SKILL.md +15 -7
- package/dist/adapter-CptK032x.js.map +0 -1
- package/dist/app-files-sgqwfjtn.js.map +0 -1
- package/dist/assemble-CxBHkavM.js +0 -1
- package/dist/assemble-Me7Whp-y.js.map +0 -1
- package/dist/capture-C3RmjPph.js.map +0 -1
- package/dist/capture-YPwxp_iK.js +0 -1
- package/dist/cli-BBOlXKPG.js.map +0 -1
- package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
- package/dist/client-tZ2WBDxf.js.map +0 -1
- package/dist/command-CFQ9c1hk.js.map +0 -1
- package/dist/command-fO0BFsTq.js.map +0 -1
- package/dist/command-qGS802uX.js.map +0 -1
- package/dist/commands-CC60TOT5.js.map +0 -1
- package/dist/config-C2UtS9vy.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-BjOg7wyr.js +0 -101
- package/dist/contrasts-BjOg7wyr.js.map +0 -1
- package/dist/cost-BJU0MAji.js +0 -1
- package/dist/db-C5SZ0510.js.map +0 -1
- package/dist/db-puzLzX4k.js +0 -2
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker-Dto8ROCq.js.map +0 -1
- package/dist/doctor-BcvfKpBX.js.map +0 -1
- package/dist/environment-CC1RV21n.js.map +0 -1
- package/dist/evaluate-NTNJ8T9H.js.map +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-DFtuRQJx.js +0 -1
- package/dist/files-oOYNR7ZH.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-Cp_pQoEe.js.map +0 -1
- package/dist/harness-DYSC4zMh.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-CNayXCp2.js.map +0 -1
- package/dist/init-8o6W3_fn.js +0 -1
- package/dist/init-D5HtmyMI.js.map +0 -1
- package/dist/ink-list-Ca-5dunO.js.map +0 -1
- package/dist/judge-DYOF69Oe.js.map +0 -1
- package/dist/list-CM_YSJXq.js +0 -374
- package/dist/list-CM_YSJXq.js.map +0 -1
- package/dist/live-DQc8jr6o.js +0 -1
- package/dist/model-Ce9l9_rF.js.map +0 -1
- package/dist/model-Cxzo4cjw.js +0 -1
- package/dist/node-DhIRSrrK.js.map +0 -1
- package/dist/opencode-providers-Co7plCea.js.map +0 -1
- package/dist/plan-DqnzfDH1.js.map +0 -1
- package/dist/plugins-B2KQ0y3P.js.map +0 -1
- package/dist/preflight-DyzmvK7i.js.map +0 -1
- package/dist/provenance-iRxjp4Pu.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-D0ob_tra.js +0 -1
- package/dist/providers-vXMXuwn-.js.map +0 -1
- package/dist/query-CvgUsvep.js +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/remarks-CauU408t.js.map +0 -1
- package/dist/remarks-NMgPd30U.js.map +0 -1
- package/dist/resolve-B1zMnPwK.js.map +0 -1
- package/dist/resolve-BIIeVT12.js +0 -1
- package/dist/resume-Dkhrpjow.js.map +0 -1
- package/dist/runtime-DcvQcr36.js.map +0 -1
- package/dist/sandbox-8zCrtuCf.js.map +0 -1
- package/dist/server-0tuj8VSa.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-llwLewOU.js +0 -1
- package/dist/src-BQ6BiXoc.js.map +0 -1
- package/dist/src-BXnv0ViU.js +0 -2
- package/dist/src-BiS8YNN_.js +0 -3
- package/dist/src-BkH4GOxu.js +0 -1
- package/dist/src-C-tH5ebd.js +0 -2
- package/dist/src-Cg1bFs9g.js +0 -2
- package/dist/src-DV_UorZv.js +0 -2
- package/dist/src-XxUpr2sa.js +0 -2
- package/dist/state-DZbAO95a.js +0 -1
- package/dist/suite-M8_7xio7.js.map +0 -1
- package/dist/suite-u2KJnGpY.js +0 -2
- package/dist/summary-Uzc9osfV.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-h_peaZTN.js +0 -1
- package/dist/views-BISrsIjw.js.map +0 -1
- package/dist/views-Bfab8fM9.js +0 -2
- package/dist/workspace-B8EOCuxl.js +0 -1
|
@@ -1,38 +1,42 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
|
-
import { a as TURN_EDITED, r as withLease } from "./client-tZ2WBDxf.js";
|
|
3
1
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-B2KQ0y3P.js";
|
|
7
|
-
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
|
|
8
|
-
import { A as evalPlan, At as agentMcp, Bt as refusedPlugins, D as cascade, Dt as AGENTS, E as brokeredIn, F as matchesTags, Ft as expectedBlockedFor, G as sandboxSetBy, Gt as judgeProblem, H as runsOf, Ht as INPUT_KEYS, I as plainHeaderError, J as unbrokeredError, Jt as decideState, K as secretVarsIn, Kt as artifactRefsOf, L as plainHeaders, Lt as keyRejected, M as harnessId, Mt as brokeringFor, Nt as claudeMcpServers, P as majorityOf, Rt as layoutOf, U as sandboxId, Ut as NO_JUDGE_INPUT, V as refuseIllegalCompare, Vt as writePluginFiles, W as sandboxProvider, Wt as inputsOn, Xt as resolveJudgeArtifacts, Yt as filePartsOf, Z as allowedHosts, _ as fellBackToDirect, _t as observedHere, at as createFakeSandbox, b as resolveModelGateway, et as egressLabel, gt as resolveStateOptions, ht as mergeJudge, j as expand, k as checkOptions, nt as isDirect, q as unbrokered, qt as badJudgeArtifacts, rt as unenforceable, v as gatewayScope, vt as observedName, w as resolveGateway, yt as observer, zt as refusedMcp } from "./config-C2UtS9vy.js";
|
|
9
|
-
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-DYOF69Oe.js";
|
|
10
|
-
import { v as vertexSettings } from "./vertex-BLQMayc1.js";
|
|
11
|
-
import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
|
|
2
|
+
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-Cfi45Bvy.js";
|
|
3
|
+
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
12
4
|
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
5
|
+
import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
|
|
6
|
+
import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-C1oY3qyC.js";
|
|
7
|
+
import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-DB8o3hGi.js";
|
|
8
|
+
import { t as evalSlug } from "./slug-IHh3f770.js";
|
|
9
|
+
import { n as attemptKey, o as recordFileName, r as evalId, s as relativeFile, t as attemptId } from "./ids-C_advdJA.js";
|
|
10
|
+
import { $ as runsOf, A as brokeredIn, B as expand, Bt as decideState, Ct as resolveStateOptions, Et as observer, Ft as NO_JUDGE_INPUT, Ht as resolveJudgeArtifacts, It as inputsOn, J as plainHeaders, K as plainHeaderError, L as checkOptions, Lt as judgeProblem, Pt as INPUT_KEYS, Q as refuseIllegalCompare, Rt as artifactRefsOf, St as mergeJudge, Tt as observedName, U as majorityOf, V as harnessId, Vt as filePartsOf, W as matchesTags, _ as fellBackToDirect, at as unbrokered, b as resolveModelGateway, et as sandboxId, gt as unenforceable, ht as isDirect, it as secretVarsIn, j as cascade, nt as sandboxSetBy, ot as unbrokeredError, pt as egressLabel, tt as sandboxProvider, ut as allowedHosts, v as gatewayScope, w as resolveGateway, wt as observedHere, z as evalPlan, zt as badJudgeArtifacts } from "./config-B9mB1kF5.js";
|
|
11
|
+
import { l as vercelCredentialProblem } from "./vercel-CGlSBv63.js";
|
|
12
|
+
import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
|
|
13
|
+
import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
|
|
14
|
+
import { f as runJudge, h as isApiError, l as judgeModelName } from "./judge-COMXT1Hx.js";
|
|
13
15
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import { a as
|
|
19
|
-
import {
|
|
20
|
-
import { r as
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
import { a as
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
16
|
+
import { ht as OPAQUE_OUTPUT, n as MCP_TIMER, yt as shellQuote } from "./src-Ck2r_EMT.js";
|
|
17
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BkmPxDiY.js";
|
|
18
|
+
import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-C6zhkxp3.js";
|
|
19
|
+
import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
|
|
20
|
+
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-BpW1f1Mx.js";
|
|
21
|
+
import { c as formatBytes } from "./debug-CIXzB4wK.js";
|
|
22
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CAdlwlNu.js";
|
|
23
|
+
import { t as attemptDebugDir } from "./files-DlQshsks.js";
|
|
24
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
|
|
25
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-iBzFiC0j.js";
|
|
26
|
+
import { t as contextTaker } from "./context-Bkv3ndjq.js";
|
|
27
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-Do0ZaYH6.js";
|
|
28
|
+
import { t as PrepareError } from "./provider-BxzZq3Th.js";
|
|
29
|
+
import { i as preparedWorkspace } from "./workspace-uuE51x35.js";
|
|
27
30
|
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
28
|
-
import { n as judgeDiff } from "./patch-
|
|
29
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
31
|
+
import { n as judgeDiff } from "./patch-DH4e_Gkd.js";
|
|
32
|
+
import { t as transcriptForJudge } from "./transcript-CCf07UmU.js";
|
|
30
33
|
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
|
|
31
|
-
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-
|
|
32
|
-
import { i as withDefaultEffort } from "./default-effort-
|
|
34
|
+
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-D3sbAzt-.js";
|
|
35
|
+
import { i as withDefaultEffort } from "./default-effort-DGhQmu95.js";
|
|
36
|
+
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DKnYkqeK.js";
|
|
37
|
+
import { z } from "zod";
|
|
33
38
|
import { dirname, join } from "node:path";
|
|
34
39
|
import { parse, stringify } from "yaml";
|
|
35
|
-
import { z } from "zod";
|
|
36
40
|
import { randomBytes } from "node:crypto";
|
|
37
41
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
38
42
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
@@ -66,7 +70,7 @@ function assertionRecords(raw) {
|
|
|
66
70
|
const seen = /* @__PURE__ */ new Map();
|
|
67
71
|
const used = /* @__PURE__ */ new Set();
|
|
68
72
|
return raw.map((r) => {
|
|
69
|
-
const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
73
|
+
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
70
74
|
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
71
75
|
let n = seen.get(base) ?? 0;
|
|
72
76
|
let label;
|
|
@@ -112,11 +116,11 @@ function tracePlan(debug, spec, ctx) {
|
|
|
112
116
|
const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
|
|
113
117
|
const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
|
|
114
118
|
debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
|
|
115
|
-
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("
|
|
119
|
+
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
|
|
116
120
|
debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
|
|
117
121
|
const env = spec.environment;
|
|
118
122
|
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
119
|
-
const envFrom = spec.cell.role === "variant" ? from("compare") : from("
|
|
123
|
+
const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
|
|
120
124
|
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
121
125
|
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
122
126
|
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
@@ -254,11 +258,12 @@ const createFakeHarness = (descriptor) => {
|
|
|
254
258
|
text,
|
|
255
259
|
status,
|
|
256
260
|
startedAt: new Date(startedAt).toISOString(),
|
|
257
|
-
durationMs: Date.now() - startedAt,
|
|
261
|
+
durationMs: descriptor.turnMs ?? Date.now() - startedAt,
|
|
258
262
|
transcript,
|
|
259
|
-
usage: null,
|
|
263
|
+
usage: descriptor.usage ?? null,
|
|
260
264
|
model: descriptor.servedModel ?? null,
|
|
261
|
-
unreported: []
|
|
265
|
+
unreported: [],
|
|
266
|
+
...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
|
|
262
267
|
});
|
|
263
268
|
const write = async (path, content) => {
|
|
264
269
|
const time = at();
|
|
@@ -386,7 +391,7 @@ async function brokerKey(descriptor, sandbox) {
|
|
|
386
391
|
const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
|
|
387
392
|
const key = process.env[credential];
|
|
388
393
|
if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
|
|
389
|
-
const network =
|
|
394
|
+
const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
|
|
390
395
|
if (!network?.setRequestTransformations) throw new InfraError("fakeHarness({ brokers: true }) needs docker(): this sandbox can't broker");
|
|
391
396
|
const placeholder = `reeval_fake_${randomBytes(24).toString("base64url")}`;
|
|
392
397
|
await network.setRequestTransformations([...new Set(endpoints.map((e) => e.host))].map((host) => ({
|
|
@@ -478,18 +483,19 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
478
483
|
};
|
|
479
484
|
}
|
|
480
485
|
// src/harness/index.ts
|
|
481
|
-
|
|
486
|
+
/** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
|
|
487
|
+
async function createHarness(descriptor, sandbox, vertex) {
|
|
482
488
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
483
|
-
const { createAgentHarness } = await import("./agent-
|
|
484
|
-
return createAgentHarness(descriptor, vertex);
|
|
489
|
+
const { createAgentHarness } = await import("./agent-CBoLndM7.js");
|
|
490
|
+
return createAgentHarness(descriptor, sandbox, vertex);
|
|
485
491
|
}
|
|
486
492
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
487
493
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
488
494
|
// src/attempt/environment.ts
|
|
489
495
|
/**
|
|
490
496
|
* Resolves the attempt's environment (files, skills, secrets and their placeholders)
|
|
491
|
-
* and finds the image
|
|
492
|
-
* signal, before the eval timer starts.
|
|
497
|
+
* and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
|
|
498
|
+
* Runs on the test's signal, before the eval timer starts.
|
|
493
499
|
*/
|
|
494
500
|
async function prepareEnvironment(a, root, runId, where, signal) {
|
|
495
501
|
const { spec, record } = a;
|
|
@@ -508,13 +514,14 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
508
514
|
const debug = attemptDebug();
|
|
509
515
|
if (debug) traceEnvironment(debug, spec, env);
|
|
510
516
|
record.image = null;
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
const {
|
|
517
|
+
const kind = sandboxKind(spec.sandbox);
|
|
518
|
+
if (kind.capabilities.images) {
|
|
519
|
+
const { preparedFor } = await import("./preflight-R-InZCQb.js");
|
|
520
|
+
const { attemptImage, createProviders, preparedImage } = await import("./providers-CVu8e0xS.js");
|
|
514
521
|
signal.throwIfAborted();
|
|
515
|
-
const provider =
|
|
522
|
+
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
516
523
|
const builds = builtPlugins(spec.environment.layers);
|
|
517
|
-
const prepared = preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
|
|
524
|
+
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
|
|
518
525
|
if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
|
|
519
526
|
record.image = attemptImage(prepared);
|
|
520
527
|
a.image = record.image.ref;
|
|
@@ -534,7 +541,16 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
534
541
|
}
|
|
535
542
|
const { ref, id, base, install, agentCli } = record.image;
|
|
536
543
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
537
|
-
} else
|
|
544
|
+
} else if (kind.capabilities.isolated) {
|
|
545
|
+
const { preparedFor } = await import("./preflight-R-InZCQb.js");
|
|
546
|
+
const { createProviders, vmImage } = await import("./providers-CVu8e0xS.js");
|
|
547
|
+
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
548
|
+
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
|
|
549
|
+
const vm = prepared ? vmImage(prepared) : null;
|
|
550
|
+
if (vm) record.vmImage = vm;
|
|
551
|
+
const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
|
|
552
|
+
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
|
|
553
|
+
} else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
538
554
|
}
|
|
539
555
|
// src/judge/size.ts
|
|
540
556
|
/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
|
|
@@ -854,18 +870,12 @@ function judgeCaller(a, readFile) {
|
|
|
854
870
|
}
|
|
855
871
|
// src/sandbox/index.ts
|
|
856
872
|
async function createSandbox(descriptor, options) {
|
|
873
|
+
const kind = sandboxKind(descriptor);
|
|
857
874
|
try {
|
|
858
|
-
|
|
859
|
-
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
860
|
-
case "docker": {
|
|
861
|
-
const { createDockerSandbox } = await import("./docker-BlSopHpZ.js");
|
|
862
|
-
return await createDockerSandbox(descriptor, options);
|
|
863
|
-
}
|
|
864
|
-
}
|
|
875
|
+
return await kind.create(descriptor, options);
|
|
865
876
|
} catch (error) {
|
|
866
877
|
if (error instanceof InfraError || options.signal.aborted) throw error;
|
|
867
|
-
|
|
868
|
-
throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
|
|
878
|
+
throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
|
|
869
879
|
}
|
|
870
880
|
}
|
|
871
881
|
// src/attempt/record.ts
|
|
@@ -1323,7 +1333,7 @@ const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s")
|
|
|
1323
1333
|
function setupAllowance(planned) {
|
|
1324
1334
|
const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
|
|
1325
1335
|
const { sandbox } = planned;
|
|
1326
|
-
return setupMs + (sandbox.
|
|
1336
|
+
return setupMs + (sandboxKind(sandbox).buildsAtStart(sandbox) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
|
|
1327
1337
|
}
|
|
1328
1338
|
/** The eval's timer, paused while the sandbox and its setup steps start. */
|
|
1329
1339
|
function evalTimer(ms) {
|
|
@@ -1405,7 +1415,7 @@ const SETUP_COMMAND_CHARS = 200;
|
|
|
1405
1415
|
*/
|
|
1406
1416
|
async function leftRunning(a, box, env) {
|
|
1407
1417
|
const names = unbrokeredChecks(env);
|
|
1408
|
-
if (a.spec.sandbox.
|
|
1418
|
+
if (!capabilitiesOf(a.spec.sandbox).processTable || !env.setup.length || !names.length) return [];
|
|
1409
1419
|
const r = await box.exec([
|
|
1410
1420
|
"for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
|
|
1411
1421
|
"for p in /proc/[0-9]*; do",
|
|
@@ -1641,7 +1651,7 @@ async function startSession(a, runId) {
|
|
|
1641
1651
|
})
|
|
1642
1652
|
});
|
|
1643
1653
|
a.sandbox = box;
|
|
1644
|
-
if (spec.sandbox.
|
|
1654
|
+
if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
|
|
1645
1655
|
a.phases = brokerPhases(box, env.substitutions);
|
|
1646
1656
|
if (a.signal.aborted) box.destroy();
|
|
1647
1657
|
a.signal.throwIfAborted();
|
|
@@ -1687,7 +1697,7 @@ async function startSession(a, runId) {
|
|
|
1687
1697
|
debugLog("env", "stdio MCP servers start through reeval's MCP timer");
|
|
1688
1698
|
} else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
|
|
1689
1699
|
if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
|
|
1690
|
-
const harness = await createHarness(spec.harness, spec.vertex);
|
|
1700
|
+
const harness = await createHarness(spec.harness, spec.sandbox, spec.vertex);
|
|
1691
1701
|
const harnessStart = performance.now();
|
|
1692
1702
|
debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
|
|
1693
1703
|
const session = await startHarness(a, harness, box, {
|
|
@@ -2070,7 +2080,8 @@ function turnSender(a, takeContext, checks) {
|
|
|
2070
2080
|
durationMs: Date.now() - sendStarted
|
|
2071
2081
|
};
|
|
2072
2082
|
});
|
|
2073
|
-
const
|
|
2083
|
+
const timedBefore = a.record.timings.agentMs;
|
|
2084
|
+
const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2074
2085
|
entry.reasoningMs = null;
|
|
2075
2086
|
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
2076
2087
|
throw error;
|
|
@@ -2078,7 +2089,8 @@ function turnSender(a, takeContext, checks) {
|
|
|
2078
2089
|
cancel();
|
|
2079
2090
|
watch?.close();
|
|
2080
2091
|
});
|
|
2081
|
-
const sendWallMs = Date.now() - sendStarted;
|
|
2092
|
+
const sendWallMs = agentMs ?? Date.now() - sendStarted;
|
|
2093
|
+
if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
|
|
2082
2094
|
const startup = await watch?.end();
|
|
2083
2095
|
const timedOut = startup?.timedOut && mcpTimeoutMessage(startup.timedOut);
|
|
2084
2096
|
const seenAt = startup?.at ?? Date.now();
|
|
@@ -2291,6 +2303,37 @@ function bestOfCell(bestOf) {
|
|
|
2291
2303
|
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2292
2304
|
});
|
|
2293
2305
|
}
|
|
2306
|
+
/** Characters a case key can't hold: ids and paths split on them, and control characters. */
|
|
2307
|
+
const BAD_KEY = /[#/›\u0000-\u001f\u007f]/;
|
|
2308
|
+
/** The name a case's eval plans under: `adds a health check [nextjs]`. */
|
|
2309
|
+
const caseEvalName = (name, key) => `${name} [${key}]`;
|
|
2310
|
+
/** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
|
|
2311
|
+
function defaultKey(item) {
|
|
2312
|
+
if (typeof item === "string") return item;
|
|
2313
|
+
if (typeof item === "number" && Number.isFinite(item)) return String(item);
|
|
2314
|
+
if (isObjectLike(item) && !Array.isArray(item)) {
|
|
2315
|
+
if (typeof item.id === "string") return item.id;
|
|
2316
|
+
if (typeof item.name === "string") return item.name;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
/**
|
|
2320
|
+
* Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
|
|
2321
|
+
* characters, with no `#`, `/`, `›` or control characters, and unique within the eval.
|
|
2322
|
+
*/
|
|
2323
|
+
function caseKeys(cases, caseKey, where) {
|
|
2324
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2325
|
+
return cases.map((item, i) => {
|
|
2326
|
+
const raw = caseKey ? caseKey(item) : defaultKey(item);
|
|
2327
|
+
if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
|
|
2328
|
+
if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
|
|
2329
|
+
const key = raw.trim();
|
|
2330
|
+
if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, /, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
|
|
2331
|
+
const earlier = seen.get(key);
|
|
2332
|
+
if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
|
|
2333
|
+
seen.set(key, i);
|
|
2334
|
+
return key;
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2294
2337
|
// src/contract/declaration.ts
|
|
2295
2338
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
2296
2339
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
@@ -2302,7 +2345,7 @@ const declaredModeSchema = z.enum([
|
|
|
2302
2345
|
const declaredFamilySchema = z.strictObject({
|
|
2303
2346
|
name: z.string(),
|
|
2304
2347
|
case: z.number().int().min(0),
|
|
2305
|
-
cases: z.number().int().min(
|
|
2348
|
+
cases: z.number().int().min(1)
|
|
2306
2349
|
});
|
|
2307
2350
|
const declaredTodoSchema = z.strictObject({
|
|
2308
2351
|
type: z.literal("todo"),
|
|
@@ -2347,6 +2390,89 @@ const NODE_SCHEMAS = {
|
|
|
2347
2390
|
* tree's `children`. Options are left to the cascade.
|
|
2348
2391
|
*/
|
|
2349
2392
|
const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
|
|
2393
|
+
// src/where.ts
|
|
2394
|
+
/** Keys a trait lookup can hit without naming a trait: JSON, promises and inspection. */
|
|
2395
|
+
const PASSTHROUGH = /* @__PURE__ */ new Set([
|
|
2396
|
+
"toJSON",
|
|
2397
|
+
"then",
|
|
2398
|
+
"constructor"
|
|
2399
|
+
]);
|
|
2400
|
+
function pickPairs(w, attempts) {
|
|
2401
|
+
const controls = attempts.filter((a) => a.cell.role === "control");
|
|
2402
|
+
const known = [...new Set(controls.flatMap((a) => Object.keys(traitsOf(a.environment.layers))))];
|
|
2403
|
+
const decisions = /* @__PURE__ */ new Map();
|
|
2404
|
+
const decide = (control) => {
|
|
2405
|
+
const id = control.cell.key;
|
|
2406
|
+
const made = decisions.get(id);
|
|
2407
|
+
if (made) return made;
|
|
2408
|
+
const env = control.environment;
|
|
2409
|
+
const traits = traitsOf(env.layers);
|
|
2410
|
+
let readHarness = false;
|
|
2411
|
+
const view = new Proxy(traits, { get(target, key, receiver) {
|
|
2412
|
+
if (typeof key === "string" && !(key in target) && !PASSTHROUGH.has(key)) {
|
|
2413
|
+
if (!known.includes(key)) {
|
|
2414
|
+
const near = closest(key, known);
|
|
2415
|
+
throw new UsageError(`${w.label}'s where reads environment.traits.${key}, which no environment defines (${known.length ? `they define: ${known.join(", ")}` : "none defines any"}).${near ? ` Did you mean "${near}"?` : ""}`);
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
return Reflect.get(target, key, receiver);
|
|
2419
|
+
} });
|
|
2420
|
+
const pair = {
|
|
2421
|
+
case: w.item,
|
|
2422
|
+
environment: Object.freeze({
|
|
2423
|
+
name: env.name,
|
|
2424
|
+
traits: view
|
|
2425
|
+
}),
|
|
2426
|
+
get harness() {
|
|
2427
|
+
readHarness = true;
|
|
2428
|
+
return control.cell.harness;
|
|
2429
|
+
}
|
|
2430
|
+
};
|
|
2431
|
+
let result;
|
|
2432
|
+
try {
|
|
2433
|
+
result = w.where(pair);
|
|
2434
|
+
} catch (error) {
|
|
2435
|
+
if (error instanceof UsageError) throw error;
|
|
2436
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
2437
|
+
throw new UsageError(`${w.label}'s where threw for case ${w.key} on ${env.name}: ${why}`);
|
|
2438
|
+
}
|
|
2439
|
+
if (typeof result !== "boolean") throw new UsageError(`${w.label}'s where returned ${JSON.stringify(result) ?? String(result)} for case ${w.key} on ${env.name}; it must return true or false.`);
|
|
2440
|
+
const decided = {
|
|
2441
|
+
keep: result,
|
|
2442
|
+
readHarness
|
|
2443
|
+
};
|
|
2444
|
+
decisions.set(id, decided);
|
|
2445
|
+
return decided;
|
|
2446
|
+
};
|
|
2447
|
+
const controlOf = (a) => a.cell.role === "control" ? a : controls.find((c) => c.cell.harness === a.cell.harness);
|
|
2448
|
+
const cells = [];
|
|
2449
|
+
const keptCells = [];
|
|
2450
|
+
const excluded = [];
|
|
2451
|
+
return {
|
|
2452
|
+
kept: attempts.filter((a) => {
|
|
2453
|
+
const { keep, readHarness } = decide(controlOf(a));
|
|
2454
|
+
const first = !cells.some((c) => c.key === a.cell.key);
|
|
2455
|
+
if (first) cells.push({
|
|
2456
|
+
key: a.cell.key,
|
|
2457
|
+
environment: a.cell.environment
|
|
2458
|
+
});
|
|
2459
|
+
if (keep) {
|
|
2460
|
+
if (first) keptCells.push(a.cell.key);
|
|
2461
|
+
return true;
|
|
2462
|
+
}
|
|
2463
|
+
const pair = {
|
|
2464
|
+
case: w.key,
|
|
2465
|
+
environment: a.cell.environment,
|
|
2466
|
+
...readHarness ? { harness: a.cell.harness } : {}
|
|
2467
|
+
};
|
|
2468
|
+
if (!excluded.some((e) => e.environment === pair.environment && e.harness === pair.harness)) excluded.push(pair);
|
|
2469
|
+
return false;
|
|
2470
|
+
}),
|
|
2471
|
+
excluded,
|
|
2472
|
+
cells,
|
|
2473
|
+
keptCells
|
|
2474
|
+
};
|
|
2475
|
+
}
|
|
2350
2476
|
// src/plan-eval.ts
|
|
2351
2477
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
2352
2478
|
const GRACE_MS = 5e3;
|
|
@@ -2355,7 +2481,7 @@ function configLevel(config) {
|
|
|
2355
2481
|
return {
|
|
2356
2482
|
harness: config.harness,
|
|
2357
2483
|
sandbox: config.sandbox,
|
|
2358
|
-
|
|
2484
|
+
environments: config.environments,
|
|
2359
2485
|
compare: void 0,
|
|
2360
2486
|
runs,
|
|
2361
2487
|
bestOf,
|
|
@@ -2390,19 +2516,80 @@ function planEval(declaration, ctx) {
|
|
|
2390
2516
|
profile: ctx.profile,
|
|
2391
2517
|
runs: ctx.runs,
|
|
2392
2518
|
skipped: declaration.mode === "skip" || declaration.mode === "todo",
|
|
2393
|
-
pooled: declaration.family
|
|
2519
|
+
pooled: (declaration.family?.cases ?? 0) >= 2
|
|
2394
2520
|
});
|
|
2395
2521
|
const { harnesses } = resolvedEval;
|
|
2396
2522
|
if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
|
|
2397
2523
|
tests: null,
|
|
2398
|
-
harnesses
|
|
2524
|
+
harnesses,
|
|
2525
|
+
traits: []
|
|
2526
|
+
};
|
|
2527
|
+
const { resolved } = resolvedEval;
|
|
2528
|
+
const declared = expand({
|
|
2529
|
+
...resolved,
|
|
2530
|
+
harness: resolved.harness.map((h) => withDefaultEffort(h, ctx.cursorEfforts))
|
|
2531
|
+
});
|
|
2532
|
+
const traits = controlTraitKeys(declared);
|
|
2533
|
+
const filters = traitFilters(ctx.traits, ctx.profile);
|
|
2534
|
+
const traitSkip = filters.length > 0 && !resolvedEval.skip && !pickTraits(declared, filters).kept.length;
|
|
2535
|
+
const narrowed = traitSkip ? {
|
|
2536
|
+
...resolvedEval,
|
|
2537
|
+
skip: noCellLeft(declared, filters)
|
|
2538
|
+
} : resolvedEval;
|
|
2539
|
+
const picks = [];
|
|
2540
|
+
if (traitSkip && declaration.where) picks.push(pickPairs(declaration.where, declared));
|
|
2541
|
+
let traitExcluded = [];
|
|
2542
|
+
const pick = (attempts) => {
|
|
2543
|
+
let kept = attempts;
|
|
2544
|
+
if (declaration.where) {
|
|
2545
|
+
const picked = pickPairs(declaration.where, attempts);
|
|
2546
|
+
picks.push(picked);
|
|
2547
|
+
kept = picked.kept;
|
|
2548
|
+
}
|
|
2549
|
+
if (!filters.length) return kept;
|
|
2550
|
+
const byTraits = pickTraits(kept, filters);
|
|
2551
|
+
traitExcluded = byTraits.excluded;
|
|
2552
|
+
return byTraits.kept;
|
|
2553
|
+
};
|
|
2554
|
+
const leftOut = () => `${filtersLabel(filters)} ${filters.length === 1 ? "leaves" : "leave"} out ${traitExcluded.join(", ")}`;
|
|
2555
|
+
let tests = planTests(declaration, ctx, where, narrowed, pick);
|
|
2556
|
+
const allLeftOut = !tests.length && traitExcluded.length > 0;
|
|
2557
|
+
if (allLeftOut) {
|
|
2558
|
+
const skip = `no cell left: ${leftOut()}`;
|
|
2559
|
+
tests = planTests(declaration, ctx, where, {
|
|
2560
|
+
...narrowed,
|
|
2561
|
+
skip
|
|
2562
|
+
}, pick);
|
|
2563
|
+
}
|
|
2564
|
+
if (traitExcluded.length) {
|
|
2565
|
+
for (const t of tests) t.attempt.plan.traitExcluded = traitExcluded;
|
|
2566
|
+
const notes = new Set(tests.map((t) => t.attempt.plan.notes));
|
|
2567
|
+
if (!allLeftOut) for (const list of notes) list.push(leftOut());
|
|
2568
|
+
}
|
|
2569
|
+
const [picked] = picks;
|
|
2570
|
+
if (!picked) return {
|
|
2571
|
+
tests,
|
|
2572
|
+
harnesses,
|
|
2573
|
+
traits
|
|
2399
2574
|
};
|
|
2575
|
+
const { excluded, cells, keptCells } = picked;
|
|
2400
2576
|
return {
|
|
2401
|
-
tests
|
|
2402
|
-
harnesses
|
|
2577
|
+
tests,
|
|
2578
|
+
harnesses,
|
|
2579
|
+
traits,
|
|
2580
|
+
picked: {
|
|
2581
|
+
excluded,
|
|
2582
|
+
cells,
|
|
2583
|
+
keptCells
|
|
2584
|
+
}
|
|
2403
2585
|
};
|
|
2404
2586
|
}
|
|
2405
|
-
|
|
2587
|
+
/** Why an eval none of whose control environments the trait filters match doesn't run. */
|
|
2588
|
+
function noCellLeft(attempts, filters) {
|
|
2589
|
+
const envs = [...new Set(attempts.filter((a) => a.cell.role === "control").map((a) => a.cell.environment))];
|
|
2590
|
+
return `no cell left: ${filtersLabel(filters)} ${filters.length === 1 ? "matches" : "match"} none of ${envs.join(", ")}`;
|
|
2591
|
+
}
|
|
2592
|
+
function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
2406
2593
|
const { name, mode, groups: enclosing } = declaration;
|
|
2407
2594
|
const groups = enclosing.map((g) => g.name);
|
|
2408
2595
|
const { resolved } = resolvedEval;
|
|
@@ -2438,10 +2625,10 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2438
2625
|
gateway: route.name
|
|
2439
2626
|
};
|
|
2440
2627
|
}).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
|
|
2441
|
-
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
2628
|
+
const plannedAttempts = resolvedEval.skip ? [] : pick(expand({
|
|
2442
2629
|
...resolved,
|
|
2443
2630
|
harness: routed
|
|
2444
|
-
}, vertex);
|
|
2631
|
+
}, vertex));
|
|
2445
2632
|
const { caseJson, ...declared } = declaration.source;
|
|
2446
2633
|
const source = {
|
|
2447
2634
|
...declared,
|
|
@@ -2473,6 +2660,8 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2473
2660
|
const why = unenforceable(planned.egress, planned.sandbox, where);
|
|
2474
2661
|
if (why) throw new UsageError(why);
|
|
2475
2662
|
const on = `${where} on ${harnessId(planned.harness)}`;
|
|
2663
|
+
const unverified = unverifiedHarness(planned.harness, planned.sandbox);
|
|
2664
|
+
if (unverified) throw new UsageError(`${on}: ${unverified}`);
|
|
2476
2665
|
const mcp = refusedMcp(planned.harness, planned.environment.layers);
|
|
2477
2666
|
if (mcp) throw new UsageError(`${on}: ${mcp}`);
|
|
2478
2667
|
const plugins = planPlugins(planned, ctx.config.root);
|
|
@@ -2486,7 +2675,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2486
2675
|
id: evalId(rankFile, [...groups, declaration.family.name]),
|
|
2487
2676
|
...declaration.family
|
|
2488
2677
|
};
|
|
2489
|
-
const harnessBy = resolved.sources?.
|
|
2678
|
+
const harnessBy = resolved.sources?.harnesses ?? ctx.config.file;
|
|
2490
2679
|
const plan = evalPlan(resolvedEval, plannedAttempts, {
|
|
2491
2680
|
configFile: ctx.config.file,
|
|
2492
2681
|
file: path
|
|
@@ -2668,7 +2857,8 @@ function planPlugins(planned, root) {
|
|
|
2668
2857
|
function attemptLimits(planned, concurrency) {
|
|
2669
2858
|
const limits = { attempts: concurrency };
|
|
2670
2859
|
const { sandbox, harness } = planned;
|
|
2671
|
-
|
|
2860
|
+
const cap = providerLimit(sandbox);
|
|
2861
|
+
if (cap) limits[`sandbox:${sandboxProvider(sandbox)}`] = cap;
|
|
2672
2862
|
if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
|
|
2673
2863
|
return limits;
|
|
2674
2864
|
}
|
|
@@ -2703,7 +2893,7 @@ function withPaths(options, where, base = callerDir() ?? process.cwd()) {
|
|
|
2703
2893
|
}
|
|
2704
2894
|
/** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
|
|
2705
2895
|
function splitArgs(what, name, a, b) {
|
|
2706
|
-
if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (
|
|
2896
|
+
if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environments: …) or in the config's env.`);
|
|
2707
2897
|
if (typeof a === "function") return [b ?? {}, a];
|
|
2708
2898
|
if (typeof b === "function") return [a ?? {}, b];
|
|
2709
2899
|
throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
|
|
@@ -2804,7 +2994,7 @@ const families = /* @__PURE__ */ new Map();
|
|
|
2804
2994
|
/** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
|
|
2805
2995
|
const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
|
|
2806
2996
|
/** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
|
|
2807
|
-
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as
|
|
2997
|
+
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as evaluate("${family.join(" › ")}") with cases in this file. Give one of them another name.${separatorHint(eval_, family)}`);
|
|
2808
2998
|
function checkUnique(where, groups, name, hint, inFamily) {
|
|
2809
2999
|
const path = [...groups, name];
|
|
2810
3000
|
const id = evalId(collectingFile() ?? "", path);
|
|
@@ -2835,11 +3025,14 @@ function checkFamily(groups, family) {
|
|
|
2835
3025
|
});
|
|
2836
3026
|
return;
|
|
2837
3027
|
}
|
|
2838
|
-
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate
|
|
3028
|
+
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate("${path.join(" › ")}") with cases is declared twice in this file. Each eval needs its own name, or their cases are pooled into one. Give each its own name.${separatorHint(path, earlier.path)}`);
|
|
2839
3029
|
earlier.seen.add(family.case);
|
|
2840
3030
|
}
|
|
2841
3031
|
function register(mode, given, own, fn, source, hint = "", givenFamily) {
|
|
2842
|
-
|
|
3032
|
+
prepare(mode, given, own, fn, source, hint, givenFamily).commit();
|
|
3033
|
+
}
|
|
3034
|
+
function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
|
|
3035
|
+
const name = shownName("eval", given, givenFamily ? " Give each case a key that holds no secret." : "");
|
|
2843
3036
|
const family = givenFamily && {
|
|
2844
3037
|
...givenFamily,
|
|
2845
3038
|
name: redactSecrets(givenFamily.name)
|
|
@@ -2858,7 +3051,7 @@ function register(mode, given, own, fn, source, hint = "", givenFamily) {
|
|
|
2858
3051
|
source,
|
|
2859
3052
|
...family ? { family } : {}
|
|
2860
3053
|
});
|
|
2861
|
-
const
|
|
3054
|
+
const planned = planEval({
|
|
2862
3055
|
name,
|
|
2863
3056
|
mode,
|
|
2864
3057
|
groups: groupStack.map(({ name, options }) => ({
|
|
@@ -2867,86 +3060,130 @@ function register(mode, given, own, fn, source, hint = "", givenFamily) {
|
|
|
2867
3060
|
})),
|
|
2868
3061
|
options: own,
|
|
2869
3062
|
source,
|
|
2870
|
-
...family ? { family } : {}
|
|
3063
|
+
...family ? { family } : {},
|
|
3064
|
+
...where ? { where } : {}
|
|
2871
3065
|
}, {
|
|
2872
3066
|
...getRunContext(),
|
|
2873
3067
|
file: collectingFile(),
|
|
2874
3068
|
order: registered
|
|
2875
3069
|
});
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
registered++;
|
|
2881
|
-
const bestOfCells = /* @__PURE__ */ new Map();
|
|
2882
|
-
for (const { name: testName, spec, ...options } of tests) {
|
|
2883
|
-
const { cell, n, bestOf } = options.attempt;
|
|
2884
|
-
let hooks = {};
|
|
2885
|
-
if (bestOf !== void 0) {
|
|
2886
|
-
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
2887
|
-
bestOfCells.set(cell.key, of);
|
|
2888
|
-
hooks = of(n);
|
|
2889
|
-
}
|
|
2890
|
-
registerTest(testName, {
|
|
2891
|
-
...options,
|
|
2892
|
-
...hooks
|
|
2893
|
-
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
3070
|
+
if (planned.tests) {
|
|
3071
|
+
const note = maskedNote([...groupStack.map((g) => g.given), given]);
|
|
3072
|
+
if (note) for (const notes of new Set(planned.tests.map((t) => t.attempt.plan.notes))) notes.push(note);
|
|
3073
|
+
registered++;
|
|
2894
3074
|
}
|
|
3075
|
+
const commit = () => {
|
|
3076
|
+
declarePlanned(planned.harnesses, planned.traits);
|
|
3077
|
+
if (!planned.tests) return;
|
|
3078
|
+
const bestOfCells = /* @__PURE__ */ new Map();
|
|
3079
|
+
for (const { name: testName, spec, ...options } of planned.tests) {
|
|
3080
|
+
const { cell, n, bestOf } = options.attempt;
|
|
3081
|
+
let hooks = {};
|
|
3082
|
+
if (bestOf !== void 0) {
|
|
3083
|
+
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
3084
|
+
bestOfCells.set(cell.key, of);
|
|
3085
|
+
hooks = of(n);
|
|
3086
|
+
}
|
|
3087
|
+
registerTest(testName, {
|
|
3088
|
+
...options,
|
|
3089
|
+
...hooks
|
|
3090
|
+
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
3091
|
+
}
|
|
3092
|
+
};
|
|
3093
|
+
return {
|
|
3094
|
+
planned,
|
|
3095
|
+
commit
|
|
3096
|
+
};
|
|
2895
3097
|
}
|
|
2896
3098
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2897
3099
|
let registered = 0;
|
|
2898
3100
|
const defineEval = (mode, declared) => (name, a, b) => {
|
|
2899
3101
|
checkOnly(mode, "evaluate");
|
|
2900
3102
|
const shown = redactSecrets(name);
|
|
2901
|
-
const [
|
|
3103
|
+
const [written, fn] = splitArgs("evaluate", shown, a, b);
|
|
2902
3104
|
const where = `evaluate("${shown}")`;
|
|
2903
|
-
|
|
3105
|
+
if (isObjectLike(written) && "where" in written && !("cases" in written)) throw new UsageError(`${where} sets where without cases. where picks case-and-environment pairs; to drop an environment, leave it out of environments.`);
|
|
3106
|
+
if (isObjectLike(written) && ("cases" in written || "caseKey" in written)) return registerCases(mode, name, written, fn, where);
|
|
3107
|
+
register(mode, name, withPaths(written, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
|
|
2904
3108
|
};
|
|
2905
|
-
/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
|
|
2906
|
-
function formatName(template, item, index) {
|
|
2907
|
-
return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => isObjectLike(item) && key in item ? String(item[key]) : whole);
|
|
2908
|
-
}
|
|
2909
|
-
function checkEachOptions(options) {
|
|
2910
|
-
if (options === void 0) return {};
|
|
2911
|
-
if (isObjectLike(options) && Object.keys(options).every((k) => k === "pool")) {
|
|
2912
|
-
const { pool } = options;
|
|
2913
|
-
if (pool === void 0) return {};
|
|
2914
|
-
if (typeof pool === "boolean") return { pool };
|
|
2915
|
-
}
|
|
2916
|
-
throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
|
|
2917
|
-
}
|
|
2918
3109
|
/**
|
|
2919
|
-
*
|
|
2920
|
-
*
|
|
3110
|
+
* An eval with `cases`: each case plans as an entry of its own named for the eval, which the
|
|
3111
|
+
* record folds back into the eval. Two or more cases pool into one pass rate and, with
|
|
3112
|
+
* `compare`, one lift paired by case.
|
|
2921
3113
|
*/
|
|
2922
|
-
|
|
2923
|
-
const {
|
|
2924
|
-
if (!Array.isArray(cases)) throw new UsageError(
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
const
|
|
3114
|
+
function registerCases(mode, name, written, fn, where) {
|
|
3115
|
+
const { cases, caseKey, where: pick, ...rest } = written;
|
|
3116
|
+
if (!Array.isArray(cases)) throw new UsageError(cases === void 0 ? `caseKey in ${where} names each case, but there are no cases. Add cases: [...], or drop caseKey.` : `cases in ${where} must be a list (got ${typeof cases}), e.g. cases: ['users', 'posts'].`);
|
|
3117
|
+
if (cases.length === 0) throw new UsageError(`cases in ${where} is an empty list. Give at least one case.`);
|
|
3118
|
+
if (caseKey !== void 0 && typeof caseKey !== "function") throw new UsageError(`caseKey in ${where} must be a function from a case to its key, e.g. caseKey: (item) => item.slug.`);
|
|
3119
|
+
if (pick !== void 0 && typeof pick !== "function") throw new UsageError(`where in ${where} must be a function from a case and an environment to true or false, e.g. where: ({ case: item, environment }) => item.stacks.includes(environment.traits.stack).`);
|
|
3120
|
+
if (isObjectLike(rest.runs) && "bestOf" in rest.runs) throw new UsageError(`${where} has cases, so it can't take runs: { bestOf }: best-of stops each cell on its own, and the cases pool over the runs. Set a fixed count, e.g. runs: 3.`);
|
|
3121
|
+
const keys = caseKeys(cases, caseKey, where);
|
|
3122
|
+
const shownKeys = keys.map((k) => redactSecrets(k));
|
|
3123
|
+
const own = withPaths(rest, where);
|
|
2930
3124
|
const file = collectingFile();
|
|
2931
|
-
const body = bodyHash(fn,
|
|
2932
|
-
cases.
|
|
2933
|
-
const
|
|
3125
|
+
const body = bodyHash(fn, where, file);
|
|
3126
|
+
const prepared = cases.map((item, i) => {
|
|
3127
|
+
const key = shownKeys[i];
|
|
3128
|
+
const family = {
|
|
2934
3129
|
name,
|
|
2935
3130
|
case: i,
|
|
2936
3131
|
cases: cases.length
|
|
2937
|
-
}
|
|
3132
|
+
};
|
|
2938
3133
|
const source = {
|
|
2939
3134
|
body,
|
|
2940
|
-
caseJson: caseJson(item,
|
|
3135
|
+
caseJson: caseJson(item, `${where} case "${key}"`, file)
|
|
2941
3136
|
};
|
|
2942
|
-
|
|
3137
|
+
const run = (c) => fn(c, item);
|
|
3138
|
+
const picks = pick ? {
|
|
3139
|
+
label: where,
|
|
3140
|
+
key,
|
|
3141
|
+
item,
|
|
3142
|
+
where: pick
|
|
3143
|
+
} : void 0;
|
|
3144
|
+
return prepare(mode, caseEvalName(name, keys[i]), own, run, source, "", family, picks);
|
|
2943
3145
|
});
|
|
2944
|
-
|
|
3146
|
+
if (pick) applyWhere(where, shownKeys, prepared);
|
|
3147
|
+
for (const p of prepared) p.commit();
|
|
3148
|
+
}
|
|
3149
|
+
/**
|
|
3150
|
+
* The checks `where` needs across an eval's cases, then the pairs it left out: each case's own on
|
|
3151
|
+
* its plan, and every case's in the eval's notes. A cell that no case runs on is refused, and so
|
|
3152
|
+
* is an eval it leaves nothing to run.
|
|
3153
|
+
*/
|
|
3154
|
+
function applyWhere(where, keys, prepared) {
|
|
3155
|
+
const picks = prepared.flatMap((p) => p.planned.picked ? [p.planned.picked] : []);
|
|
3156
|
+
if (!picks.length) return;
|
|
3157
|
+
const excluded = picks.flatMap((p) => p.excluded);
|
|
3158
|
+
const kept = new Set(picks.flatMap((p) => p.keptCells));
|
|
3159
|
+
if (!kept.size) throw new UsageError(`${where}'s where excludes all ${excluded.length} case-and-environment pairs, so it has nothing to run.`);
|
|
3160
|
+
const empty = picks.flatMap((p) => p.cells).find((c) => !kept.has(c.key));
|
|
3161
|
+
if (empty) throw new UsageError(`${where}'s where leaves cell ${empty.key} with no cases. Leave ${empty.environment} out of environments, or widen where.`);
|
|
3162
|
+
if (!excluded.length) return;
|
|
3163
|
+
const byEnv = /* @__PURE__ */ new Map();
|
|
3164
|
+
for (const e of excluded) {
|
|
3165
|
+
const on = e.harness ? `${e.environment} for ${e.harness}` : e.environment;
|
|
3166
|
+
const list = byEnv.get(on) ?? [];
|
|
3167
|
+
if (!list.includes(e.case)) list.push(e.case);
|
|
3168
|
+
byEnv.set(on, list);
|
|
3169
|
+
}
|
|
3170
|
+
const order = (k) => keys.indexOf(k);
|
|
3171
|
+
const notes = [`where excludes ${excluded.length} ${excluded.length === 1 ? "pair" : "pairs"}`, ...[...byEnv].map(([on, list]) => `excluded on ${on}: ${list.sort((a, b) => order(a) - order(b)).join(", ")}`)];
|
|
3172
|
+
const plans = /* @__PURE__ */ new Set();
|
|
3173
|
+
for (const p of prepared) {
|
|
3174
|
+
const own = p.planned.picked?.excluded ?? [];
|
|
3175
|
+
for (const t of p.planned.tests ?? []) {
|
|
3176
|
+
plans.add(t.attempt.plan);
|
|
3177
|
+
if (own.length) t.attempt.plan.excluded = own;
|
|
3178
|
+
if (t.attempt.family) t.attempt.plan.familyExcluded = excluded;
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
for (const list of new Set([...plans].map((p) => p.notes))) list.push(...notes);
|
|
3182
|
+
}
|
|
2945
3183
|
const evaluate = Object.assign(defineEval("run"), {
|
|
2946
3184
|
skip: defineEval("skip"),
|
|
2947
3185
|
only: defineEval("only"),
|
|
2948
|
-
todo: registerTodo
|
|
2949
|
-
each: evalEach
|
|
3186
|
+
todo: registerTodo
|
|
2950
3187
|
});
|
|
2951
3188
|
/**
|
|
2952
3189
|
* Declares a tree's groups and evals as the calls that made it would, in the file being
|
|
@@ -2978,32 +3215,30 @@ function declareNodes(nodes, bodyFor, base) {
|
|
|
2978
3215
|
*/
|
|
2979
3216
|
function optionsFrom(written, where, base) {
|
|
2980
3217
|
refuseIllegalCompare(written, where);
|
|
2981
|
-
const {
|
|
3218
|
+
const { environments, compare, ...rest } = withPaths(written, where, base);
|
|
2982
3219
|
const missing = missingIn(where);
|
|
2983
3220
|
const asEnv = (env, field) => environmentFrom(environmentPaths(env, base, field, missing));
|
|
2984
|
-
const
|
|
3221
|
+
const asEnvs = (value, field) => {
|
|
3222
|
+
if (Array.isArray(value)) return value.map((e, i) => asEnv(e, `${field}[${i}]`));
|
|
3223
|
+
if ("only" in value) return value;
|
|
3224
|
+
if ("add" in value) return { add: [value.add].flat().map((e, i) => asEnv(e, `${field}.add[${i}]`)) };
|
|
3225
|
+
return asEnv(value, field);
|
|
3226
|
+
};
|
|
3227
|
+
const own = environments !== void 0 ? { environments: asEnvs(environments, "environments") } : {};
|
|
2985
3228
|
if (compare === void 0) return {
|
|
2986
3229
|
...rest,
|
|
2987
3230
|
...own
|
|
2988
3231
|
};
|
|
2989
3232
|
const variants = compare.variants.map((v, i) => asEnv(v, `compare.variants[${i}]`));
|
|
2990
|
-
if (compare.baseline !== void 0) return {
|
|
2991
|
-
...rest,
|
|
2992
|
-
compare: {
|
|
2993
|
-
...compare,
|
|
2994
|
-
baseline: asEnv(compare.baseline, "compare.baseline"),
|
|
2995
|
-
variants
|
|
2996
|
-
}
|
|
2997
|
-
};
|
|
2998
3233
|
return {
|
|
2999
3234
|
...rest,
|
|
3000
|
-
...own,
|
|
3001
3235
|
compare: {
|
|
3002
3236
|
...compare,
|
|
3237
|
+
control: asEnv(compare.control, "compare.control"),
|
|
3003
3238
|
variants
|
|
3004
3239
|
}
|
|
3005
3240
|
};
|
|
3006
3241
|
}
|
|
3007
3242
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
3008
3243
|
|
|
3009
|
-
//# sourceMappingURL=evaluate-
|
|
3244
|
+
//# sourceMappingURL=evaluate-CwKeN_fo.js.map
|