reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,38 +1,42 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
|
-
import { r as withLease } from "./client-CF-DOjOU.js";
|
|
3
1
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import { t as
|
|
23
|
-
import {
|
|
24
|
-
import { n as
|
|
25
|
-
import { t as
|
|
26
|
-
import {
|
|
2
|
+
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-Cfi45Bvy.js";
|
|
3
|
+
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
4
|
+
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
5
|
+
import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
|
|
6
|
+
import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-C1oY3qyC.js";
|
|
7
|
+
import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-DB8o3hGi.js";
|
|
8
|
+
import { t as evalSlug } from "./slug-IHh3f770.js";
|
|
9
|
+
import { n as attemptKey, o as recordFileName, r as evalId, s as relativeFile, t as attemptId } from "./ids-C_advdJA.js";
|
|
10
|
+
import { $ as runsOf, A as brokeredIn, B as expand, Bt as decideState, Ct as resolveStateOptions, Et as observer, Ft as NO_JUDGE_INPUT, Ht as resolveJudgeArtifacts, It as inputsOn, J as plainHeaders, K as plainHeaderError, L as checkOptions, Lt as judgeProblem, Pt as INPUT_KEYS, Q as refuseIllegalCompare, Rt as artifactRefsOf, St as mergeJudge, Tt as observedName, U as majorityOf, V as harnessId, Vt as filePartsOf, W as matchesTags, _ as fellBackToDirect, at as unbrokered, b as resolveModelGateway, et as sandboxId, gt as unenforceable, ht as isDirect, it as secretVarsIn, j as cascade, nt as sandboxSetBy, ot as unbrokeredError, pt as egressLabel, tt as sandboxProvider, ut as allowedHosts, v as gatewayScope, w as resolveGateway, wt as observedHere, z as evalPlan, zt as badJudgeArtifacts } from "./config-B9mB1kF5.js";
|
|
11
|
+
import { l as vercelCredentialProblem } from "./vercel-CGlSBv63.js";
|
|
12
|
+
import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
|
|
13
|
+
import { _ as redactValue, d as maskedVariables, f as redactJson, g as redactSplitSecrets, h as redactSecrets, o as findSecrets } from "./redact-BhjtnjrU.js";
|
|
14
|
+
import { f as runJudge, h as isApiError, l as judgeModelName } from "./judge-COMXT1Hx.js";
|
|
15
|
+
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
16
|
+
import { ht as OPAQUE_OUTPUT, n as MCP_TIMER, yt as shellQuote } from "./src-Ck2r_EMT.js";
|
|
17
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BkmPxDiY.js";
|
|
18
|
+
import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-C6zhkxp3.js";
|
|
19
|
+
import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
|
|
20
|
+
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-BpW1f1Mx.js";
|
|
21
|
+
import { c as formatBytes } from "./debug-CIXzB4wK.js";
|
|
22
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CAdlwlNu.js";
|
|
23
|
+
import { t as attemptDebugDir } from "./files-DlQshsks.js";
|
|
24
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Cw1z8-sq.js";
|
|
25
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-iBzFiC0j.js";
|
|
26
|
+
import { t as contextTaker } from "./context-Bkv3ndjq.js";
|
|
27
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-Do0ZaYH6.js";
|
|
28
|
+
import { t as PrepareError } from "./provider-BxzZq3Th.js";
|
|
29
|
+
import { i as preparedWorkspace } from "./workspace-uuE51x35.js";
|
|
27
30
|
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
28
|
-
import { n as judgeDiff } from "./patch-
|
|
29
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
31
|
+
import { n as judgeDiff } from "./patch-DH4e_Gkd.js";
|
|
32
|
+
import { t as transcriptForJudge } from "./transcript-CCf07UmU.js";
|
|
30
33
|
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
|
|
31
|
-
import {
|
|
32
|
-
import { i as withDefaultEffort } from "./default-effort-
|
|
34
|
+
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-D3sbAzt-.js";
|
|
35
|
+
import { i as withDefaultEffort } from "./default-effort-DGhQmu95.js";
|
|
36
|
+
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-DKnYkqeK.js";
|
|
37
|
+
import { z } from "zod";
|
|
33
38
|
import { dirname, join } from "node:path";
|
|
34
39
|
import { parse, stringify } from "yaml";
|
|
35
|
-
import { z } from "zod";
|
|
36
40
|
import { randomBytes } from "node:crypto";
|
|
37
41
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
38
42
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
@@ -66,7 +70,7 @@ function assertionRecords(raw) {
|
|
|
66
70
|
const seen = /* @__PURE__ */ new Map();
|
|
67
71
|
const used = /* @__PURE__ */ new Set();
|
|
68
72
|
return raw.map((r) => {
|
|
69
|
-
const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
73
|
+
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
70
74
|
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
71
75
|
let n = seen.get(base) ?? 0;
|
|
72
76
|
let label;
|
|
@@ -112,11 +116,11 @@ function tracePlan(debug, spec, ctx) {
|
|
|
112
116
|
const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
|
|
113
117
|
const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
|
|
114
118
|
debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
|
|
115
|
-
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("
|
|
119
|
+
debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harnesses")}`);
|
|
116
120
|
debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
|
|
117
121
|
const env = spec.environment;
|
|
118
122
|
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
119
|
-
const envFrom = spec.cell.role === "variant" ? from("compare") : from("
|
|
123
|
+
const envFrom = spec.cell.role === "variant" || spec.sources.compare ? from("compare") : from("environments", "none set");
|
|
120
124
|
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
121
125
|
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
122
126
|
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
@@ -254,11 +258,12 @@ const createFakeHarness = (descriptor) => {
|
|
|
254
258
|
text,
|
|
255
259
|
status,
|
|
256
260
|
startedAt: new Date(startedAt).toISOString(),
|
|
257
|
-
durationMs: Date.now() - startedAt,
|
|
261
|
+
durationMs: descriptor.turnMs ?? Date.now() - startedAt,
|
|
258
262
|
transcript,
|
|
259
|
-
usage: null,
|
|
263
|
+
usage: descriptor.usage ?? null,
|
|
260
264
|
model: descriptor.servedModel ?? null,
|
|
261
|
-
unreported: []
|
|
265
|
+
unreported: [],
|
|
266
|
+
...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
|
|
262
267
|
});
|
|
263
268
|
const write = async (path, content) => {
|
|
264
269
|
const time = at();
|
|
@@ -386,7 +391,7 @@ async function brokerKey(descriptor, sandbox) {
|
|
|
386
391
|
const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
|
|
387
392
|
const key = process.env[credential];
|
|
388
393
|
if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
|
|
389
|
-
const network =
|
|
394
|
+
const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
|
|
390
395
|
if (!network?.setRequestTransformations) throw new InfraError("fakeHarness({ brokers: true }) needs docker(): this sandbox can't broker");
|
|
391
396
|
const placeholder = `reeval_fake_${randomBytes(24).toString("base64url")}`;
|
|
392
397
|
await network.setRequestTransformations([...new Set(endpoints.map((e) => e.host))].map((host) => ({
|
|
@@ -478,18 +483,19 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
478
483
|
};
|
|
479
484
|
}
|
|
480
485
|
// src/harness/index.ts
|
|
481
|
-
|
|
486
|
+
/** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
|
|
487
|
+
async function createHarness(descriptor, sandbox, vertex) {
|
|
482
488
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
483
|
-
const { createAgentHarness } = await import("./agent-
|
|
484
|
-
return createAgentHarness(descriptor, vertex);
|
|
489
|
+
const { createAgentHarness } = await import("./agent-CBoLndM7.js");
|
|
490
|
+
return createAgentHarness(descriptor, sandbox, vertex);
|
|
485
491
|
}
|
|
486
492
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
487
493
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
488
494
|
// src/attempt/environment.ts
|
|
489
495
|
/**
|
|
490
496
|
* Resolves the attempt's environment (files, skills, secrets and their placeholders)
|
|
491
|
-
* and finds the image
|
|
492
|
-
* signal, before the eval timer starts.
|
|
497
|
+
* and finds the image its sandbox starts from, when its kind has one, which preflight prepared.
|
|
498
|
+
* Runs on the test's signal, before the eval timer starts.
|
|
493
499
|
*/
|
|
494
500
|
async function prepareEnvironment(a, root, runId, where, signal) {
|
|
495
501
|
const { spec, record } = a;
|
|
@@ -508,13 +514,14 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
508
514
|
const debug = attemptDebug();
|
|
509
515
|
if (debug) traceEnvironment(debug, spec, env);
|
|
510
516
|
record.image = null;
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
const {
|
|
517
|
+
const kind = sandboxKind(spec.sandbox);
|
|
518
|
+
if (kind.capabilities.images) {
|
|
519
|
+
const { preparedFor } = await import("./preflight-R-InZCQb.js");
|
|
520
|
+
const { attemptImage, createProviders, preparedImage } = await import("./providers-CVu8e0xS.js");
|
|
514
521
|
signal.throwIfAborted();
|
|
515
|
-
const provider =
|
|
522
|
+
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
516
523
|
const builds = builtPlugins(spec.environment.layers);
|
|
517
|
-
const prepared = preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
|
|
524
|
+
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
|
|
518
525
|
if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
|
|
519
526
|
record.image = attemptImage(prepared);
|
|
520
527
|
a.image = record.image.ref;
|
|
@@ -534,7 +541,16 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
534
541
|
}
|
|
535
542
|
const { ref, id, base, install, agentCli } = record.image;
|
|
536
543
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
537
|
-
} else
|
|
544
|
+
} else if (kind.capabilities.isolated) {
|
|
545
|
+
const { preparedFor } = await import("./preflight-R-InZCQb.js");
|
|
546
|
+
const { createProviders, vmImage } = await import("./providers-CVu8e0xS.js");
|
|
547
|
+
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
548
|
+
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
|
|
549
|
+
const vm = prepared ? vmImage(prepared) : null;
|
|
550
|
+
if (vm) record.vmImage = vm;
|
|
551
|
+
const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
|
|
552
|
+
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
|
|
553
|
+
} else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
538
554
|
}
|
|
539
555
|
// src/judge/size.ts
|
|
540
556
|
/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
|
|
@@ -609,6 +625,7 @@ function stateOf(d) {
|
|
|
609
625
|
...d.diff !== void 0 ? { diff: d.diff } : {},
|
|
610
626
|
...d.contents ? { contents: d.contents } : {},
|
|
611
627
|
...d.checks ? { checks: d.checks } : {},
|
|
628
|
+
...d.observations ? { observations: d.observations } : {},
|
|
612
629
|
...transcript !== void 0 ? { transcript } : {}
|
|
613
630
|
};
|
|
614
631
|
}
|
|
@@ -643,6 +660,7 @@ async function draftState(src) {
|
|
|
643
660
|
...contents ? { inFull: new Set(Object.keys(contents)) } : {}
|
|
644
661
|
});
|
|
645
662
|
const checks = checksShown(wanted.checks, src.checks);
|
|
663
|
+
const observations = observationsShown(wanted.observations, src.observations);
|
|
646
664
|
return {
|
|
647
665
|
draft: {
|
|
648
666
|
all: wanted.turns === "all",
|
|
@@ -650,7 +668,8 @@ async function draftState(src) {
|
|
|
650
668
|
...wanted.files ? { files: shown.files } : {},
|
|
651
669
|
...wanted.diff !== false ? { diff: shown.patch } : {},
|
|
652
670
|
...contents ? { contents } : {},
|
|
653
|
-
...checks ? { checks } : {}
|
|
671
|
+
...checks ? { checks } : {},
|
|
672
|
+
...observations ? { observations } : {}
|
|
654
673
|
},
|
|
655
674
|
dropped: wanted.diff !== false ? shown.dropped : []
|
|
656
675
|
};
|
|
@@ -682,6 +701,10 @@ function checksShown(which, ran) {
|
|
|
682
701
|
stderr
|
|
683
702
|
}));
|
|
684
703
|
}
|
|
704
|
+
function observationsShown(which, observed = []) {
|
|
705
|
+
if (!which) return void 0;
|
|
706
|
+
return Object.fromEntries(observed.filter((o) => which === true || which.includes(o.name)).map((o) => [o.name, o.value]));
|
|
707
|
+
}
|
|
685
708
|
/**
|
|
686
709
|
* Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
|
|
687
710
|
* what they record stays in the order they were made.
|
|
@@ -706,7 +729,7 @@ function judgeCaller(a, readFile) {
|
|
|
706
729
|
if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
|
|
707
730
|
const wanted = resolveStateOptions(call);
|
|
708
731
|
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
|
|
709
|
-
const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
|
|
732
|
+
const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, observations: _observations, artifacts, ...callJudge } = call;
|
|
710
733
|
const options = mergeJudge(a.spec.judge, callJudge);
|
|
711
734
|
const { model } = options;
|
|
712
735
|
if (model === void 0) {
|
|
@@ -716,12 +739,15 @@ function judgeCaller(a, readFile) {
|
|
|
716
739
|
}
|
|
717
740
|
const { maxConcurrent, ...judgeOptions } = options;
|
|
718
741
|
const output = on && "value" in on ? on.value : subject.text;
|
|
742
|
+
const unobserved = Array.isArray(call.observations) ? call.observations.find((n) => !observedHere(a, n)) : void 0;
|
|
743
|
+
if (unobserved !== void 0) throw new Error(`c.judge() observations names a value this attempt hasn't observed: ${JSON.stringify(observedName(unobserved))}`);
|
|
719
744
|
const { draft, dropped } = await draftState({
|
|
720
745
|
turns: a.turns,
|
|
721
746
|
subject,
|
|
722
747
|
output,
|
|
723
748
|
wanted,
|
|
724
749
|
checks: a.record.checks,
|
|
750
|
+
observations: a.record.observations,
|
|
725
751
|
readFile,
|
|
726
752
|
listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
|
|
727
753
|
});
|
|
@@ -844,18 +870,12 @@ function judgeCaller(a, readFile) {
|
|
|
844
870
|
}
|
|
845
871
|
// src/sandbox/index.ts
|
|
846
872
|
async function createSandbox(descriptor, options) {
|
|
873
|
+
const kind = sandboxKind(descriptor);
|
|
847
874
|
try {
|
|
848
|
-
|
|
849
|
-
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
850
|
-
case "docker": {
|
|
851
|
-
const { createDockerSandbox } = await import("./docker-CIzHzwlC.js");
|
|
852
|
-
return await createDockerSandbox(descriptor, options);
|
|
853
|
-
}
|
|
854
|
-
}
|
|
875
|
+
return await kind.create(descriptor, options);
|
|
855
876
|
} catch (error) {
|
|
856
877
|
if (error instanceof InfraError || options.signal.aborted) throw error;
|
|
857
|
-
|
|
858
|
-
throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
|
|
878
|
+
throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
|
|
859
879
|
}
|
|
860
880
|
}
|
|
861
881
|
// src/attempt/record.ts
|
|
@@ -1276,6 +1296,9 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1276
1296
|
ended = true;
|
|
1277
1297
|
await starting?.catch(() => {});
|
|
1278
1298
|
},
|
|
1299
|
+
get ended() {
|
|
1300
|
+
return ended;
|
|
1301
|
+
},
|
|
1279
1302
|
report: (update) => progress({
|
|
1280
1303
|
at: Date.now(),
|
|
1281
1304
|
...update
|
|
@@ -1310,7 +1333,7 @@ const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s")
|
|
|
1310
1333
|
function setupAllowance(planned) {
|
|
1311
1334
|
const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
|
|
1312
1335
|
const { sandbox } = planned;
|
|
1313
|
-
return setupMs + (sandbox.
|
|
1336
|
+
return setupMs + (sandboxKind(sandbox).buildsAtStart(sandbox) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
|
|
1314
1337
|
}
|
|
1315
1338
|
/** The eval's timer, paused while the sandbox and its setup steps start. */
|
|
1316
1339
|
function evalTimer(ms) {
|
|
@@ -1392,7 +1415,7 @@ const SETUP_COMMAND_CHARS = 200;
|
|
|
1392
1415
|
*/
|
|
1393
1416
|
async function leftRunning(a, box, env) {
|
|
1394
1417
|
const names = unbrokeredChecks(env);
|
|
1395
|
-
if (a.spec.sandbox.
|
|
1418
|
+
if (!capabilitiesOf(a.spec.sandbox).processTable || !env.setup.length || !names.length) return [];
|
|
1396
1419
|
const r = await box.exec([
|
|
1397
1420
|
"for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
|
|
1398
1421
|
"for p in /proc/[0-9]*; do",
|
|
@@ -1628,7 +1651,7 @@ async function startSession(a, runId) {
|
|
|
1628
1651
|
})
|
|
1629
1652
|
});
|
|
1630
1653
|
a.sandbox = box;
|
|
1631
|
-
if (spec.sandbox.
|
|
1654
|
+
if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
|
|
1632
1655
|
a.phases = brokerPhases(box, env.substitutions);
|
|
1633
1656
|
if (a.signal.aborted) box.destroy();
|
|
1634
1657
|
a.signal.throwIfAborted();
|
|
@@ -1674,7 +1697,7 @@ async function startSession(a, runId) {
|
|
|
1674
1697
|
debugLog("env", "stdio MCP servers start through reeval's MCP timer");
|
|
1675
1698
|
} else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
|
|
1676
1699
|
if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
|
|
1677
|
-
const harness = await createHarness(spec.harness, spec.vertex);
|
|
1700
|
+
const harness = await createHarness(spec.harness, spec.sandbox, spec.vertex);
|
|
1678
1701
|
const harnessStart = performance.now();
|
|
1679
1702
|
debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
|
|
1680
1703
|
const session = await startHarness(a, harness, box, {
|
|
@@ -2016,6 +2039,7 @@ function watchMcpStartup(log) {
|
|
|
2016
2039
|
function turnSender(a, takeContext, checks) {
|
|
2017
2040
|
return async (prompt) => {
|
|
2018
2041
|
const [box, agent] = await a.start("c.send()");
|
|
2042
|
+
const before = await a.timed("diffMs", () => workspaceBlobs(box, a.signal));
|
|
2019
2043
|
const entry = {
|
|
2020
2044
|
prompt,
|
|
2021
2045
|
text: "",
|
|
@@ -2056,7 +2080,8 @@ function turnSender(a, takeContext, checks) {
|
|
|
2056
2080
|
durationMs: Date.now() - sendStarted
|
|
2057
2081
|
};
|
|
2058
2082
|
});
|
|
2059
|
-
const
|
|
2083
|
+
const timedBefore = a.record.timings.agentMs;
|
|
2084
|
+
const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2060
2085
|
entry.reasoningMs = null;
|
|
2061
2086
|
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
2062
2087
|
throw error;
|
|
@@ -2064,7 +2089,8 @@ function turnSender(a, takeContext, checks) {
|
|
|
2064
2089
|
cancel();
|
|
2065
2090
|
watch?.close();
|
|
2066
2091
|
});
|
|
2067
|
-
const sendWallMs = Date.now() - sendStarted;
|
|
2092
|
+
const sendWallMs = agentMs ?? Date.now() - sendStarted;
|
|
2093
|
+
if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
|
|
2068
2094
|
const startup = await watch?.end();
|
|
2069
2095
|
const timedOut = startup?.timedOut && mcpTimeoutMessage(startup.timedOut);
|
|
2070
2096
|
const seenAt = startup?.at ?? Date.now();
|
|
@@ -2089,6 +2115,16 @@ function turnSender(a, takeContext, checks) {
|
|
|
2089
2115
|
await takeContext(box, a.signal);
|
|
2090
2116
|
const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
|
|
2091
2117
|
const failure = result.transcript.find((e) => e.type === "error");
|
|
2118
|
+
const infra = timedOut || result.status !== "aborted" && rejectedKey;
|
|
2119
|
+
let changes;
|
|
2120
|
+
let diffError;
|
|
2121
|
+
if (!infra && !a.signal.aborted) try {
|
|
2122
|
+
changes = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
|
|
2123
|
+
} catch (error) {
|
|
2124
|
+
diffError = error;
|
|
2125
|
+
}
|
|
2126
|
+
const edited = changes ? changedBetween(before, changes.blobs) : void 0;
|
|
2127
|
+
const filesChanged = edited?.length;
|
|
2092
2128
|
a.report({
|
|
2093
2129
|
kind: "turn-end",
|
|
2094
2130
|
turn: turnNumber,
|
|
@@ -2096,18 +2132,19 @@ function turnSender(a, takeContext, checks) {
|
|
|
2096
2132
|
ms: result.durationMs,
|
|
2097
2133
|
tools: count("tool", "web-search", "web-fetch"),
|
|
2098
2134
|
fileEdits: count("file-edit"),
|
|
2135
|
+
...filesChanged === void 0 ? {} : { filesChanged },
|
|
2099
2136
|
...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
|
|
2100
2137
|
});
|
|
2101
|
-
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "
|
|
2138
|
+
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${filesChanged === void 0 ? "" : `${plural(filesChanged, "file")} changed · `}${plural(count("file-edit"), "edit tool call")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
2102
2139
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
2103
|
-
const infra = timedOut || result.status !== "aborted" && rejectedKey;
|
|
2104
2140
|
if (infra) {
|
|
2105
2141
|
const error = new InfraError(infra);
|
|
2106
2142
|
a.startFailure ??= { error };
|
|
2107
2143
|
throw error;
|
|
2108
2144
|
}
|
|
2109
2145
|
a.signal.throwIfAborted();
|
|
2110
|
-
|
|
2146
|
+
if (diffError !== void 0 || !changes || !edited) throw diffError;
|
|
2147
|
+
const { diff } = changes;
|
|
2111
2148
|
a.record.diff = diff;
|
|
2112
2149
|
if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
|
|
2113
2150
|
const data = {
|
|
@@ -2115,10 +2152,10 @@ function turnSender(a, takeContext, checks) {
|
|
|
2115
2152
|
...result,
|
|
2116
2153
|
diff
|
|
2117
2154
|
};
|
|
2118
|
-
const turn = Object.freeze({
|
|
2155
|
+
const turn = Object.freeze(Object.assign({
|
|
2119
2156
|
...data,
|
|
2120
2157
|
...checks
|
|
2121
|
-
});
|
|
2158
|
+
}, { [TURN_EDITED]: edited }));
|
|
2122
2159
|
a.turns.push(turn);
|
|
2123
2160
|
return turn;
|
|
2124
2161
|
};
|
|
@@ -2165,6 +2202,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2165
2202
|
exec,
|
|
2166
2203
|
readFile,
|
|
2167
2204
|
writeFile,
|
|
2205
|
+
observe: observer(a),
|
|
2168
2206
|
serve: (command, port) => serveCommand(a, command, port),
|
|
2169
2207
|
capture: (target, options) => capturePage(a, target, options),
|
|
2170
2208
|
attempt: {
|
|
@@ -2265,6 +2303,37 @@ function bestOfCell(bestOf) {
|
|
|
2265
2303
|
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2266
2304
|
});
|
|
2267
2305
|
}
|
|
2306
|
+
/** Characters a case key can't hold: ids and paths split on them, and control characters. */
|
|
2307
|
+
const BAD_KEY = /[#/›\u0000-\u001f\u007f]/;
|
|
2308
|
+
/** The name a case's eval plans under: `adds a health check [nextjs]`. */
|
|
2309
|
+
const caseEvalName = (name, key) => `${name} [${key}]`;
|
|
2310
|
+
/** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
|
|
2311
|
+
function defaultKey(item) {
|
|
2312
|
+
if (typeof item === "string") return item;
|
|
2313
|
+
if (typeof item === "number" && Number.isFinite(item)) return String(item);
|
|
2314
|
+
if (isObjectLike(item) && !Array.isArray(item)) {
|
|
2315
|
+
if (typeof item.id === "string") return item.id;
|
|
2316
|
+
if (typeof item.name === "string") return item.name;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
/**
|
|
2320
|
+
* Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
|
|
2321
|
+
* characters, with no `#`, `/`, `›` or control characters, and unique within the eval.
|
|
2322
|
+
*/
|
|
2323
|
+
function caseKeys(cases, caseKey, where) {
|
|
2324
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2325
|
+
return cases.map((item, i) => {
|
|
2326
|
+
const raw = caseKey ? caseKey(item) : defaultKey(item);
|
|
2327
|
+
if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
|
|
2328
|
+
if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
|
|
2329
|
+
const key = raw.trim();
|
|
2330
|
+
if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, /, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
|
|
2331
|
+
const earlier = seen.get(key);
|
|
2332
|
+
if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
|
|
2333
|
+
seen.set(key, i);
|
|
2334
|
+
return key;
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2268
2337
|
// src/contract/declaration.ts
|
|
2269
2338
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
2270
2339
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
@@ -2276,7 +2345,7 @@ const declaredModeSchema = z.enum([
|
|
|
2276
2345
|
const declaredFamilySchema = z.strictObject({
|
|
2277
2346
|
name: z.string(),
|
|
2278
2347
|
case: z.number().int().min(0),
|
|
2279
|
-
cases: z.number().int().min(
|
|
2348
|
+
cases: z.number().int().min(1)
|
|
2280
2349
|
});
|
|
2281
2350
|
const declaredTodoSchema = z.strictObject({
|
|
2282
2351
|
type: z.literal("todo"),
|
|
@@ -2285,7 +2354,7 @@ const declaredTodoSchema = z.strictObject({
|
|
|
2285
2354
|
const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
|
|
2286
2355
|
const declaredSourceSchema = z.strictObject({
|
|
2287
2356
|
body: hash(),
|
|
2288
|
-
|
|
2357
|
+
caseJson: z.string().optional()
|
|
2289
2358
|
});
|
|
2290
2359
|
const declaredEvalSchema = z.strictObject({
|
|
2291
2360
|
type: z.literal("eval"),
|
|
@@ -2321,6 +2390,89 @@ const NODE_SCHEMAS = {
|
|
|
2321
2390
|
* tree's `children`. Options are left to the cascade.
|
|
2322
2391
|
*/
|
|
2323
2392
|
const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
|
|
2393
|
+
// src/where.ts
|
|
2394
|
+
/** Keys a trait lookup can hit without naming a trait: JSON, promises and inspection. */
|
|
2395
|
+
const PASSTHROUGH = /* @__PURE__ */ new Set([
|
|
2396
|
+
"toJSON",
|
|
2397
|
+
"then",
|
|
2398
|
+
"constructor"
|
|
2399
|
+
]);
|
|
2400
|
+
function pickPairs(w, attempts) {
|
|
2401
|
+
const controls = attempts.filter((a) => a.cell.role === "control");
|
|
2402
|
+
const known = [...new Set(controls.flatMap((a) => Object.keys(traitsOf(a.environment.layers))))];
|
|
2403
|
+
const decisions = /* @__PURE__ */ new Map();
|
|
2404
|
+
const decide = (control) => {
|
|
2405
|
+
const id = control.cell.key;
|
|
2406
|
+
const made = decisions.get(id);
|
|
2407
|
+
if (made) return made;
|
|
2408
|
+
const env = control.environment;
|
|
2409
|
+
const traits = traitsOf(env.layers);
|
|
2410
|
+
let readHarness = false;
|
|
2411
|
+
const view = new Proxy(traits, { get(target, key, receiver) {
|
|
2412
|
+
if (typeof key === "string" && !(key in target) && !PASSTHROUGH.has(key)) {
|
|
2413
|
+
if (!known.includes(key)) {
|
|
2414
|
+
const near = closest(key, known);
|
|
2415
|
+
throw new UsageError(`${w.label}'s where reads environment.traits.${key}, which no environment defines (${known.length ? `they define: ${known.join(", ")}` : "none defines any"}).${near ? ` Did you mean "${near}"?` : ""}`);
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
return Reflect.get(target, key, receiver);
|
|
2419
|
+
} });
|
|
2420
|
+
const pair = {
|
|
2421
|
+
case: w.item,
|
|
2422
|
+
environment: Object.freeze({
|
|
2423
|
+
name: env.name,
|
|
2424
|
+
traits: view
|
|
2425
|
+
}),
|
|
2426
|
+
get harness() {
|
|
2427
|
+
readHarness = true;
|
|
2428
|
+
return control.cell.harness;
|
|
2429
|
+
}
|
|
2430
|
+
};
|
|
2431
|
+
let result;
|
|
2432
|
+
try {
|
|
2433
|
+
result = w.where(pair);
|
|
2434
|
+
} catch (error) {
|
|
2435
|
+
if (error instanceof UsageError) throw error;
|
|
2436
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
2437
|
+
throw new UsageError(`${w.label}'s where threw for case ${w.key} on ${env.name}: ${why}`);
|
|
2438
|
+
}
|
|
2439
|
+
if (typeof result !== "boolean") throw new UsageError(`${w.label}'s where returned ${JSON.stringify(result) ?? String(result)} for case ${w.key} on ${env.name}; it must return true or false.`);
|
|
2440
|
+
const decided = {
|
|
2441
|
+
keep: result,
|
|
2442
|
+
readHarness
|
|
2443
|
+
};
|
|
2444
|
+
decisions.set(id, decided);
|
|
2445
|
+
return decided;
|
|
2446
|
+
};
|
|
2447
|
+
const controlOf = (a) => a.cell.role === "control" ? a : controls.find((c) => c.cell.harness === a.cell.harness);
|
|
2448
|
+
const cells = [];
|
|
2449
|
+
const keptCells = [];
|
|
2450
|
+
const excluded = [];
|
|
2451
|
+
return {
|
|
2452
|
+
kept: attempts.filter((a) => {
|
|
2453
|
+
const { keep, readHarness } = decide(controlOf(a));
|
|
2454
|
+
const first = !cells.some((c) => c.key === a.cell.key);
|
|
2455
|
+
if (first) cells.push({
|
|
2456
|
+
key: a.cell.key,
|
|
2457
|
+
environment: a.cell.environment
|
|
2458
|
+
});
|
|
2459
|
+
if (keep) {
|
|
2460
|
+
if (first) keptCells.push(a.cell.key);
|
|
2461
|
+
return true;
|
|
2462
|
+
}
|
|
2463
|
+
const pair = {
|
|
2464
|
+
case: w.key,
|
|
2465
|
+
environment: a.cell.environment,
|
|
2466
|
+
...readHarness ? { harness: a.cell.harness } : {}
|
|
2467
|
+
};
|
|
2468
|
+
if (!excluded.some((e) => e.environment === pair.environment && e.harness === pair.harness)) excluded.push(pair);
|
|
2469
|
+
return false;
|
|
2470
|
+
}),
|
|
2471
|
+
excluded,
|
|
2472
|
+
cells,
|
|
2473
|
+
keptCells
|
|
2474
|
+
};
|
|
2475
|
+
}
|
|
2324
2476
|
// src/plan-eval.ts
|
|
2325
2477
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
2326
2478
|
const GRACE_MS = 5e3;
|
|
@@ -2329,7 +2481,7 @@ function configLevel(config) {
|
|
|
2329
2481
|
return {
|
|
2330
2482
|
harness: config.harness,
|
|
2331
2483
|
sandbox: config.sandbox,
|
|
2332
|
-
|
|
2484
|
+
environments: config.environments,
|
|
2333
2485
|
compare: void 0,
|
|
2334
2486
|
runs,
|
|
2335
2487
|
bestOf,
|
|
@@ -2364,26 +2516,83 @@ function planEval(declaration, ctx) {
|
|
|
2364
2516
|
profile: ctx.profile,
|
|
2365
2517
|
runs: ctx.runs,
|
|
2366
2518
|
skipped: declaration.mode === "skip" || declaration.mode === "todo",
|
|
2367
|
-
pooled: declaration.family
|
|
2519
|
+
pooled: (declaration.family?.cases ?? 0) >= 2
|
|
2368
2520
|
});
|
|
2369
2521
|
const { harnesses } = resolvedEval;
|
|
2370
2522
|
if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
|
|
2371
2523
|
tests: null,
|
|
2372
|
-
harnesses
|
|
2524
|
+
harnesses,
|
|
2525
|
+
traits: []
|
|
2526
|
+
};
|
|
2527
|
+
const { resolved } = resolvedEval;
|
|
2528
|
+
const declared = expand({
|
|
2529
|
+
...resolved,
|
|
2530
|
+
harness: resolved.harness.map((h) => withDefaultEffort(h, ctx.cursorEfforts))
|
|
2531
|
+
});
|
|
2532
|
+
const traits = controlTraitKeys(declared);
|
|
2533
|
+
const filters = traitFilters(ctx.traits, ctx.profile);
|
|
2534
|
+
const traitSkip = filters.length > 0 && !resolvedEval.skip && !pickTraits(declared, filters).kept.length;
|
|
2535
|
+
const narrowed = traitSkip ? {
|
|
2536
|
+
...resolvedEval,
|
|
2537
|
+
skip: noCellLeft(declared, filters)
|
|
2538
|
+
} : resolvedEval;
|
|
2539
|
+
const picks = [];
|
|
2540
|
+
if (traitSkip && declaration.where) picks.push(pickPairs(declaration.where, declared));
|
|
2541
|
+
let traitExcluded = [];
|
|
2542
|
+
const pick = (attempts) => {
|
|
2543
|
+
let kept = attempts;
|
|
2544
|
+
if (declaration.where) {
|
|
2545
|
+
const picked = pickPairs(declaration.where, attempts);
|
|
2546
|
+
picks.push(picked);
|
|
2547
|
+
kept = picked.kept;
|
|
2548
|
+
}
|
|
2549
|
+
if (!filters.length) return kept;
|
|
2550
|
+
const byTraits = pickTraits(kept, filters);
|
|
2551
|
+
traitExcluded = byTraits.excluded;
|
|
2552
|
+
return byTraits.kept;
|
|
2553
|
+
};
|
|
2554
|
+
const leftOut = () => `${filtersLabel(filters)} ${filters.length === 1 ? "leaves" : "leave"} out ${traitExcluded.join(", ")}`;
|
|
2555
|
+
let tests = planTests(declaration, ctx, where, narrowed, pick);
|
|
2556
|
+
const allLeftOut = !tests.length && traitExcluded.length > 0;
|
|
2557
|
+
if (allLeftOut) {
|
|
2558
|
+
const skip = `no cell left: ${leftOut()}`;
|
|
2559
|
+
tests = planTests(declaration, ctx, where, {
|
|
2560
|
+
...narrowed,
|
|
2561
|
+
skip
|
|
2562
|
+
}, pick);
|
|
2563
|
+
}
|
|
2564
|
+
if (traitExcluded.length) {
|
|
2565
|
+
for (const t of tests) t.attempt.plan.traitExcluded = traitExcluded;
|
|
2566
|
+
const notes = new Set(tests.map((t) => t.attempt.plan.notes));
|
|
2567
|
+
if (!allLeftOut) for (const list of notes) list.push(leftOut());
|
|
2568
|
+
}
|
|
2569
|
+
const [picked] = picks;
|
|
2570
|
+
if (!picked) return {
|
|
2571
|
+
tests,
|
|
2572
|
+
harnesses,
|
|
2573
|
+
traits
|
|
2373
2574
|
};
|
|
2575
|
+
const { excluded, cells, keptCells } = picked;
|
|
2374
2576
|
return {
|
|
2375
|
-
tests
|
|
2376
|
-
harnesses
|
|
2577
|
+
tests,
|
|
2578
|
+
harnesses,
|
|
2579
|
+
traits,
|
|
2580
|
+
picked: {
|
|
2581
|
+
excluded,
|
|
2582
|
+
cells,
|
|
2583
|
+
keptCells
|
|
2584
|
+
}
|
|
2377
2585
|
};
|
|
2378
2586
|
}
|
|
2379
|
-
|
|
2587
|
+
/** Why an eval none of whose control environments the trait filters match doesn't run. */
|
|
2588
|
+
function noCellLeft(attempts, filters) {
|
|
2589
|
+
const envs = [...new Set(attempts.filter((a) => a.cell.role === "control").map((a) => a.cell.environment))];
|
|
2590
|
+
return `no cell left: ${filtersLabel(filters)} ${filters.length === 1 ? "matches" : "match"} none of ${envs.join(", ")}`;
|
|
2591
|
+
}
|
|
2592
|
+
function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
2380
2593
|
const { name, mode, groups: enclosing } = declaration;
|
|
2381
2594
|
const groups = enclosing.map((g) => g.name);
|
|
2382
2595
|
const { resolved } = resolvedEval;
|
|
2383
|
-
const source = {
|
|
2384
|
-
...declaration.source,
|
|
2385
|
-
options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
|
|
2386
|
-
};
|
|
2387
2596
|
const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
|
|
2388
2597
|
const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
|
|
2389
2598
|
const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
|
|
@@ -2416,10 +2625,16 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2416
2625
|
gateway: route.name
|
|
2417
2626
|
};
|
|
2418
2627
|
}).map((harness) => withDefaultEffort(harness, ctx.cursorEfforts));
|
|
2419
|
-
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
2628
|
+
const plannedAttempts = resolvedEval.skip ? [] : pick(expand({
|
|
2420
2629
|
...resolved,
|
|
2421
2630
|
harness: routed
|
|
2422
|
-
}, vertex);
|
|
2631
|
+
}, vertex));
|
|
2632
|
+
const { caseJson, ...declared } = declaration.source;
|
|
2633
|
+
const source = {
|
|
2634
|
+
...declared,
|
|
2635
|
+
...caseJson === void 0 ? {} : { case: caseHash(caseJson, process.env, secretValues(resolved.env, resolvedEval.skip ? expand(resolved) : plannedAttempts)) },
|
|
2636
|
+
options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
|
|
2637
|
+
};
|
|
2423
2638
|
let judge;
|
|
2424
2639
|
if (plannedAttempts.length) try {
|
|
2425
2640
|
const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
|
|
@@ -2445,6 +2660,8 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2445
2660
|
const why = unenforceable(planned.egress, planned.sandbox, where);
|
|
2446
2661
|
if (why) throw new UsageError(why);
|
|
2447
2662
|
const on = `${where} on ${harnessId(planned.harness)}`;
|
|
2663
|
+
const unverified = unverifiedHarness(planned.harness, planned.sandbox);
|
|
2664
|
+
if (unverified) throw new UsageError(`${on}: ${unverified}`);
|
|
2448
2665
|
const mcp = refusedMcp(planned.harness, planned.environment.layers);
|
|
2449
2666
|
if (mcp) throw new UsageError(`${on}: ${mcp}`);
|
|
2450
2667
|
const plugins = planPlugins(planned, ctx.config.root);
|
|
@@ -2458,7 +2675,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2458
2675
|
id: evalId(rankFile, [...groups, declaration.family.name]),
|
|
2459
2676
|
...declaration.family
|
|
2460
2677
|
};
|
|
2461
|
-
const harnessBy = resolved.sources?.
|
|
2678
|
+
const harnessBy = resolved.sources?.harnesses ?? ctx.config.file;
|
|
2462
2679
|
const plan = evalPlan(resolvedEval, plannedAttempts, {
|
|
2463
2680
|
configFile: ctx.config.file,
|
|
2464
2681
|
file: path
|
|
@@ -2575,6 +2792,13 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2575
2792
|
});
|
|
2576
2793
|
}
|
|
2577
2794
|
/**
|
|
2795
|
+
* The values of the `secret()` variables the attempts read. The record masks them once the run
|
|
2796
|
+
* has planned, and the case hash is taken here, before that, so it masks them itself.
|
|
2797
|
+
*/
|
|
2798
|
+
function secretValues(env, attempts) {
|
|
2799
|
+
return [...new Set(attempts.flatMap((a) => secretVarsIn(env, a)))].flatMap((name) => process.env[name] ?? []);
|
|
2800
|
+
}
|
|
2801
|
+
/**
|
|
2578
2802
|
* What the plan can tell of an attempt's plugins before any is fetched: a build outside Docker, a
|
|
2579
2803
|
* local plugin's content, and the harness's refusal, as a refusal; the commands that fetch a
|
|
2580
2804
|
* package at start, as notes under `proxy` and `allow-all`; and a label for each plugin. Git and npm plugins
|
|
@@ -2633,7 +2857,8 @@ function planPlugins(planned, root) {
|
|
|
2633
2857
|
function attemptLimits(planned, concurrency) {
|
|
2634
2858
|
const limits = { attempts: concurrency };
|
|
2635
2859
|
const { sandbox, harness } = planned;
|
|
2636
|
-
|
|
2860
|
+
const cap = providerLimit(sandbox);
|
|
2861
|
+
if (cap) limits[`sandbox:${sandboxProvider(sandbox)}`] = cap;
|
|
2637
2862
|
if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
|
|
2638
2863
|
return limits;
|
|
2639
2864
|
}
|
|
@@ -2668,7 +2893,7 @@ function withPaths(options, where, base = callerDir() ?? process.cwd()) {
|
|
|
2668
2893
|
}
|
|
2669
2894
|
/** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
|
|
2670
2895
|
function splitArgs(what, name, a, b) {
|
|
2671
|
-
if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (
|
|
2896
|
+
if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environments: …) or in the config's env.`);
|
|
2672
2897
|
if (typeof a === "function") return [b ?? {}, a];
|
|
2673
2898
|
if (typeof b === "function") return [a ?? {}, b];
|
|
2674
2899
|
throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
|
|
@@ -2680,8 +2905,40 @@ function checkOnly(mode, what) {
|
|
|
2680
2905
|
error.name = "OnlyInCiError";
|
|
2681
2906
|
throw error;
|
|
2682
2907
|
}
|
|
2683
|
-
|
|
2908
|
+
/** Each name path as its file wrote it, by kind and the path it's shown as. */
|
|
2909
|
+
const givenPaths = /* @__PURE__ */ new Map();
|
|
2910
|
+
/**
|
|
2911
|
+
* `name` as it's declared, recorded and shown: masked as everything reeval writes is, so the
|
|
2912
|
+
* eval's id, its artifact directory and every view hold no secret. Two names that mask to the
|
|
2913
|
+
* same name are refused rather than numbered: the id is the key history, compare and query
|
|
2914
|
+
* group attempts by across runs, and a number would depend on the order of the declarations.
|
|
2915
|
+
*/
|
|
2916
|
+
function shownName(kind, name, hint = "") {
|
|
2917
|
+
const shown = redactSecrets(name);
|
|
2918
|
+
const path = [...groupStack.map((g) => g.name), shown];
|
|
2919
|
+
const given = [...groupStack.map((g) => g.given), name].join("\0");
|
|
2920
|
+
const key = [
|
|
2921
|
+
collectingFile() ?? "",
|
|
2922
|
+
kind === "group" ? "group" : "eval",
|
|
2923
|
+
...path
|
|
2924
|
+
].join("\0");
|
|
2925
|
+
const earlier = givenPaths.get(key);
|
|
2926
|
+
if (earlier !== void 0 && earlier !== given) throw new UsageError(`${kind === "group" ? "group" : kind === "todo" ? "evaluate.todo" : "evaluate"}("${path.join(" › ")}") names a secret, and its masked name is another ${kind === "group" ? "group" : "eval"}'s in this file: name it without the secret.${hint}`);
|
|
2927
|
+
givenPaths.set(key, given);
|
|
2928
|
+
return shown;
|
|
2929
|
+
}
|
|
2930
|
+
/**
|
|
2931
|
+
* A note naming the variables whose values masked part of an eval's name: unlike a credential
|
|
2932
|
+
* shape, a value is masked only where the variable holds it, so the eval's id differs elsewhere.
|
|
2933
|
+
*/
|
|
2934
|
+
function maskedNote(names) {
|
|
2935
|
+
const vars = [...new Set(names.flatMap((n) => maskedVariables(n)))];
|
|
2936
|
+
if (!vars.length) return void 0;
|
|
2937
|
+
return `name masked: it holds the value of ${vars.join(", ")}, so its id differs where ${vars.length === 1 ? "that variable has" : "those variables have"} another value`;
|
|
2938
|
+
}
|
|
2939
|
+
const defineGroup = (mode) => (given, a, b) => {
|
|
2684
2940
|
checkOnly(mode, "group");
|
|
2941
|
+
const name = shownName("group", given);
|
|
2685
2942
|
const [written, fn] = splitArgs("group", name, a, b);
|
|
2686
2943
|
const options = withPaths(written, `group("${name}")`);
|
|
2687
2944
|
checkOptions(options, `group("${name}")`);
|
|
@@ -2695,6 +2952,7 @@ const defineGroup = (mode) => (name, a, b) => {
|
|
|
2695
2952
|
declare(node);
|
|
2696
2953
|
const stack = [...groupStack, {
|
|
2697
2954
|
name,
|
|
2955
|
+
given,
|
|
2698
2956
|
options,
|
|
2699
2957
|
node
|
|
2700
2958
|
}];
|
|
@@ -2717,7 +2975,8 @@ const group = Object.assign(defineGroup("run"), {
|
|
|
2717
2975
|
* A placeholder, `evaluate.todo` or `group.todo`: one eval entry that never runs, counted as
|
|
2718
2976
|
* skipped and shown as todo. The enclosing groups' tags filter it as they do an eval.
|
|
2719
2977
|
*/
|
|
2720
|
-
function registerTodo(
|
|
2978
|
+
function registerTodo(given) {
|
|
2979
|
+
const name = shownName("todo", given);
|
|
2721
2980
|
declare({
|
|
2722
2981
|
type: "todo",
|
|
2723
2982
|
name
|
|
@@ -2735,7 +2994,7 @@ const families = /* @__PURE__ */ new Map();
|
|
|
2735
2994
|
/** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
|
|
2736
2995
|
const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
|
|
2737
2996
|
/** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
|
|
2738
|
-
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as
|
|
2997
|
+
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as evaluate("${family.join(" › ")}") with cases in this file. Give one of them another name.${separatorHint(eval_, family)}`);
|
|
2739
2998
|
function checkUnique(where, groups, name, hint, inFamily) {
|
|
2740
2999
|
const path = [...groups, name];
|
|
2741
3000
|
const id = evalId(collectingFile() ?? "", path);
|
|
@@ -2766,10 +3025,18 @@ function checkFamily(groups, family) {
|
|
|
2766
3025
|
});
|
|
2767
3026
|
return;
|
|
2768
3027
|
}
|
|
2769
|
-
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate
|
|
3028
|
+
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate("${path.join(" › ")}") with cases is declared twice in this file. Each eval needs its own name, or their cases are pooled into one. Give each its own name.${separatorHint(path, earlier.path)}`);
|
|
2770
3029
|
earlier.seen.add(family.case);
|
|
2771
3030
|
}
|
|
2772
|
-
function register(mode,
|
|
3031
|
+
function register(mode, given, own, fn, source, hint = "", givenFamily) {
|
|
3032
|
+
prepare(mode, given, own, fn, source, hint, givenFamily).commit();
|
|
3033
|
+
}
|
|
3034
|
+
function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
|
|
3035
|
+
const name = shownName("eval", given, givenFamily ? " Give each case a key that holds no secret." : "");
|
|
3036
|
+
const family = givenFamily && {
|
|
3037
|
+
...givenFamily,
|
|
3038
|
+
name: redactSecrets(givenFamily.name)
|
|
3039
|
+
};
|
|
2773
3040
|
const groups = groupStack.map((g) => g.name);
|
|
2774
3041
|
if (family) checkFamily(groups, family);
|
|
2775
3042
|
checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
|
|
@@ -2784,7 +3051,7 @@ function register(mode, name, own, fn, source, hint = "", family) {
|
|
|
2784
3051
|
source,
|
|
2785
3052
|
...family ? { family } : {}
|
|
2786
3053
|
});
|
|
2787
|
-
const
|
|
3054
|
+
const planned = planEval({
|
|
2788
3055
|
name,
|
|
2789
3056
|
mode,
|
|
2790
3057
|
groups: groupStack.map(({ name, options }) => ({
|
|
@@ -2793,82 +3060,130 @@ function register(mode, name, own, fn, source, hint = "", family) {
|
|
|
2793
3060
|
})),
|
|
2794
3061
|
options: own,
|
|
2795
3062
|
source,
|
|
2796
|
-
...family ? { family } : {}
|
|
3063
|
+
...family ? { family } : {},
|
|
3064
|
+
...where ? { where } : {}
|
|
2797
3065
|
}, {
|
|
2798
3066
|
...getRunContext(),
|
|
2799
3067
|
file: collectingFile(),
|
|
2800
3068
|
order: registered
|
|
2801
3069
|
});
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
for (const { name: testName, spec, ...options } of tests) {
|
|
2807
|
-
const { cell, n, bestOf } = options.attempt;
|
|
2808
|
-
let hooks = {};
|
|
2809
|
-
if (bestOf !== void 0) {
|
|
2810
|
-
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
2811
|
-
bestOfCells.set(cell.key, of);
|
|
2812
|
-
hooks = of(n);
|
|
2813
|
-
}
|
|
2814
|
-
registerTest(testName, {
|
|
2815
|
-
...options,
|
|
2816
|
-
...hooks
|
|
2817
|
-
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
3070
|
+
if (planned.tests) {
|
|
3071
|
+
const note = maskedNote([...groupStack.map((g) => g.given), given]);
|
|
3072
|
+
if (note) for (const notes of new Set(planned.tests.map((t) => t.attempt.plan.notes))) notes.push(note);
|
|
3073
|
+
registered++;
|
|
2818
3074
|
}
|
|
3075
|
+
const commit = () => {
|
|
3076
|
+
declarePlanned(planned.harnesses, planned.traits);
|
|
3077
|
+
if (!planned.tests) return;
|
|
3078
|
+
const bestOfCells = /* @__PURE__ */ new Map();
|
|
3079
|
+
for (const { name: testName, spec, ...options } of planned.tests) {
|
|
3080
|
+
const { cell, n, bestOf } = options.attempt;
|
|
3081
|
+
let hooks = {};
|
|
3082
|
+
if (bestOf !== void 0) {
|
|
3083
|
+
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
3084
|
+
bestOfCells.set(cell.key, of);
|
|
3085
|
+
hooks = of(n);
|
|
3086
|
+
}
|
|
3087
|
+
registerTest(testName, {
|
|
3088
|
+
...options,
|
|
3089
|
+
...hooks
|
|
3090
|
+
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
3091
|
+
}
|
|
3092
|
+
};
|
|
3093
|
+
return {
|
|
3094
|
+
planned,
|
|
3095
|
+
commit
|
|
3096
|
+
};
|
|
2819
3097
|
}
|
|
2820
3098
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2821
3099
|
let registered = 0;
|
|
2822
3100
|
const defineEval = (mode, declared) => (name, a, b) => {
|
|
2823
3101
|
checkOnly(mode, "evaluate");
|
|
2824
|
-
const
|
|
2825
|
-
const
|
|
2826
|
-
|
|
3102
|
+
const shown = redactSecrets(name);
|
|
3103
|
+
const [written, fn] = splitArgs("evaluate", shown, a, b);
|
|
3104
|
+
const where = `evaluate("${shown}")`;
|
|
3105
|
+
if (isObjectLike(written) && "where" in written && !("cases" in written)) throw new UsageError(`${where} sets where without cases. where picks case-and-environment pairs; to drop an environment, leave it out of environments.`);
|
|
3106
|
+
if (isObjectLike(written) && ("cases" in written || "caseKey" in written)) return registerCases(mode, name, written, fn, where);
|
|
3107
|
+
register(mode, name, withPaths(written, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
|
|
2827
3108
|
};
|
|
2828
|
-
/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
|
|
2829
|
-
function formatName(template, item, index) {
|
|
2830
|
-
return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => isObjectLike(item) && key in item ? String(item[key]) : whole);
|
|
2831
|
-
}
|
|
2832
|
-
function checkEachOptions(options) {
|
|
2833
|
-
if (options === void 0) return {};
|
|
2834
|
-
if (isObjectLike(options) && Object.keys(options).every((k) => k === "pool")) {
|
|
2835
|
-
const { pool } = options;
|
|
2836
|
-
if (pool === void 0) return {};
|
|
2837
|
-
if (typeof pool === "boolean") return { pool };
|
|
2838
|
-
}
|
|
2839
|
-
throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
|
|
2840
|
-
}
|
|
2841
3109
|
/**
|
|
2842
|
-
*
|
|
2843
|
-
*
|
|
3110
|
+
* An eval with `cases`: each case plans as an entry of its own named for the eval, which the
|
|
3111
|
+
* record folds back into the eval. Two or more cases pool into one pass rate and, with
|
|
3112
|
+
* `compare`, one lift paired by case.
|
|
2844
3113
|
*/
|
|
2845
|
-
|
|
2846
|
-
const {
|
|
2847
|
-
if (!Array.isArray(cases)) throw new UsageError(
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
3114
|
+
function registerCases(mode, name, written, fn, where) {
|
|
3115
|
+
const { cases, caseKey, where: pick, ...rest } = written;
|
|
3116
|
+
if (!Array.isArray(cases)) throw new UsageError(cases === void 0 ? `caseKey in ${where} names each case, but there are no cases. Add cases: [...], or drop caseKey.` : `cases in ${where} must be a list (got ${typeof cases}), e.g. cases: ['users', 'posts'].`);
|
|
3117
|
+
if (cases.length === 0) throw new UsageError(`cases in ${where} is an empty list. Give at least one case.`);
|
|
3118
|
+
if (caseKey !== void 0 && typeof caseKey !== "function") throw new UsageError(`caseKey in ${where} must be a function from a case to its key, e.g. caseKey: (item) => item.slug.`);
|
|
3119
|
+
if (pick !== void 0 && typeof pick !== "function") throw new UsageError(`where in ${where} must be a function from a case and an environment to true or false, e.g. where: ({ case: item, environment }) => item.stacks.includes(environment.traits.stack).`);
|
|
3120
|
+
if (isObjectLike(rest.runs) && "bestOf" in rest.runs) throw new UsageError(`${where} has cases, so it can't take runs: { bestOf }: best-of stops each cell on its own, and the cases pool over the runs. Set a fixed count, e.g. runs: 3.`);
|
|
3121
|
+
const keys = caseKeys(cases, caseKey, where);
|
|
3122
|
+
const shownKeys = keys.map((k) => redactSecrets(k));
|
|
3123
|
+
const own = withPaths(rest, where);
|
|
2852
3124
|
const file = collectingFile();
|
|
2853
|
-
const body = bodyHash(fn,
|
|
2854
|
-
cases.
|
|
2855
|
-
const
|
|
3125
|
+
const body = bodyHash(fn, where, file);
|
|
3126
|
+
const prepared = cases.map((item, i) => {
|
|
3127
|
+
const key = shownKeys[i];
|
|
3128
|
+
const family = {
|
|
2856
3129
|
name,
|
|
2857
3130
|
case: i,
|
|
2858
3131
|
cases: cases.length
|
|
2859
|
-
}
|
|
3132
|
+
};
|
|
2860
3133
|
const source = {
|
|
2861
3134
|
body,
|
|
2862
|
-
|
|
3135
|
+
caseJson: caseJson(item, `${where} case "${key}"`, file)
|
|
2863
3136
|
};
|
|
2864
|
-
|
|
3137
|
+
const run = (c) => fn(c, item);
|
|
3138
|
+
const picks = pick ? {
|
|
3139
|
+
label: where,
|
|
3140
|
+
key,
|
|
3141
|
+
item,
|
|
3142
|
+
where: pick
|
|
3143
|
+
} : void 0;
|
|
3144
|
+
return prepare(mode, caseEvalName(name, keys[i]), own, run, source, "", family, picks);
|
|
2865
3145
|
});
|
|
2866
|
-
|
|
3146
|
+
if (pick) applyWhere(where, shownKeys, prepared);
|
|
3147
|
+
for (const p of prepared) p.commit();
|
|
3148
|
+
}
|
|
3149
|
+
/**
|
|
3150
|
+
* The checks `where` needs across an eval's cases, then the pairs it left out: each case's own on
|
|
3151
|
+
* its plan, and every case's in the eval's notes. A cell that no case runs on is refused, and so
|
|
3152
|
+
* is an eval it leaves nothing to run.
|
|
3153
|
+
*/
|
|
3154
|
+
function applyWhere(where, keys, prepared) {
|
|
3155
|
+
const picks = prepared.flatMap((p) => p.planned.picked ? [p.planned.picked] : []);
|
|
3156
|
+
if (!picks.length) return;
|
|
3157
|
+
const excluded = picks.flatMap((p) => p.excluded);
|
|
3158
|
+
const kept = new Set(picks.flatMap((p) => p.keptCells));
|
|
3159
|
+
if (!kept.size) throw new UsageError(`${where}'s where excludes all ${excluded.length} case-and-environment pairs, so it has nothing to run.`);
|
|
3160
|
+
const empty = picks.flatMap((p) => p.cells).find((c) => !kept.has(c.key));
|
|
3161
|
+
if (empty) throw new UsageError(`${where}'s where leaves cell ${empty.key} with no cases. Leave ${empty.environment} out of environments, or widen where.`);
|
|
3162
|
+
if (!excluded.length) return;
|
|
3163
|
+
const byEnv = /* @__PURE__ */ new Map();
|
|
3164
|
+
for (const e of excluded) {
|
|
3165
|
+
const on = e.harness ? `${e.environment} for ${e.harness}` : e.environment;
|
|
3166
|
+
const list = byEnv.get(on) ?? [];
|
|
3167
|
+
if (!list.includes(e.case)) list.push(e.case);
|
|
3168
|
+
byEnv.set(on, list);
|
|
3169
|
+
}
|
|
3170
|
+
const order = (k) => keys.indexOf(k);
|
|
3171
|
+
const notes = [`where excludes ${excluded.length} ${excluded.length === 1 ? "pair" : "pairs"}`, ...[...byEnv].map(([on, list]) => `excluded on ${on}: ${list.sort((a, b) => order(a) - order(b)).join(", ")}`)];
|
|
3172
|
+
const plans = /* @__PURE__ */ new Set();
|
|
3173
|
+
for (const p of prepared) {
|
|
3174
|
+
const own = p.planned.picked?.excluded ?? [];
|
|
3175
|
+
for (const t of p.planned.tests ?? []) {
|
|
3176
|
+
plans.add(t.attempt.plan);
|
|
3177
|
+
if (own.length) t.attempt.plan.excluded = own;
|
|
3178
|
+
if (t.attempt.family) t.attempt.plan.familyExcluded = excluded;
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
for (const list of new Set([...plans].map((p) => p.notes))) list.push(...notes);
|
|
3182
|
+
}
|
|
2867
3183
|
const evaluate = Object.assign(defineEval("run"), {
|
|
2868
3184
|
skip: defineEval("skip"),
|
|
2869
3185
|
only: defineEval("only"),
|
|
2870
|
-
todo: registerTodo
|
|
2871
|
-
each: evalEach
|
|
3186
|
+
todo: registerTodo
|
|
2872
3187
|
});
|
|
2873
3188
|
/**
|
|
2874
3189
|
* Declares a tree's groups and evals as the calls that made it would, in the file being
|
|
@@ -2886,13 +3201,13 @@ function declareNodes(nodes, bodyFor, base) {
|
|
|
2886
3201
|
for (const node of nodes) if (node.type === "todo") registerTodo(node.name);
|
|
2887
3202
|
else if (node.type === "group") {
|
|
2888
3203
|
const children = node.children;
|
|
2889
|
-
const where = `group("${node.name}")`;
|
|
3204
|
+
const where = `group("${redactSecrets(node.name)}")`;
|
|
2890
3205
|
defineGroup(node.mode)(node.name, optionsFrom(node.options, where, base), () => declareNodes(children, bodyFor, base));
|
|
2891
3206
|
} else if (node.family) {
|
|
2892
|
-
const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
|
|
3207
|
+
const options = optionsFrom(node.options, `evaluate("${redactSecrets(node.name)}")`, base);
|
|
2893
3208
|
checkOnly(node.mode, "evaluate");
|
|
2894
3209
|
register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
|
|
2895
|
-
} else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
3210
|
+
} else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${redactSecrets(node.name)}")`, base), bodyFor(node.body));
|
|
2896
3211
|
}
|
|
2897
3212
|
/**
|
|
2898
3213
|
* Options from their data forms, with host paths absolute against `base`: environments become
|
|
@@ -2900,32 +3215,30 @@ function declareNodes(nodes, bodyFor, base) {
|
|
|
2900
3215
|
*/
|
|
2901
3216
|
function optionsFrom(written, where, base) {
|
|
2902
3217
|
refuseIllegalCompare(written, where);
|
|
2903
|
-
const {
|
|
3218
|
+
const { environments, compare, ...rest } = withPaths(written, where, base);
|
|
2904
3219
|
const missing = missingIn(where);
|
|
2905
3220
|
const asEnv = (env, field) => environmentFrom(environmentPaths(env, base, field, missing));
|
|
2906
|
-
const
|
|
3221
|
+
const asEnvs = (value, field) => {
|
|
3222
|
+
if (Array.isArray(value)) return value.map((e, i) => asEnv(e, `${field}[${i}]`));
|
|
3223
|
+
if ("only" in value) return value;
|
|
3224
|
+
if ("add" in value) return { add: [value.add].flat().map((e, i) => asEnv(e, `${field}.add[${i}]`)) };
|
|
3225
|
+
return asEnv(value, field);
|
|
3226
|
+
};
|
|
3227
|
+
const own = environments !== void 0 ? { environments: asEnvs(environments, "environments") } : {};
|
|
2907
3228
|
if (compare === void 0) return {
|
|
2908
3229
|
...rest,
|
|
2909
3230
|
...own
|
|
2910
3231
|
};
|
|
2911
3232
|
const variants = compare.variants.map((v, i) => asEnv(v, `compare.variants[${i}]`));
|
|
2912
|
-
if (compare.baseline !== void 0) return {
|
|
2913
|
-
...rest,
|
|
2914
|
-
compare: {
|
|
2915
|
-
...compare,
|
|
2916
|
-
baseline: asEnv(compare.baseline, "compare.baseline"),
|
|
2917
|
-
variants
|
|
2918
|
-
}
|
|
2919
|
-
};
|
|
2920
3233
|
return {
|
|
2921
3234
|
...rest,
|
|
2922
|
-
...own,
|
|
2923
3235
|
compare: {
|
|
2924
3236
|
...compare,
|
|
3237
|
+
control: asEnv(compare.control, "compare.control"),
|
|
2925
3238
|
variants
|
|
2926
3239
|
}
|
|
2927
3240
|
};
|
|
2928
3241
|
}
|
|
2929
3242
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2930
3243
|
|
|
2931
|
-
//# sourceMappingURL=evaluate-
|
|
3244
|
+
//# sourceMappingURL=evaluate-CwKeN_fo.js.map
|