reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,61 +1,3 @@
|
|
|
1
|
-
/** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
|
|
2
|
-
type DeclaredMode = 'run' | 'skip' | 'only';
|
|
3
|
-
/** A `group`: options its evals and groups inherit, and its children in declaration order. */
|
|
4
|
-
type DeclaredGroup = {
|
|
5
|
-
type: 'group';
|
|
6
|
-
/** The group's name, as passed to `group`. */
|
|
7
|
-
name: string;
|
|
8
|
-
mode: DeclaredMode;
|
|
9
|
-
/** The group's own options, before the cascade. */
|
|
10
|
-
options: EvalOptionsData;
|
|
11
|
-
children: DeclaredNode[];
|
|
12
|
-
};
|
|
13
|
-
/** An `evaluate`: one eval, planned into a test per attempt. */
|
|
14
|
-
type DeclaredEval = {
|
|
15
|
-
type: 'eval';
|
|
16
|
-
/** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
|
|
17
|
-
name: string;
|
|
18
|
-
mode: DeclaredMode;
|
|
19
|
-
/** The eval's own options, before the cascade. */
|
|
20
|
-
options: EvalOptionsData;
|
|
21
|
-
/** Names the eval's body to whoever runs it. */
|
|
22
|
-
body: string;
|
|
23
|
-
source: DeclaredSource;
|
|
24
|
-
/** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
|
|
25
|
-
family?: DeclaredFamily;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
|
|
29
|
-
* text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
|
|
30
|
-
* whitespace, non-ASCII characters as themselves).
|
|
31
|
-
*/
|
|
32
|
-
type DeclaredSource = {
|
|
33
|
-
/** sha256 of the body function's source text. */
|
|
34
|
-
body: string;
|
|
35
|
-
/** sha256 of the case's data; set on each case of an `evaluate.each`. */
|
|
36
|
-
case?: string;
|
|
37
|
-
};
|
|
38
|
-
/** Where a case of a pooled `evaluate.each` sits in its list. */
|
|
39
|
-
type DeclaredFamily = {
|
|
40
|
-
/** The name template passed to `evaluate.each`. */
|
|
41
|
-
name: string;
|
|
42
|
-
/** The case's index, from 0. */
|
|
43
|
-
case: number;
|
|
44
|
-
/** How many cases the list has. */
|
|
45
|
-
cases: number;
|
|
46
|
-
};
|
|
47
|
-
/** `evaluate.todo` or `group.todo`. */
|
|
48
|
-
type DeclaredTodo = {
|
|
49
|
-
type: 'todo';
|
|
50
|
-
/** The name passed to `.todo`. */
|
|
51
|
-
name: string;
|
|
52
|
-
};
|
|
53
|
-
/** One node of the tree: a group, an eval or a todo. */
|
|
54
|
-
type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
|
|
55
|
-
/** An eval file's top-level groups, evals and todos, in declaration order. */
|
|
56
|
-
type DeclarationTree = {
|
|
57
|
-
children: DeclaredNode[];
|
|
58
|
-
};
|
|
59
1
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
60
2
|
declare const EFFORT_ALPHA = 0.05;
|
|
61
3
|
/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
|
|
@@ -315,13 +257,23 @@ type UpstreamCount = {
|
|
|
315
257
|
provider: string;
|
|
316
258
|
calls: number;
|
|
317
259
|
};
|
|
260
|
+
/** An infra error an attempt ended with before it ran again, and when that try started. */
|
|
261
|
+
type AttemptRetry = {
|
|
262
|
+
error: AttemptError;
|
|
263
|
+
at: string | null;
|
|
264
|
+
};
|
|
318
265
|
type AttemptRecord = {
|
|
319
|
-
/**
|
|
266
|
+
/**
|
|
267
|
+
* `<eval key> # <cell key>#<n>`, or `<eval key> [<case>] # <cell key>#<n>` on an eval with
|
|
268
|
+
* `cases`: unique in the run.
|
|
269
|
+
*/
|
|
320
270
|
key: string;
|
|
321
|
-
/** `<cell key>#<n
|
|
271
|
+
/** `<cell key>#<n>`, or `<cell key>#<case>#<n>` on an eval with `cases`: unique in its evaluation. */
|
|
322
272
|
id: string;
|
|
323
273
|
/** The cell's key; see `EvalEntry.cells`. */
|
|
324
274
|
cell: string;
|
|
275
|
+
/** The case's key, on an eval with `cases`; see `EvalEntry.cases`. */
|
|
276
|
+
case?: string;
|
|
325
277
|
/** Run number within the cell, from 1. */
|
|
326
278
|
n: number;
|
|
327
279
|
/**
|
|
@@ -335,6 +287,11 @@ type AttemptRecord = {
|
|
|
335
287
|
startedAt: string | null;
|
|
336
288
|
/** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
|
|
337
289
|
imageId: string | null;
|
|
290
|
+
/**
|
|
291
|
+
* The image a `tart()` attempt's VM was cloned from, with the digest Tart had for it. Absent in
|
|
292
|
+
* other sandboxes, and in attempts that didn't reach their sandbox.
|
|
293
|
+
*/
|
|
294
|
+
vmImage?: VmImage;
|
|
338
295
|
/** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
|
|
339
296
|
env: {
|
|
340
297
|
vars: string[];
|
|
@@ -463,6 +420,13 @@ type AttemptRecord = {
|
|
|
463
420
|
judgements: JudgmentRecord[];
|
|
464
421
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
465
422
|
startRetries?: StartRetry[];
|
|
423
|
+
/**
|
|
424
|
+
* The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
|
|
425
|
+
* it again, oldest first: each `{ error, at }`, `at` being when that try started (null when it
|
|
426
|
+
* never did). The attempt's own `outcome` and `error` are its last try's. Absent when it never
|
|
427
|
+
* ran again.
|
|
428
|
+
*/
|
|
429
|
+
retries?: AttemptRetry[];
|
|
466
430
|
/**
|
|
467
431
|
* Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
|
|
468
432
|
* without `hosts` in their environment, so they hold its real value while the agent works and
|
|
@@ -516,6 +480,11 @@ type AttemptRecord = {
|
|
|
516
480
|
* in the attempt's directory, `<n>` the question's place in the eval's `questions`.
|
|
517
481
|
*/
|
|
518
482
|
answers?: AttemptAnswer[];
|
|
483
|
+
/**
|
|
484
|
+
* Each `c.observe(name, value)` the attempt made, one per name in first-call order with its last
|
|
485
|
+
* value; unset without any. They annotate: `outcome` never depends on them.
|
|
486
|
+
*/
|
|
487
|
+
observations?: Observation[];
|
|
519
488
|
};
|
|
520
489
|
type EvalEntry = {
|
|
521
490
|
/** `file › group › … › name`, as shown. */
|
|
@@ -550,11 +519,23 @@ type EvalEntry = {
|
|
|
550
519
|
/**
|
|
551
520
|
* Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
|
|
552
521
|
* `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
553
|
-
* cell's failures don't fail the evaluation.
|
|
522
|
+
* cell's failures don't fail the evaluation. With `cases`, a cell's outcome is over every case.
|
|
554
523
|
*/
|
|
555
524
|
outcome: Outcome;
|
|
525
|
+
/**
|
|
526
|
+
* In records from before an eval with `cases` was one entry, `family` on a case whose
|
|
527
|
+
* `gate: { passRate }` its family held over the mean of its cases. Never written now: see
|
|
528
|
+
* `CaseRecord.verdict`.
|
|
529
|
+
*/
|
|
530
|
+
verdict?: 'family';
|
|
556
531
|
/** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
|
|
557
532
|
todo: boolean;
|
|
533
|
+
/**
|
|
534
|
+
* Declared not to run: `evaluate.skip`, inside a `group.skip`, or left out by an `.only` in its
|
|
535
|
+
* file. Its outcome is `skipped`. Unset otherwise, so in a run that isn't `complete` a skipped
|
|
536
|
+
* eval with neither `todo` nor `skip` hasn't run yet. Unset on records from before it.
|
|
537
|
+
*/
|
|
538
|
+
skip?: true;
|
|
558
539
|
/** pass@1, pass / (pass + fail), over the gated cells' attempts; infra errors and skips excluded. */
|
|
559
540
|
passRate: number | null;
|
|
560
541
|
cells: CellRecord[];
|
|
@@ -568,13 +549,63 @@ type EvalEntry = {
|
|
|
568
549
|
* with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
|
|
569
550
|
*/
|
|
570
551
|
answers?: CompareAnswer[];
|
|
571
|
-
/**
|
|
552
|
+
/**
|
|
553
|
+
* In records from before an eval with `cases` was one entry, set on each case, which was an
|
|
554
|
+
* entry of its own. Never written now; reading such a record folds the cases into one entry.
|
|
555
|
+
*/
|
|
572
556
|
family?: {
|
|
573
557
|
id: string;
|
|
574
558
|
name: string;
|
|
575
559
|
case: number;
|
|
576
560
|
cases: number;
|
|
577
561
|
};
|
|
562
|
+
/**
|
|
563
|
+
* The environments (and harnesses) `where` left out, one entry per case and environment,
|
|
564
|
+
* including those of a case it left out everywhere; unset when none.
|
|
565
|
+
*/
|
|
566
|
+
excluded?: ExcludedPair[];
|
|
567
|
+
/**
|
|
568
|
+
* An eval with `cases`: each case that has attempts in this run, in list order, with its own
|
|
569
|
+
* cells. `cells` and `attempts` are over every case. Unset without cases.
|
|
570
|
+
*/
|
|
571
|
+
cases?: CaseRecord[];
|
|
572
|
+
/** With `cases`: how many the list has, `cases` holding those with attempts in this run. */
|
|
573
|
+
caseCount?: number;
|
|
574
|
+
/**
|
|
575
|
+
* With `cases` and `compare`: each variant cell against its control, paired by case. Its
|
|
576
|
+
* `minLift` gates the lift's lower bound, and a lift below it fails the variant cell.
|
|
577
|
+
*/
|
|
578
|
+
lifts?: FamilyLift[];
|
|
579
|
+
};
|
|
580
|
+
/** One case of an eval with `cases`, and how its attempts went. */
|
|
581
|
+
type CaseRecord = {
|
|
582
|
+
/** The case's key: `nextjs` for the case keyed `nextjs`. */
|
|
583
|
+
key: string;
|
|
584
|
+
/** Its place in the case list, from 0. */
|
|
585
|
+
index: number;
|
|
586
|
+
/**
|
|
587
|
+
* The name it ran under in a record from before an eval with `cases` was one entry, when that
|
|
588
|
+
* isn't `<eval name> [<key>]` (a name template's, `fixes GH-101` for `fixes $id`).
|
|
589
|
+
*/
|
|
590
|
+
name?: string;
|
|
591
|
+
/** `EvalEntry.source.case`: the hash of the case's data. */
|
|
592
|
+
hash?: string;
|
|
593
|
+
/**
|
|
594
|
+
* What the case did, over its own cells by each cell's gate. Under a `gate: { passRate }`, the
|
|
595
|
+
* eval's cell gates on the mean over the cases, so `verdict` says this outcome decides nothing.
|
|
596
|
+
*/
|
|
597
|
+
outcome: Outcome;
|
|
598
|
+
/** `family` when a cell's `gate: { passRate }` gates the mean over the cases, not this case. */
|
|
599
|
+
verdict?: 'family';
|
|
600
|
+
/** pass@1 over its gated cells' attempts. */
|
|
601
|
+
passRate: number | null;
|
|
602
|
+
fingerprint: string;
|
|
603
|
+
/** Its own cells: their attempts are those with this `case`. */
|
|
604
|
+
cells: CellRecord[];
|
|
605
|
+
/** The environments (and harnesses) `where` left this case out of; unset when none. */
|
|
606
|
+
excluded?: ExcludedPair[];
|
|
607
|
+
/** The eval's pairwise questions, answered over this case's attempts; unset without any. */
|
|
608
|
+
answers?: CompareAnswer[];
|
|
578
609
|
};
|
|
579
610
|
/** A 95% interval on a pooled rate or difference. */
|
|
580
611
|
type FamilyInterval = {
|
|
@@ -588,7 +619,7 @@ type FamilyInterval = {
|
|
|
588
619
|
*/
|
|
589
620
|
method: 'wilson' | 'newcombe-paired' | 't-cases';
|
|
590
621
|
};
|
|
591
|
-
/** One cell's pass@1 over
|
|
622
|
+
/** One cell's pass@1 over an eval's cases. */
|
|
592
623
|
type FamilyRate = {
|
|
593
624
|
/** The cell's key, the same in every case. */
|
|
594
625
|
cell: string;
|
|
@@ -600,13 +631,19 @@ type FamilyRate = {
|
|
|
600
631
|
/** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
|
|
601
632
|
rate: number;
|
|
602
633
|
interval: FamilyInterval;
|
|
634
|
+
/** The cell's `gate: { passRate }`: with cases it gates `rate`, not each case's pass@1. */
|
|
635
|
+
passRate?: number;
|
|
636
|
+
/** Set with `passRate`: `fail` when `rate` is below it. */
|
|
637
|
+
outcome?: 'pass' | 'fail';
|
|
603
638
|
};
|
|
639
|
+
/** A cell's pass@1 over an eval's cases. */
|
|
640
|
+
type CellRate = Omit<FamilyRate, 'cell'>;
|
|
604
641
|
/**
|
|
605
642
|
* `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
|
|
606
643
|
* discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
|
|
607
644
|
*/
|
|
608
645
|
type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
|
|
609
|
-
/** One effort metric over
|
|
646
|
+
/** One effort metric over an eval's paired cases. */
|
|
610
647
|
type FamilyEffortMetric = {
|
|
611
648
|
/** Cases with a value on both sides. */
|
|
612
649
|
pairs: number;
|
|
@@ -627,7 +664,7 @@ type FamilyEffort = {
|
|
|
627
664
|
/** Over the cases where every counted attempt reported usage; omitted when none did. */
|
|
628
665
|
tokens?: FamilyEffortMetric;
|
|
629
666
|
};
|
|
630
|
-
/** A variant cell against its control over
|
|
667
|
+
/** A variant cell against its control over an eval's cases, paired by case. */
|
|
631
668
|
type FamilyLift = {
|
|
632
669
|
/** The variant cell's key. */
|
|
633
670
|
variant: string;
|
|
@@ -651,13 +688,17 @@ type FamilyLift = {
|
|
|
651
688
|
/** The interval spans 0. */
|
|
652
689
|
withinNoise: boolean;
|
|
653
690
|
verdict: FamilyVerdict;
|
|
654
|
-
/** The compare's `minLift`:
|
|
691
|
+
/** The compare's `minLift`: with cases it gates this lower bound, not each case's. */
|
|
655
692
|
minLift?: number;
|
|
656
693
|
/** `fail` when `interval.lo` is below `minLift`. */
|
|
657
694
|
outcome: 'pass' | 'fail';
|
|
658
695
|
effort?: FamilyEffort;
|
|
659
696
|
};
|
|
660
|
-
/**
|
|
697
|
+
/**
|
|
698
|
+
* In records from before an eval with `cases` was one entry: the eval over its cases, each case
|
|
699
|
+
* an entry of its own. Reading such a record folds it into the eval's `cases`, `lifts` and each
|
|
700
|
+
* cell's `rate`.
|
|
701
|
+
*/
|
|
661
702
|
type FamilyRecord = {
|
|
662
703
|
/** `file › group › … › template`. */
|
|
663
704
|
id: string;
|
|
@@ -667,13 +708,24 @@ type FamilyRecord = {
|
|
|
667
708
|
name: string;
|
|
668
709
|
/** Cases in the list. */
|
|
669
710
|
cases: number;
|
|
670
|
-
/**
|
|
711
|
+
/**
|
|
712
|
+
* Cases with an attempt that ran (not skipped) in this run; fewer when a filter or tag left
|
|
713
|
+
* some out, or an interrupted run or a filtered resume hasn't run them yet.
|
|
714
|
+
*/
|
|
671
715
|
ran: number;
|
|
672
716
|
/** One per cell. */
|
|
673
717
|
rates: FamilyRate[];
|
|
674
718
|
/** One per variant cell, with `compare`. */
|
|
675
719
|
lifts: FamilyLift[];
|
|
676
|
-
/**
|
|
720
|
+
/**
|
|
721
|
+
* Every case and environment (and harness) pair `where` left out, including those of a case it
|
|
722
|
+
* left out everywhere, which has no evaluation; unset when none.
|
|
723
|
+
*/
|
|
724
|
+
excluded?: ExcludedPair[];
|
|
725
|
+
/**
|
|
726
|
+
* `fail` when a lift is below its `minLift` or a rate below its `passRate`. A case still fails
|
|
727
|
+
* on its own under `gate: 'all'`.
|
|
728
|
+
*/
|
|
677
729
|
outcome: 'pass' | 'fail';
|
|
678
730
|
};
|
|
679
731
|
/** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
|
|
@@ -710,6 +762,7 @@ type FingerprintParts = {
|
|
|
710
762
|
/** Over the harness descriptor, less concurrency caps. */
|
|
711
763
|
harness: string;
|
|
712
764
|
environment: string | null;
|
|
765
|
+
/** The Docker image's id, or the digest of the image a `tart()` VM was cloned from. */
|
|
713
766
|
image: string | null;
|
|
714
767
|
agentCli: string | null;
|
|
715
768
|
/** Over the judge's model, threshold and version. */
|
|
@@ -770,7 +823,13 @@ type CellRecord = Cell & {
|
|
|
770
823
|
* reports the worst outcome. A variant below its `minLift` fails too.
|
|
771
824
|
*/
|
|
772
825
|
outcome: Outcome;
|
|
826
|
+
/** Unset on an eval with several `cases`, whose variants have `EvalEntry.lifts`. */
|
|
773
827
|
lift?: Lift;
|
|
828
|
+
/**
|
|
829
|
+
* On an eval with several `cases`: the cell's pass@1 over them. A `gate: { passRate }` gates
|
|
830
|
+
* `rate.rate`, the mean of the cases' pass@1, and sets `rate.outcome`.
|
|
831
|
+
*/
|
|
832
|
+
rate?: CellRate;
|
|
774
833
|
/**
|
|
775
834
|
* Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
|
|
776
835
|
* attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
|
|
@@ -817,6 +876,24 @@ type Cost = {
|
|
|
817
876
|
usage: TokenUsage | null;
|
|
818
877
|
source: 'reported' | 'unreported';
|
|
819
878
|
};
|
|
879
|
+
/**
|
|
880
|
+
* The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
|
|
881
|
+
* them, so it finishes the attempts the run planned.
|
|
882
|
+
*/
|
|
883
|
+
type RunInvocation = {
|
|
884
|
+
/** `--runs`; unset when each cell ran its own `runs`. */
|
|
885
|
+
runs?: number;
|
|
886
|
+
/** The profile the run was sized with; unset when it had none. */
|
|
887
|
+
profile?: string;
|
|
888
|
+
/** `--concurrency` or `REEVAL_CONCURRENCY`; unset when the config or the machine sized it. */
|
|
889
|
+
concurrency?: number;
|
|
890
|
+
/** The filter arguments. */
|
|
891
|
+
filters: string[];
|
|
892
|
+
/** `--tag`, without the profile's own tags. */
|
|
893
|
+
tags: string[];
|
|
894
|
+
/** `--trait`, as given (`key=value`); unset without any. */
|
|
895
|
+
traits?: string[];
|
|
896
|
+
};
|
|
820
897
|
type RunRecord = {
|
|
821
898
|
kind: 'reeval-run';
|
|
822
899
|
schemaVersion: typeof SCHEMA_VERSION;
|
|
@@ -826,7 +903,13 @@ type RunRecord = {
|
|
|
826
903
|
status: 'complete' | 'interrupted' | 'budget';
|
|
827
904
|
startedAt: string;
|
|
828
905
|
endedAt: string;
|
|
906
|
+
/** How long the run ran, counting each `--retry-errors` of it but not the time between them. */
|
|
829
907
|
durationMs: number;
|
|
908
|
+
/**
|
|
909
|
+
* When each `reeval run --retry-errors` of the run started, oldest first; unset for a run never
|
|
910
|
+
* retried. A `--resume` of the run while it's `interrupted` finishes the last one.
|
|
911
|
+
*/
|
|
912
|
+
retriedAt?: string[];
|
|
830
913
|
author: string;
|
|
831
914
|
git: GitInfo | null;
|
|
832
915
|
runner: RunnerInfo;
|
|
@@ -861,8 +944,11 @@ type RunRecord = {
|
|
|
861
944
|
maxVariants: number;
|
|
862
945
|
};
|
|
863
946
|
harness?: string[];
|
|
947
|
+
traits?: Record<string, string | string[]>;
|
|
864
948
|
};
|
|
865
949
|
} | null;
|
|
950
|
+
/** How `reeval run` was started; `--resume` plans with it again. Unset in older records. */
|
|
951
|
+
invocation?: RunInvocation;
|
|
866
952
|
/**
|
|
867
953
|
* What the run planned before it started, and each level whose sandbox ran as root or with
|
|
868
954
|
* capabilities added back.
|
|
@@ -907,7 +993,10 @@ type RunRecord = {
|
|
|
907
993
|
};
|
|
908
994
|
summary: RunSummary;
|
|
909
995
|
evals: EvalEntry[];
|
|
910
|
-
/**
|
|
996
|
+
/**
|
|
997
|
+
* In records from before an eval with `cases` was one entry, each eval with `cases` that ran.
|
|
998
|
+
* Never written now; reading such a record folds it into its eval.
|
|
999
|
+
*/
|
|
911
1000
|
families?: FamilyRecord[];
|
|
912
1001
|
/**
|
|
913
1002
|
* These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
|
|
@@ -1020,7 +1109,11 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
|
|
|
1020
1109
|
/** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
|
|
1021
1110
|
canceled?: number;
|
|
1022
1111
|
};
|
|
1023
|
-
/**
|
|
1112
|
+
/** One case of an eval with `cases` in a `HistoryEntry`. */
|
|
1113
|
+
type HistoryCase = Pick<CaseRecord, 'key' | 'outcome' | 'verdict' | 'fingerprint' | 'hash'> & {
|
|
1114
|
+
cells: HistoryCell[];
|
|
1115
|
+
};
|
|
1116
|
+
/** A pooled lift as the history listing keeps it: enough to compare its gate. */
|
|
1024
1117
|
type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
|
|
1025
1118
|
lo: number;
|
|
1026
1119
|
hi: number;
|
|
@@ -1065,12 +1158,14 @@ type HistoryEntry = {
|
|
|
1065
1158
|
outcome: Outcome;
|
|
1066
1159
|
passRate: number | null;
|
|
1067
1160
|
cells: HistoryCell[];
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
lifts
|
|
1161
|
+
/** An eval with `cases`: each case that ran, with its own cells; unset without cases. */
|
|
1162
|
+
cases?: HistoryCase[];
|
|
1163
|
+
/** With `cases`: how many the list has. */
|
|
1164
|
+
caseCount?: number;
|
|
1165
|
+
/** With `cases` and `compare`: each variant's lift paired by case. */
|
|
1166
|
+
lifts?: FamilyLiftSummary[];
|
|
1167
|
+
/** With `cases`: its cells' rates over the cases that a `gate: { passRate }` gates; unset when none. */
|
|
1168
|
+
rates?: Array<Pick<FamilyRate, 'cell' | 'rate'> & Required<Pick<FamilyRate, 'passRate' | 'outcome'>>>;
|
|
1074
1169
|
}>;
|
|
1075
1170
|
/** The run's errors outside any attempt; unset when there were none. */
|
|
1076
1171
|
errors?: string[];
|
|
@@ -1519,6 +1614,8 @@ type JudgeInputs = {
|
|
|
1519
1614
|
transcript?: boolean | 'tools' | 'full';
|
|
1520
1615
|
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
|
|
1521
1616
|
checks?: boolean | string[];
|
|
1617
|
+
/** `c.observe()` values so far, by name; a list shows only those names. */
|
|
1618
|
+
observations?: boolean | string[];
|
|
1522
1619
|
/**
|
|
1523
1620
|
* Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
|
|
1524
1621
|
* capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
|
|
@@ -1553,6 +1650,8 @@ type ResolvedJudgeStateOptions = {
|
|
|
1553
1650
|
transcript: boolean | 'tools' | 'full';
|
|
1554
1651
|
turns: 'last' | 'all';
|
|
1555
1652
|
checks: boolean | string[];
|
|
1653
|
+
/** Set only when the call turned it on, so never `false` as written. */
|
|
1654
|
+
observations?: boolean | string[];
|
|
1556
1655
|
};
|
|
1557
1656
|
type Verdict = {
|
|
1558
1657
|
verdict: 'pass' | 'fail';
|
|
@@ -1628,6 +1727,12 @@ type AttemptContext = {
|
|
|
1628
1727
|
* made to it afterwards.
|
|
1629
1728
|
*/
|
|
1630
1729
|
writeFile(path: string, contents: string | Uint8Array): Promise<void>;
|
|
1730
|
+
/**
|
|
1731
|
+
* Records a fact about the attempt under `name`, such as what the agent installed. It never
|
|
1732
|
+
* affects the outcome. Calling it again with a name replaces its value. The value is JSON, at
|
|
1733
|
+
* most 4 KB, and is masked like the rest of the record.
|
|
1734
|
+
*/
|
|
1735
|
+
observe(name: string, value: JSONValue): void;
|
|
1631
1736
|
/** Every turn so far, oldest first. */
|
|
1632
1737
|
readonly turns: readonly Turn[];
|
|
1633
1738
|
/** Aborted on timeout or Ctrl-C. */
|
|
@@ -1640,6 +1745,49 @@ type AttemptContext = {
|
|
|
1640
1745
|
};
|
|
1641
1746
|
};
|
|
1642
1747
|
type EvalFn = (c: AttemptContext) => Promise<void>;
|
|
1748
|
+
/** The body of an eval with `cases`: the attempt and its case. */
|
|
1749
|
+
type CaseFn<T> = (c: AttemptContext, item: T) => Promise<void>;
|
|
1750
|
+
/**
|
|
1751
|
+
* One eval over several inputs: each case runs the body with every cell and run, and the cases
|
|
1752
|
+
* pool into one pass rate (and, with `compare`, one lift paired by case). Per eval only.
|
|
1753
|
+
*/
|
|
1754
|
+
type CaseOptions<T> = {
|
|
1755
|
+
/** The inputs, each JSON data; the body gets one as its second argument. */
|
|
1756
|
+
cases: readonly T[];
|
|
1757
|
+
/**
|
|
1758
|
+
* Each case's key, which names it in the output and the record. Without it, a string or a
|
|
1759
|
+
* finite number is its own key, and an object uses its `id`, else its `name`.
|
|
1760
|
+
*/
|
|
1761
|
+
caseKey?: (item: T) => string;
|
|
1762
|
+
/**
|
|
1763
|
+
* Which case and environment pairs run, decided at plan time. It sees each environment's
|
|
1764
|
+
* `traits`, and with `compare` the control, whose answer holds for every variant. Reading
|
|
1765
|
+
* `harness` decides each harness on its own. The pairs it leaves out are listed in `--dry-run`
|
|
1766
|
+
* and the record's `excluded`.
|
|
1767
|
+
*/
|
|
1768
|
+
where?: Where<T>;
|
|
1769
|
+
};
|
|
1770
|
+
/** Filter facts about an environment, for `where`: never sent to the agent, never prompt data. */
|
|
1771
|
+
type Traits = Readonly<Record<string, string | readonly string[]>>;
|
|
1772
|
+
/** `where`'s predicate: true runs the case on that environment (and harness). */
|
|
1773
|
+
type Where<T> = (pair: {
|
|
1774
|
+
case: T;
|
|
1775
|
+
environment: {
|
|
1776
|
+
readonly name: string;
|
|
1777
|
+
/** The environment's traits; reading a key that no listed environment defines is refused. */
|
|
1778
|
+
readonly traits: Readonly<Record<string, string | readonly string[] | undefined>>;
|
|
1779
|
+
};
|
|
1780
|
+
/** The harness id; reading it decides each harness on its own. */
|
|
1781
|
+
readonly harness: string;
|
|
1782
|
+
}) => boolean;
|
|
1783
|
+
/** A case and environment pair `where` left out. */
|
|
1784
|
+
type ExcludedPair = {
|
|
1785
|
+
/** The case key. */
|
|
1786
|
+
case: string;
|
|
1787
|
+
environment: string;
|
|
1788
|
+
/** Set only when `where` read the harness. */
|
|
1789
|
+
harness?: string;
|
|
1790
|
+
};
|
|
1643
1791
|
/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
|
|
1644
1792
|
type FakeHarnessDescriptor = {
|
|
1645
1793
|
kind: 'fake';
|
|
@@ -1658,6 +1806,11 @@ type FakeHarnessDescriptor = {
|
|
|
1658
1806
|
delayMs?: number;
|
|
1659
1807
|
/** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
|
|
1660
1808
|
ignoreAbort?: boolean;
|
|
1809
|
+
/**
|
|
1810
|
+
* Each turn's recorded agent time, exactly, whatever the wall clock says: a test can compare
|
|
1811
|
+
* agent time without scheduling jitter, which four runs a side can tell apart.
|
|
1812
|
+
*/
|
|
1813
|
+
turnMs?: number;
|
|
1661
1814
|
/** Throw before the first turn starts, to simulate an infra failure. */
|
|
1662
1815
|
failWith?: string;
|
|
1663
1816
|
/** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
|
|
@@ -1691,6 +1844,14 @@ type FakeHarnessDescriptor = {
|
|
|
1691
1844
|
};
|
|
1692
1845
|
/** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
|
|
1693
1846
|
flakeRate?: number;
|
|
1847
|
+
/** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
|
|
1848
|
+
usage?: {
|
|
1849
|
+
input: number;
|
|
1850
|
+
output: number;
|
|
1851
|
+
cacheRead: number;
|
|
1852
|
+
cacheWrite: number;
|
|
1853
|
+
reasoning: number | null;
|
|
1854
|
+
};
|
|
1694
1855
|
/**
|
|
1695
1856
|
* Real sandbox actions, one list per `c.send` (the last list repeats), run in order before
|
|
1696
1857
|
* `files`. `exec` runs as the agent: with the sandbox's own variables, none of the checks'.
|
|
@@ -2118,7 +2279,62 @@ type DockerSandboxDescriptor = {
|
|
|
2118
2279
|
/** Processes and threads at once. Default 1024. */
|
|
2119
2280
|
pids?: number;
|
|
2120
2281
|
};
|
|
2121
|
-
|
|
2282
|
+
/** A macOS VM on this Mac, cloned from `image` for each attempt by Tart, made by `tart()`. */
|
|
2283
|
+
type TartSandboxDescriptor = {
|
|
2284
|
+
kind: 'tart';
|
|
2285
|
+
/** Cell id; defaults to `tart:<image>`, the image's last path segment. */
|
|
2286
|
+
id?: string;
|
|
2287
|
+
/**
|
|
2288
|
+
* The VM each attempt clones: an OCI image such as
|
|
2289
|
+
* `ghcr.io/cirruslabs/macos-sequoia-base:latest`, or a local VM's name. Tart must have it
|
|
2290
|
+
* before the run: `reeval image build` (or `tart pull`) pulls it, and `reeval run` refuses one
|
|
2291
|
+
* `tart list` doesn't show.
|
|
2292
|
+
*/
|
|
2293
|
+
image: string;
|
|
2294
|
+
/**
|
|
2295
|
+
* The most Tart VMs at once, across the run: 1 or 2, and 2 when unset, since macOS runs at most
|
|
2296
|
+
* two macOS VMs at a time.
|
|
2297
|
+
*/
|
|
2298
|
+
maxConcurrent?: number;
|
|
2299
|
+
/** CPUs each clone gets; the image's when unset. */
|
|
2300
|
+
cpus?: number;
|
|
2301
|
+
/** Memory each clone gets: `'8g'`, `'4096m'`, or bytes; the image's when unset. */
|
|
2302
|
+
memory?: string | number;
|
|
2303
|
+
/**
|
|
2304
|
+
* Reach the VM over SSH when its Tart Guest Agent doesn't answer `tart exec`: as `user`
|
|
2305
|
+
* (`admin` when unset), with `key`, a private key the image trusts, absolute or relative to the
|
|
2306
|
+
* file it's written in.
|
|
2307
|
+
*/
|
|
2308
|
+
ssh?: {
|
|
2309
|
+
user?: string;
|
|
2310
|
+
key: string;
|
|
2311
|
+
};
|
|
2312
|
+
};
|
|
2313
|
+
/** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
|
|
2314
|
+
type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
|
|
2315
|
+
/** The host variable holding a Boat API key, made by `secret.fromEnv`. */
|
|
2316
|
+
type BoatApiKey = {
|
|
2317
|
+
kind: 'secret';
|
|
2318
|
+
from: string;
|
|
2319
|
+
};
|
|
2320
|
+
/** A boat.dev VM per attempt, made by `boat()` from `reeval/sandbox/boat`. */
|
|
2321
|
+
type BoatSandboxDescriptor = {
|
|
2322
|
+
kind: 'boat';
|
|
2323
|
+
/** Cell id; defaults to `boat`, or `boat:<type>` with a `type`. */
|
|
2324
|
+
id?: string;
|
|
2325
|
+
/** The host variable with the Boat API key, read on the host only. */
|
|
2326
|
+
apiKey: BoatApiKey;
|
|
2327
|
+
/**
|
|
2328
|
+
* The start of every sandbox name reeval gives Boat, so a run's cleanup finds only its own:
|
|
2329
|
+
* lowercase letters, digits and dashes.
|
|
2330
|
+
*/
|
|
2331
|
+
prefix: string;
|
|
2332
|
+
/** The machine size. Default `small`. */
|
|
2333
|
+
type?: BoatMachineType;
|
|
2334
|
+
/** The most Boat sandboxes at once, across the run. */
|
|
2335
|
+
maxConcurrent?: number;
|
|
2336
|
+
};
|
|
2337
|
+
type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor | TartSandboxDescriptor | BoatSandboxDescriptor;
|
|
2122
2338
|
/**
|
|
2123
2339
|
* A host environment variable, redacted everywhere reeval writes and recorded by name only. Made by
|
|
2124
2340
|
* the `secret` helper of `defineEnvironment` and `defineConfig` callbacks.
|
|
@@ -2329,6 +2545,12 @@ type EnvironmentSpec = {
|
|
|
2329
2545
|
* parent's plugin of the same name.
|
|
2330
2546
|
*/
|
|
2331
2547
|
plugins?: PluginSource[];
|
|
2548
|
+
/**
|
|
2549
|
+
* Filter facts for `where`, such as `{ stack: 'next' }`: a key to a string or a list of
|
|
2550
|
+
* strings. A child's key replaces its parent's value whole; keys it doesn't set are inherited.
|
|
2551
|
+
* Recorded on the cell, and never written to the sandbox, the prompt or the judge.
|
|
2552
|
+
*/
|
|
2553
|
+
traits?: Traits;
|
|
2332
2554
|
};
|
|
2333
2555
|
/**
|
|
2334
2556
|
* A part of a plugin, as Claude Code's layout bundles it: `skills/<name>/SKILL.md`,
|
|
@@ -2439,6 +2661,8 @@ type Environment = {
|
|
|
2439
2661
|
readonly name: string;
|
|
2440
2662
|
/** Its `extend` chain, root first. */
|
|
2441
2663
|
readonly layers: readonly EnvironmentLayer[];
|
|
2664
|
+
/** Its layers' `traits`, merged by key with the child's winning; `{}` when none. */
|
|
2665
|
+
readonly traits: Traits;
|
|
2442
2666
|
/**
|
|
2443
2667
|
* A child environment layered on this one; wherever it sets a key this one has, it wins.
|
|
2444
2668
|
* `sandbox` is replaced entirely when the child sets one; when the child omits it, the
|
|
@@ -2540,26 +2764,39 @@ type HarnessEdit = {
|
|
|
2540
2764
|
type HarnessData = HarnessDescriptor | HarnessDescriptor[] | HarnessEdit;
|
|
2541
2765
|
/** `HarnessData`, or a function that edits the inherited list (TypeScript only). */
|
|
2542
2766
|
type HarnessOption = HarnessData | ((inherited: HarnessDescriptor[]) => HarnessDescriptor | HarnessDescriptor[]);
|
|
2543
|
-
type CompareGate = 'variants' | 'all' | 'none';
|
|
2544
2767
|
/**
|
|
2545
|
-
*
|
|
2768
|
+
* An edit of the inherited environment list: `add` appends environments, `only` keeps the
|
|
2769
|
+
* inherited ones with these names, in their order.
|
|
2770
|
+
*/
|
|
2771
|
+
type EnvironmentEdit<E = Environment> = {
|
|
2772
|
+
add: E | E[];
|
|
2773
|
+
} | {
|
|
2774
|
+
only: string | string[];
|
|
2775
|
+
};
|
|
2776
|
+
/** One environment, a list (each is its own cell), or an edit of the inherited list, as data. */
|
|
2777
|
+
type EnvironmentsData<E = EnvironmentData> = E | E[] | EnvironmentEdit<E>;
|
|
2778
|
+
/** `EnvironmentsData`, or a function that edits the inherited list (TypeScript only). */
|
|
2779
|
+
type EnvironmentsOption = EnvironmentsData<Environment> | ((inherited: Environment[]) => Environment | Environment[]);
|
|
2780
|
+
type CompareGate = 'variants' | 'all' | 'observe';
|
|
2781
|
+
/**
|
|
2782
|
+
* The control and each variant run as their own environments, with the same prompt and checks.
|
|
2546
2783
|
* `compare` does not layer them. Share setup first with `environment.extend` / `extendEnvironment`.
|
|
2547
2784
|
* `false` is not a value here: a profile turns compare off with `limit.compare`.
|
|
2548
2785
|
*/
|
|
2549
2786
|
type CompareOptions<E = Environment> = {
|
|
2550
2787
|
/**
|
|
2551
|
-
* The
|
|
2552
|
-
*
|
|
2788
|
+
* The environment each variant is measured against. Not layered under the variants, and it
|
|
2789
|
+
* replaces any `environments` the eval inherits.
|
|
2553
2790
|
*/
|
|
2554
|
-
|
|
2791
|
+
control: E;
|
|
2555
2792
|
/** Other environments, each planned and run as itself. */
|
|
2556
2793
|
variants: E[];
|
|
2557
|
-
/** Which cells decide pass/fail: `variants` (default), `all`, or `
|
|
2794
|
+
/** Which cells decide pass/fail: `variants` (default), `all`, or `observe` (none of them). */
|
|
2558
2795
|
gate?: CompareGate;
|
|
2559
2796
|
/** Lift in points from -1 to 1; a variant cell below it fails. */
|
|
2560
2797
|
minLift?: number;
|
|
2561
2798
|
/** Harnesses (ids or descriptors) to compare on; the others run the control only. */
|
|
2562
|
-
|
|
2799
|
+
harnesses?: Array<string | HarnessDescriptor>;
|
|
2563
2800
|
};
|
|
2564
2801
|
/**
|
|
2565
2802
|
* A question a language model answers after the attempts: its text, or the text and its options.
|
|
@@ -2612,6 +2849,8 @@ type QuestionInputs = {
|
|
|
2612
2849
|
transcript?: boolean | 'tools' | 'full';
|
|
2613
2850
|
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
|
|
2614
2851
|
checks?: boolean | string[];
|
|
2852
|
+
/** `c.observe()` values, by name; a list shows only those names. */
|
|
2853
|
+
observations?: boolean | string[];
|
|
2615
2854
|
/** Each attempt's own captures, by name; `true` sends all of them. */
|
|
2616
2855
|
artifacts?: true | string[];
|
|
2617
2856
|
};
|
|
@@ -2624,6 +2863,8 @@ type ResolvedQuestionStateOptions = {
|
|
|
2624
2863
|
transcript: boolean | 'tools' | 'full';
|
|
2625
2864
|
turns: 'last' | 'all';
|
|
2626
2865
|
checks: boolean | string[];
|
|
2866
|
+
/** Set only when the question turned it on, so never `false` as written. */
|
|
2867
|
+
observations?: boolean | string[];
|
|
2627
2868
|
};
|
|
2628
2869
|
/** Defaults for every question below it. What a question is shown is its own, never a default. */
|
|
2629
2870
|
type QuestionOptions = {
|
|
@@ -2706,18 +2947,22 @@ type AttemptAnswer = {
|
|
|
2706
2947
|
input?: FileRef;
|
|
2707
2948
|
};
|
|
2708
2949
|
/** An option a profile's `limit` can narrow, and `pin` can protect. */
|
|
2709
|
-
type PinnableKey = '
|
|
2950
|
+
type PinnableKey = 'harnesses' | 'runs' | 'compare';
|
|
2710
2951
|
/**
|
|
2711
2952
|
* How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
|
|
2712
2953
|
* passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
|
|
2713
|
-
* least that share
|
|
2954
|
+
* least that share; `'observe'` gates nothing, so the eval's attempts run and are recorded, and
|
|
2955
|
+
* only run and infra errors fail it.
|
|
2714
2956
|
*/
|
|
2715
|
-
type GatePolicy = 'all' | {
|
|
2957
|
+
type GatePolicy = 'all' | 'observe' | {
|
|
2716
2958
|
passRate: number;
|
|
2717
2959
|
};
|
|
2718
|
-
/**
|
|
2719
|
-
|
|
2720
|
-
|
|
2960
|
+
/**
|
|
2961
|
+
* A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's, or
|
|
2962
|
+
* any cell under `gate: 'observe'`).
|
|
2963
|
+
*/
|
|
2964
|
+
type CellGate = Exclude<GatePolicy, 'observe'> | 'none';
|
|
2965
|
+
/** A compare object: `control` and `variants`. Same as `CompareOptions`. */
|
|
2721
2966
|
type CompareSpec<E = Environment> = CompareOptions<E>;
|
|
2722
2967
|
type OddDigit = '1' | '3' | '5' | '7' | '9';
|
|
2723
2968
|
/**
|
|
@@ -2732,12 +2977,12 @@ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${s
|
|
|
2732
2977
|
type Runs<N extends number = number> = number | {
|
|
2733
2978
|
bestOf: OddAtLeast3<N>;
|
|
2734
2979
|
};
|
|
2735
|
-
/** Options other than `
|
|
2980
|
+
/** Options other than `environments` and `compare`, which are exclusive of each other. */
|
|
2736
2981
|
type EvalOptionFields<N extends number = number> = {
|
|
2737
2982
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2738
|
-
|
|
2983
|
+
harnesses?: HarnessOption;
|
|
2739
2984
|
/**
|
|
2740
|
-
* The default machine when the resolved environment omits `sandbox`. Cascades like `
|
|
2985
|
+
* The default machine when the resolved environment omits `sandbox`. Cascades like `harnesses`.
|
|
2741
2986
|
* Not an environment: it does not invent skills, files or a name.
|
|
2742
2987
|
*/
|
|
2743
2988
|
sandbox?: SandboxDescriptor;
|
|
@@ -2745,7 +2990,7 @@ type EvalOptionFields<N extends number = number> = {
|
|
|
2745
2990
|
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
|
|
2746
2991
|
* a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
|
|
2747
2992
|
* Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
|
|
2748
|
-
* a verdict, and anything else runs once, a
|
|
2993
|
+
* a verdict, and anything else runs once, a case of an eval with `cases` included.
|
|
2749
2994
|
*/
|
|
2750
2995
|
runs?: Runs<N>;
|
|
2751
2996
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
@@ -2794,51 +3039,51 @@ type Revision = number | string;
|
|
|
2794
3039
|
* An eval's own source as hashes: its body and case data as declared, and its options and its
|
|
2795
3040
|
* enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
|
|
2796
3041
|
*/
|
|
2797
|
-
type EvalSource =
|
|
3042
|
+
type EvalSource = {
|
|
3043
|
+
/** sha256 of the body function's source text. */
|
|
3044
|
+
body: string;
|
|
3045
|
+
/**
|
|
3046
|
+
* sha256 of the case's data as canonical JSON, with secrets in its strings masked as the record
|
|
3047
|
+
* masks them; set on each case of an eval with `cases`.
|
|
3048
|
+
*/
|
|
3049
|
+
case?: string;
|
|
2798
3050
|
/** sha256 of the canonical options. */
|
|
2799
3051
|
options: string;
|
|
2800
3052
|
};
|
|
2801
3053
|
/**
|
|
2802
3054
|
* `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
|
|
2803
|
-
* A compare
|
|
2804
|
-
*
|
|
2805
|
-
* own, or an inherited one) is the control.
|
|
3055
|
+
* A compare names every arm, so an inherited `environments` is not applied, and the same object
|
|
3056
|
+
* can't set both.
|
|
2806
3057
|
*/
|
|
2807
|
-
type EnvironmentOrCompare<E> = {
|
|
2808
|
-
/**
|
|
2809
|
-
|
|
3058
|
+
type EnvironmentOrCompare<E, Envs> = {
|
|
3059
|
+
/**
|
|
3060
|
+
* What this group or eval runs in: one environment, or a list, each its own cell beside
|
|
3061
|
+
* each harness. `{ add }` and `{ only }` edit the inherited list.
|
|
3062
|
+
*/
|
|
3063
|
+
environments?: Envs;
|
|
2810
3064
|
compare?: never;
|
|
2811
3065
|
} | {
|
|
2812
|
-
/** Forbidden beside `compare
|
|
2813
|
-
|
|
2814
|
-
/** Environments to run side by side; `
|
|
2815
|
-
compare: CompareSpec<E
|
|
2816
|
-
baseline: E;
|
|
2817
|
-
};
|
|
2818
|
-
} | {
|
|
2819
|
-
/** The control: the compare has no `baseline`. */
|
|
2820
|
-
environment?: E;
|
|
2821
|
-
/** Environments to run side by side against `environment`, the control. */
|
|
2822
|
-
compare: CompareSpec<E> & {
|
|
2823
|
-
baseline?: undefined;
|
|
2824
|
-
};
|
|
3066
|
+
/** Forbidden beside `compare`, whose `control` names what the eval runs in. */
|
|
3067
|
+
environments?: never;
|
|
3068
|
+
/** Environments to run side by side; `control` is what each variant is measured against. */
|
|
3069
|
+
compare: CompareSpec<E>;
|
|
2825
3070
|
};
|
|
2826
3071
|
/** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
|
|
2827
|
-
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
|
|
3072
|
+
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment, EnvironmentsOption>;
|
|
2828
3073
|
type GroupOptions<N extends number = number> = EvalOptions<N>;
|
|
2829
3074
|
/**
|
|
2830
3075
|
* `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
|
|
2831
3076
|
* by name or a fake judge, and environments without `extend`. The same `environment` / compare
|
|
2832
3077
|
* object split as `EvalOptions`.
|
|
2833
3078
|
*/
|
|
2834
|
-
type EvalOptionsData = Omit<EvalOptionFields, '
|
|
3079
|
+
type EvalOptionsData = Omit<EvalOptionFields, 'harnesses' | 'judge'> & {
|
|
2835
3080
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2836
|
-
|
|
3081
|
+
harnesses?: HarnessData;
|
|
2837
3082
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
2838
3083
|
judge?: ConfigJudgeOptions;
|
|
2839
|
-
} & EnvironmentOrCompare<EnvironmentData>;
|
|
3084
|
+
} & EnvironmentOrCompare<EnvironmentData, EnvironmentsData>;
|
|
2840
3085
|
/** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
|
|
2841
|
-
type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
|
|
3086
|
+
type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend' | 'traits'>;
|
|
2842
3087
|
/** The config file is plain data sent to workers, so its judge model can't be an object. */
|
|
2843
3088
|
type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
2844
3089
|
/** `'provider/model'`, or a fake judge. */
|
|
@@ -2860,11 +3105,11 @@ type Budget = {
|
|
|
2860
3105
|
/** What evals get when they don't say otherwise: under groups and evals. */
|
|
2861
3106
|
type ProfileDefaults = {
|
|
2862
3107
|
/** The agents every eval runs on, unless a group or eval replaces them. */
|
|
2863
|
-
|
|
3108
|
+
harnesses?: HarnessDescriptor | HarnessDescriptor[];
|
|
2864
3109
|
/** The default machine when the resolved environment omits `sandbox`. */
|
|
2865
3110
|
sandbox?: SandboxDescriptor;
|
|
2866
|
-
/** What every eval runs in when it names no environment. */
|
|
2867
|
-
|
|
3111
|
+
/** What every eval runs in when it names no environment: one, or a list (a cell each). */
|
|
3112
|
+
environments?: Environment | Environment[];
|
|
2868
3113
|
/** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
|
|
2869
3114
|
runs?: Runs;
|
|
2870
3115
|
/** How each cell's attempts decide its outcome. */
|
|
@@ -2893,7 +3138,13 @@ type ProfileLimit = {
|
|
|
2893
3138
|
maxVariants: number;
|
|
2894
3139
|
};
|
|
2895
3140
|
/** Harness ids to allow; evals left with none are skipped. */
|
|
2896
|
-
|
|
3141
|
+
harnesses?: string[];
|
|
3142
|
+
/**
|
|
3143
|
+
* Cells to run, by their environment's traits (with `compare`, the control's): each key's trait
|
|
3144
|
+
* must equal the value, or one of a list's, or for a list trait contain it. Evals left with no
|
|
3145
|
+
* cell are skipped.
|
|
3146
|
+
*/
|
|
3147
|
+
traits?: Record<string, string | string[]>;
|
|
2897
3148
|
};
|
|
2898
3149
|
/** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
|
|
2899
3150
|
type Profile = {
|
|
@@ -2985,6 +3236,14 @@ type BlindSide = BlindTurn & {
|
|
|
2985
3236
|
command: string;
|
|
2986
3237
|
notRun: true;
|
|
2987
3238
|
}>;
|
|
3239
|
+
/** Its `c.observe()` values; a name the question lists that it never observed is `missing`. */
|
|
3240
|
+
observations?: Array<{
|
|
3241
|
+
name: string;
|
|
3242
|
+
value: JSONValue;
|
|
3243
|
+
} | {
|
|
3244
|
+
name: string;
|
|
3245
|
+
missing: true;
|
|
3246
|
+
}>;
|
|
2988
3247
|
/** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
|
|
2989
3248
|
artifacts?: Array<{
|
|
2990
3249
|
label: string;
|
|
@@ -3006,21 +3265,24 @@ type AnalysisConfig = {
|
|
|
3006
3265
|
instructions?: string;
|
|
3007
3266
|
/** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
|
|
3008
3267
|
maxInputTokens?: number;
|
|
3009
|
-
/** The most tokens the model may answer with (default
|
|
3268
|
+
/** The most tokens the model may answer with, reasoning included (default 32000). reeval has no prices, so tokens are the cap. */
|
|
3010
3269
|
maxOutputTokens?: number;
|
|
3011
3270
|
};
|
|
3012
3271
|
type ReevalConfig = {
|
|
3013
3272
|
/** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
|
|
3014
3273
|
include?: string[];
|
|
3015
3274
|
/** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
|
|
3016
|
-
|
|
3275
|
+
harnesses?: HarnessDescriptor | HarnessDescriptor[];
|
|
3017
3276
|
/**
|
|
3018
3277
|
* The default machine when the resolved environment omits `sandbox`. Not an environment:
|
|
3019
3278
|
* it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
|
|
3020
3279
|
*/
|
|
3021
3280
|
sandbox?: SandboxDescriptor;
|
|
3022
|
-
/**
|
|
3023
|
-
|
|
3281
|
+
/**
|
|
3282
|
+
* What every eval runs in when it names no environment and has no compare: one, or a list (a
|
|
3283
|
+
* cell each).
|
|
3284
|
+
*/
|
|
3285
|
+
environments?: Environment | Environment[];
|
|
3024
3286
|
/**
|
|
3025
3287
|
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
|
|
3026
3288
|
* variants runs 4 a side and anything else once.
|
|
@@ -3158,9 +3420,8 @@ type PhaseTimings = {
|
|
|
3158
3420
|
type CellRole = 'control' | 'variant';
|
|
3159
3421
|
/**
|
|
3160
3422
|
* A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
|
|
3161
|
-
* environment's sandbox id. With `compare`, the control is `
|
|
3162
|
-
*
|
|
3163
|
-
* including its sandbox.
|
|
3423
|
+
* environment's sandbox id. With `compare`, the control is `compare.control` and each variant is
|
|
3424
|
+
* that environment on its own, including its sandbox.
|
|
3164
3425
|
*/
|
|
3165
3426
|
type Cell = {
|
|
3166
3427
|
/** The harness id, the environment name, and the sandbox id, joined with `/`. */
|
|
@@ -3180,9 +3441,16 @@ type Cell = {
|
|
|
3180
3441
|
* eight-character hash.
|
|
3181
3442
|
*/
|
|
3182
3443
|
sandbox: string;
|
|
3444
|
+
/**
|
|
3445
|
+
* `variant` on a compare's variant cell, else `control`: the compare's control, or any cell of
|
|
3446
|
+
* an eval without `compare`, each environment of an `environments` grid included. A variant's
|
|
3447
|
+
* lift is against the `control` cell on its harness and sandbox.
|
|
3448
|
+
*/
|
|
3183
3449
|
role: CellRole;
|
|
3184
3450
|
/** The variant's name, for variant cells. */
|
|
3185
3451
|
variant?: string;
|
|
3452
|
+
/** The environment's merged traits; unset when it has none. Not part of the key. */
|
|
3453
|
+
traits?: Traits;
|
|
3186
3454
|
};
|
|
3187
3455
|
/** A plugin as an attempt placed it. */
|
|
3188
3456
|
type PluginRecord = {
|
|
@@ -3358,6 +3626,13 @@ type AttemptImage = {
|
|
|
3358
3626
|
digest: string | null;
|
|
3359
3627
|
};
|
|
3360
3628
|
};
|
|
3629
|
+
/** The image a `tart()` attempt's VM was cloned from. */
|
|
3630
|
+
type VmImage = {
|
|
3631
|
+
/** The image as `tart({ image })` names it. */
|
|
3632
|
+
ref: string;
|
|
3633
|
+
/** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
|
|
3634
|
+
digest: string | null;
|
|
3635
|
+
};
|
|
3361
3636
|
/** A harness session start that timed out and was started again. */
|
|
3362
3637
|
type StartRetry = {
|
|
3363
3638
|
/** `<harness kind>: session did not become ready within <startTimeout>`. */
|
|
@@ -3415,6 +3690,14 @@ type AssertionRecord = {
|
|
|
3415
3690
|
/** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
|
|
3416
3691
|
durationMs?: number;
|
|
3417
3692
|
};
|
|
3693
|
+
/** A `c.observe(name, value)` call: the last value given for the name. */
|
|
3694
|
+
type Observation = {
|
|
3695
|
+
name: string;
|
|
3696
|
+
/** Masked like the rest of the record. */
|
|
3697
|
+
value: JSONValue;
|
|
3698
|
+
/** How many turns the attempt had sent when the value was given: 0 before the first. */
|
|
3699
|
+
turn: number;
|
|
3700
|
+
};
|
|
3418
3701
|
/**
|
|
3419
3702
|
* A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
|
|
3420
3703
|
* folder), or why it wasn't.
|
|
@@ -3480,10 +3763,11 @@ type CaptureStep = {
|
|
|
3480
3763
|
};
|
|
3481
3764
|
/** Which environment ran as the control, so the studio can show it. */
|
|
3482
3765
|
type EvalCompareMeta = {
|
|
3766
|
+
/** The control's environment name. */
|
|
3483
3767
|
baseline: string;
|
|
3484
3768
|
/**
|
|
3485
|
-
*
|
|
3486
|
-
* an empty
|
|
3769
|
+
* Not written, since a compare names its control. A record that has it ran the eval's cascaded
|
|
3770
|
+
* environment (`'environment'`) or an empty one named `baseline` (`'empty'`) as the control.
|
|
3487
3771
|
*/
|
|
3488
3772
|
from?: 'environment' | 'empty';
|
|
3489
3773
|
};
|
|
@@ -3505,7 +3789,7 @@ type SandboxWarning = {
|
|
|
3505
3789
|
file: string | null;
|
|
3506
3790
|
/** The capabilities added back, for `capabilities`. */
|
|
3507
3791
|
capabilities?: string[];
|
|
3508
|
-
/** The line reeval prints
|
|
3792
|
+
/** The line reeval prints after `⚠ Warning:`, in red for root and orange for capabilities. */
|
|
3509
3793
|
message: string;
|
|
3510
3794
|
};
|
|
3511
3795
|
/** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
|
|
@@ -3516,5 +3800,5 @@ type ResolvedGateway = {
|
|
|
3516
3800
|
why: string;
|
|
3517
3801
|
}>;
|
|
3518
3802
|
};
|
|
3519
|
-
export {
|
|
3520
|
-
//# sourceMappingURL=types-
|
|
3803
|
+
export { EvalFn as $, directional as $n, Secret as $t, CellRole as A, HistoryEntry as An, JudgeState as At, Diff as B, EFFORT_MIN_RUNS as Bn, PinnableKey as Bt, CaptureUseAction as C, FamilyLift as Cn, JSONValue as Ct, CaseFn as D, FamilyVerdict as Dn, JudgeInputs as Dt, CaptureWindow as E, FamilyRecord as En, JudgeFailure as Et, CompareCut as F, StackSummary as Fn, OpenRouterGatewayDescriptor as Ft, EgressPolicy as G, Spread as Gn, ProfileLimit as Gt, DirectoryOptions as H, EffortMetric as Hn, PluginSource as Ht, CompareOptions as I, UpstreamCount as In, Outcome as It, Environment as J, bothUnderATenth as Jn, QuestionOptions as Jt, EgressRecord as K, TOKENS_UNREPORTED as Kn, Question as Kt, ConfigHelpers as L, CellEffort as Ln, PairFactor as Lt, ClaudeCodeHarnessDescriptor as M, Lift as Mn, McpServer as Mt, CodexHarnessDescriptor as N, RunInvocation as Nn, Observation as Nt, CaseOptions as O, FileRef as On, JudgeModel as Ot, CompareAnswer as P, RunRecord as Pn, OpenCodeHarnessDescriptor as Pt, ErrorKind as Q, contrastEffort as Qn, SandboxWarning as Qt, ConfigJudgeOptions as R, DIRECTIONAL_MIN as Rn, PairVote as Rt, CaptureUse as S, EvalEntry as Sn, HostFile as St, CaptureWait as T, FamilyRate as Tn, JudgeCallOptions as Tt, DockerSandboxDescriptor as U, EffortSample as Un, Profile as Ut, DirectGatewayDescriptor as V, EffortContrast as Vn, PluginPart as Vt, Duration as W, EffortVerdict as Wn, ProfileDefaults as Wt, EnvironmentInput as X, cellEffort as Xn, ResolvedGateway as Xt, EnvironmentHelpers as Y, cellContrast as Yn, ReevalConfig as Yt, EnvironmentSpec as Z, cellSamples as Zn, SandboxDescriptor as Zt, CaptureInstruction as _, WebEventType as _n, HarnessData as _t, AttemptContext as a, TartSandboxDescriptor as an, effortVerdict as ar, FakeAnswer as at, CapturePage as b, CellRecord as bn, HarnessOption as bt, BlindPair as c, Turn as cn, formatRatio as cr, FakeSandboxDescriptor as ct, BoatSandboxDescriptor as d, VercelGatewayDescriptor as dn, FxHarnessDescriptor as dt, SecretTemplate as en, effortChangeLabel as er, EvalOptions as et, Budget as f, Verdict as fn, GatewayDescriptor as ft, CaptureFailedRequest as g, Wait as gn, GroupOptions as gt, CaptureConsoleMessage as h, Viewport as hn, GrokBuildHarnessDescriptor as ht, AttemptAnswer as i, StatusRange as in, effortSummary as ir, FakeAnalysisModel as it, ClaudeCodeEffort as j, JudgmentRecord as jn, JudgeSubject as jt, Cell as k, HistoryCell as kn, JudgeOptions as kt, BoatApiKey as l, TurnData as ln, formatSeconds as lr, FakeScriptStep as lt, CaptureCall as m, VertexPublisher as mn, GooseHarnessDescriptor as mt, AnalysisConfig as n, SetupStep as nn, effortOf as nr, ExecResult as nt, BackgroundHandle as o, TokenUsage as on, filesChanged as or, FakeHarnessDescriptor as ot, Capture as p, VertexGatewayDescriptor as pn, GatewayName as pt, EnvVars as q, VERDICT_TEXT as qn, QuestionInputs as qt, AntigravityHarnessDescriptor as r, SkillSource as rn, effortSample as rr, ExecRetry as rt, BlindAttempt as s, TranscriptEvent as sn, formatCount as sr, FakeJudgeModel as st, AgentHarnessDescriptor as t, ServeHandle as tn, effortLabel as tr, EvalOptionsData as tt, BoatMachineType as u, TurnStatus as un, tokenValues as ur, FakeVote as ut, CaptureKey as v, ArtifactRef as vn, HarnessDescriptor as vt, CaptureUseScript as w, FamilyLiftSummary as wn, JudgeArtifactRef as wt, CapturePageFile as x, CheckTally as xn, HistoryConfig as xt, CaptureOptions as y, AttemptRecord as yn, HarnessEdit as yt, CursorHarnessDescriptor as z, EFFORT_ALPHA as zn, PiHarnessDescriptor as zt };
|
|
3804
|
+
//# sourceMappingURL=types-DBwFsGem.d.ts.map
|