reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,189 +1,23 @@
|
|
|
1
1
|
import { r as isTimeout } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import {
|
|
3
|
-
import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
|
|
4
|
-
import { B as profileJudge, H as runsOf, M as harnessId, P as majorityOf, U as sandboxId, Y as mergeJudge, _t as servedEffortOf, ft as UNRECORDED_QUESTION_STATE, gt as resolvedEffortOf, ht as harnessEffort, z as plannedSandboxes } from "./config-ImY6AlRE.js";
|
|
5
|
-
import { l as judgeModelName } from "./judge-ByEUlzOM.js";
|
|
6
|
-
import { f as redactJsonText, h as redactSplitSecrets, m as redactSecrets } from "./redact-BEEGc37e.js";
|
|
7
|
-
import { i as formatDuration } from "./text-D--Q5KsA.js";
|
|
8
|
-
import { i as passRateOf, o as worstOutcome, r as gatePart, t as cellOutcome } from "./gate-BoDKeuJu.js";
|
|
9
|
-
import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-D-Gqwqt0.js";
|
|
2
|
+
import { i as formatDuration } from "./text-B8-W3Cvn.js";
|
|
10
3
|
import { i as sumUsage, r as promptTokens } from "./usage-DJiDnh2D.js";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
4
|
+
import { T as profileSandbox } from "./environment-DB8o3hGi.js";
|
|
5
|
+
import { t as evalSlug } from "./slug-IHh3f770.js";
|
|
6
|
+
import { c as sha256, n as attemptKey, o as recordFileName, t as attemptId } from "./ids-C_advdJA.js";
|
|
7
|
+
import { $ as runsOf, Mt as resolvedEffortOf, Nt as servedEffortOf, St as mergeJudge, U as majorityOf, V as harnessId, X as plannedSandboxes, Z as profileJudge, et as sandboxId, jt as harnessEffort, yt as UNRECORDED_QUESTION_STATE } from "./config-B9mB1kF5.js";
|
|
8
|
+
import { g as redactSplitSecrets, h as redactSecrets, p as redactJsonText } from "./redact-BhjtnjrU.js";
|
|
9
|
+
import { l as judgeModelName } from "./judge-COMXT1Hx.js";
|
|
10
|
+
import { a as liftInterval, m as round3, r as containsZero } from "./stats-C18W4xPX.js";
|
|
11
|
+
import { i as passRateOf, o as worstOutcome, r as gatePart, t as cellOutcome } from "./gate-CgYdaprG.js";
|
|
12
|
+
import { r as patchForJudge, t as collapseDependencies } from "./patch-DH4e_Gkd.js";
|
|
13
|
+
import { i as evalFile, o as sourcesPart } from "./eval-source-D3sbAzt-.js";
|
|
14
|
+
import { c as foldCases, i as checkTallies, o as discrimination } from "./cases-z9E6DCd5.js";
|
|
15
|
+
import { i as checkTries } from "./judgments-CzgpsTqv.js";
|
|
16
|
+
import { v as familiesOf, y as pooledPassRate } from "./suite-COglUhPd.js";
|
|
17
|
+
import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-BOLt8pJg.js";
|
|
18
|
+
import { a as isKeptAttempt, f as previousAttempt, i as isCompletedAttempt, p as readRetries } from "./resume-AW5oXzUc.js";
|
|
17
19
|
import { join } from "node:path";
|
|
18
20
|
import { readFile, stat } from "node:fs/promises";
|
|
19
|
-
// src/history/families.ts
|
|
20
|
-
const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
21
|
-
function sideOf(entry, cell) {
|
|
22
|
-
const counted = entry.attempts.filter((a) => a.cell === cell && isCounted(a));
|
|
23
|
-
if (!counted.length) return null;
|
|
24
|
-
return {
|
|
25
|
-
pass: counted.filter((a) => a.outcome === "pass").length,
|
|
26
|
-
counted: counted.length,
|
|
27
|
-
samples: counted.map(effortSample)
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
const clamp = (x, lo, hi) => Math.min(hi, Math.max(lo, x));
|
|
31
|
-
const interval = (lo, hi, method, bound) => ({
|
|
32
|
-
lo: round3(clamp(lo, ...bound)),
|
|
33
|
-
hi: round3(clamp(hi, ...bound)),
|
|
34
|
-
level: .95,
|
|
35
|
-
method
|
|
36
|
-
});
|
|
37
|
-
function rateOf(cell, sides) {
|
|
38
|
-
const pass = sides.reduce((s, x) => s + x.pass, 0);
|
|
39
|
-
const counted = sides.reduce((s, x) => s + x.counted, 0);
|
|
40
|
-
const single = sides.every((x) => x.counted === 1);
|
|
41
|
-
const rates = sides.map((x) => x.pass / x.counted);
|
|
42
|
-
const t = meanInterval(rates);
|
|
43
|
-
const w = wilson(pass, counted);
|
|
44
|
-
return {
|
|
45
|
-
cell,
|
|
46
|
-
cases: sides.length,
|
|
47
|
-
pass,
|
|
48
|
-
counted,
|
|
49
|
-
rate: round3(t.mean),
|
|
50
|
-
interval: single ? interval(w.low, w.high, "wilson", [0, 1]) : interval(t.low, t.high, "t-cases", [0, 1])
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
function metric(pairs) {
|
|
54
|
-
const differences = pairs.map(([v, c]) => v - c);
|
|
55
|
-
const control = median(pairs.map(([, c]) => c)) ?? 0;
|
|
56
|
-
const variant = median(pairs.map(([v]) => v)) ?? 0;
|
|
57
|
-
const test = signedRank(differences);
|
|
58
|
-
const verdict = differences.filter((d) => d !== 0).length < 6 ? "insufficient" : test === null || test.p >= .05 ? "noise" : test.sign < 0 ? "fewer" : "more";
|
|
59
|
-
return {
|
|
60
|
-
pairs: pairs.length,
|
|
61
|
-
ratio: control === 0 ? null : round3(variant / control),
|
|
62
|
-
p: test === null ? null : round3(test.p),
|
|
63
|
-
verdict
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
function effortOf(pairs) {
|
|
67
|
-
const per = (pick) => pairs.flatMap(([v, c]) => {
|
|
68
|
-
const vs = v.samples.map(pick);
|
|
69
|
-
const cs = c.samples.map(pick);
|
|
70
|
-
if (!vs.every((x) => x !== void 0) || !cs.every((x) => x !== void 0)) return [];
|
|
71
|
-
return [[median(vs) ?? 0, median(cs) ?? 0]];
|
|
72
|
-
});
|
|
73
|
-
const tokens = per((s) => s.tokens);
|
|
74
|
-
return {
|
|
75
|
-
tools: metric(per((s) => s.tools)),
|
|
76
|
-
agentMs: metric(per((s) => s.agentMs)),
|
|
77
|
-
...tokens.length ? { tokens: metric(tokens) } : {}
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
function liftOf$1(variant, control, cases, minLift) {
|
|
81
|
-
const pairs = cases.flatMap((e) => {
|
|
82
|
-
const v = sideOf(e, variant);
|
|
83
|
-
const c = sideOf(e, control);
|
|
84
|
-
return v && c ? [[v, c]] : [];
|
|
85
|
-
});
|
|
86
|
-
if (!pairs.length) return null;
|
|
87
|
-
const single = pairs.every(([v, c]) => v.counted === 1 && c.counted === 1);
|
|
88
|
-
const vRates = pairs.map(([v]) => v.pass / v.counted);
|
|
89
|
-
const cRates = pairs.map(([, c]) => c.pass / c.counted);
|
|
90
|
-
const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
91
|
-
let points;
|
|
92
|
-
let range;
|
|
93
|
-
let p;
|
|
94
|
-
let discordant;
|
|
95
|
-
let decided;
|
|
96
|
-
if (single) {
|
|
97
|
-
const counts = {
|
|
98
|
-
both: 0,
|
|
99
|
-
variantOnly: 0,
|
|
100
|
-
controlOnly: 0,
|
|
101
|
-
neither: 0
|
|
102
|
-
};
|
|
103
|
-
for (const [v, c] of pairs) if (v.pass && c.pass) counts.both++;
|
|
104
|
-
else if (v.pass) counts.variantOnly++;
|
|
105
|
-
else if (c.pass) counts.controlOnly++;
|
|
106
|
-
else counts.neither++;
|
|
107
|
-
const paired = pairedLiftInterval(counts);
|
|
108
|
-
points = paired.points;
|
|
109
|
-
range = interval(paired.low, paired.high, "newcombe-paired", [-1, 1]);
|
|
110
|
-
p = round3(mcnemar(counts.variantOnly, counts.controlOnly));
|
|
111
|
-
discordant = {
|
|
112
|
-
variantOnly: counts.variantOnly,
|
|
113
|
-
controlOnly: counts.controlOnly
|
|
114
|
-
};
|
|
115
|
-
decided = counts.variantOnly + counts.controlOnly >= 6;
|
|
116
|
-
} else {
|
|
117
|
-
const differences = pairs.map(([v, c]) => (v.pass * c.counted - c.pass * v.counted) / (v.counted * c.counted));
|
|
118
|
-
const t = meanInterval(differences);
|
|
119
|
-
points = t.mean;
|
|
120
|
-
range = interval(t.low, t.high, "t-cases", [-1, 1]);
|
|
121
|
-
const rank = signedRank(differences);
|
|
122
|
-
p = rank === null ? null : round3(rank.p);
|
|
123
|
-
decided = pairs.length >= 6;
|
|
124
|
-
}
|
|
125
|
-
const verdict = !decided ? "insufficient" : range.lo > 0 ? "lift" : range.hi < 0 ? "drop" : "noise";
|
|
126
|
-
return {
|
|
127
|
-
variant,
|
|
128
|
-
vs: control,
|
|
129
|
-
pairs: pairs.length,
|
|
130
|
-
variantRate: round3(mean(vRates)),
|
|
131
|
-
controlRate: round3(mean(cRates)),
|
|
132
|
-
...discordant ? { discordant } : {},
|
|
133
|
-
points: round3(points),
|
|
134
|
-
interval: range,
|
|
135
|
-
p,
|
|
136
|
-
withinNoise: containsZero(range.lo, range.hi),
|
|
137
|
-
verdict,
|
|
138
|
-
...minLift !== void 0 ? { minLift } : {},
|
|
139
|
-
outcome: minLift !== void 0 && range.lo < minLift ? "fail" : "pass",
|
|
140
|
-
effort: effortOf(pairs)
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
/**
|
|
144
|
-
* The run's pooled families, in the order their first case appears; `minLiftOf` gives a variant
|
|
145
|
-
* cell's `compare.minLift`.
|
|
146
|
-
*/
|
|
147
|
-
function familiesOf(evals, minLiftOf) {
|
|
148
|
-
const byId = /* @__PURE__ */ new Map();
|
|
149
|
-
for (const e of evals) {
|
|
150
|
-
if (!e.family || e.todo) continue;
|
|
151
|
-
byId.set(e.family.id, [...byId.get(e.family.id) ?? [], e]);
|
|
152
|
-
}
|
|
153
|
-
return [...byId.values()].map((members) => {
|
|
154
|
-
const cases = [...members].sort((a, b) => (a.family?.case ?? 0) - (b.family?.case ?? 0));
|
|
155
|
-
const first = cases[0];
|
|
156
|
-
const family = first.family;
|
|
157
|
-
const cellKeys = [];
|
|
158
|
-
for (const e of cases) for (const c of e.cells) if (!cellKeys.includes(c.key)) cellKeys.push(c.key);
|
|
159
|
-
const rates = cellKeys.flatMap((key) => {
|
|
160
|
-
const sides = cases.flatMap((e) => sideOf(e, key) ?? []);
|
|
161
|
-
return sides.length ? [rateOf(key, sides)] : [];
|
|
162
|
-
});
|
|
163
|
-
const lifts = [];
|
|
164
|
-
for (const key of cellKeys) {
|
|
165
|
-
const holder = cases.find((e) => e.cells.some((c) => c.key === key && c.role === "variant"));
|
|
166
|
-
const variant = holder?.cells.find((c) => c.key === key);
|
|
167
|
-
if (!holder || !variant) continue;
|
|
168
|
-
const control = holder.cells.find((c) => c.role === "control" && c.harness === variant.harness && c.sandbox === variant.sandbox);
|
|
169
|
-
if (!control) continue;
|
|
170
|
-
const minLift = cases.flatMap((e) => e.cells.filter((c) => c.key === key)).map(minLiftOf).find((m) => m !== void 0);
|
|
171
|
-
const lift = liftOf$1(key, control.key, cases, minLift);
|
|
172
|
-
if (lift) lifts.push(lift);
|
|
173
|
-
}
|
|
174
|
-
return {
|
|
175
|
-
id: family.id,
|
|
176
|
-
file: first.file,
|
|
177
|
-
group: first.group,
|
|
178
|
-
name: family.name,
|
|
179
|
-
cases: family.cases,
|
|
180
|
-
ran: cases.length,
|
|
181
|
-
rates,
|
|
182
|
-
lifts,
|
|
183
|
-
outcome: lifts.some((l) => l.outcome === "fail") ? "fail" : "pass"
|
|
184
|
-
};
|
|
185
|
-
});
|
|
186
|
-
}
|
|
187
21
|
// src/history/assemble.ts
|
|
188
22
|
/** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
|
|
189
23
|
const TURN_TEXT_LIMIT = 8192;
|
|
@@ -266,6 +100,15 @@ function mergeOutcome(e, record) {
|
|
|
266
100
|
*/
|
|
267
101
|
const withoutCaps = ({ maxConcurrent: _, startTimeout: __, gateway: ___, ...rest }) => rest;
|
|
268
102
|
const harnessModel = (h) => h.kind === "fake" ? null : h.model ?? null;
|
|
103
|
+
/**
|
|
104
|
+
* The harness's part of a fingerprint: its descriptor, less the operational limits, and less the
|
|
105
|
+
* fake harness's `failWith`, its stand-in for an outage, which a retry of the attempt outlives.
|
|
106
|
+
*/
|
|
107
|
+
const harnessPart = (harness) => {
|
|
108
|
+
if (!harness) return sha256("null");
|
|
109
|
+
const { failWith: _, ...measured } = withoutCaps(harness);
|
|
110
|
+
return sha256(JSON.stringify(measured));
|
|
111
|
+
};
|
|
269
112
|
/** A cell fingerprint: the hash of its parts, so each part can be compared on its own. */
|
|
270
113
|
const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
|
|
271
114
|
/** The gateway's part of a fingerprint: none on `direct`, so a cell without a gateway keeps its fingerprint. */
|
|
@@ -473,6 +316,8 @@ function judgeInputs(record, diff, diffPath) {
|
|
|
473
316
|
}
|
|
474
317
|
/** What the run's attempts have in common, gathered while their records are read. */
|
|
475
318
|
var Seen = class {
|
|
319
|
+
/** Each attempt's earlier infra errors, from staging's `retries.json`, by key. */
|
|
320
|
+
retries = /* @__PURE__ */ new Map();
|
|
476
321
|
sources = /* @__PURE__ */ new Map();
|
|
477
322
|
byEval = /* @__PURE__ */ new Map();
|
|
478
323
|
contexts = /* @__PURE__ */ new Map();
|
|
@@ -481,6 +326,8 @@ var Seen = class {
|
|
|
481
326
|
environments = /* @__PURE__ */ new Map();
|
|
482
327
|
harnesses = /* @__PURE__ */ new Map();
|
|
483
328
|
resolvedHarnesses = /* @__PURE__ */ new Map();
|
|
329
|
+
/** Every pair `where` left out of a pooled family, by family id, from its cases' plans. */
|
|
330
|
+
familyExcluded = /* @__PURE__ */ new Map();
|
|
484
331
|
/** Keys of the attempts kept from the previous record on resume. */
|
|
485
332
|
kept = /* @__PURE__ */ new Set();
|
|
486
333
|
/** Registry digests found by the preflight, by image id. */
|
|
@@ -550,18 +397,22 @@ var Seen = class {
|
|
|
550
397
|
tags: e.attempt?.tags ?? [],
|
|
551
398
|
...e.attempt?.plan.compare ? { compare: e.attempt.plan.compare } : {},
|
|
552
399
|
...e.attempt?.family ? { family: familyOf(e.attempt.family) } : {},
|
|
400
|
+
...e.attempt?.plan.excluded ? { excluded: e.attempt.plan.excluded } : {},
|
|
553
401
|
fingerprint: "",
|
|
554
402
|
sources,
|
|
555
403
|
...from.source ? { source: from.source } : {},
|
|
556
404
|
...from.revision !== void 0 ? { revision: from.revision } : {},
|
|
557
405
|
outcome: "skipped",
|
|
558
406
|
todo: e.todo === true,
|
|
407
|
+
...e.skip ? { skip: true } : {},
|
|
559
408
|
passRate: null,
|
|
560
409
|
cells: [],
|
|
561
410
|
attempts: []
|
|
562
411
|
};
|
|
563
412
|
this.byEval.set(e.id, entry);
|
|
564
413
|
}
|
|
414
|
+
const familyExcluded = e.attempt?.plan.familyExcluded;
|
|
415
|
+
if (entry.family && familyExcluded) this.familyExcluded.set(entry.family.id, familyExcluded);
|
|
565
416
|
return entry;
|
|
566
417
|
}
|
|
567
418
|
};
|
|
@@ -578,6 +429,14 @@ const UNKNOWN_CELL = {
|
|
|
578
429
|
sandbox: "unknown",
|
|
579
430
|
role: "control"
|
|
580
431
|
};
|
|
432
|
+
/** A cell known only by its key, `<harness>/<environment>/<sandbox>`. */
|
|
433
|
+
const cellOfKey = (key) => ({
|
|
434
|
+
...UNKNOWN_CELL,
|
|
435
|
+
key,
|
|
436
|
+
harness: key.split("/")[0] ?? "unknown",
|
|
437
|
+
environment: key.split("/")[1] ?? "unknown",
|
|
438
|
+
sandbox: key.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
|
|
439
|
+
});
|
|
581
440
|
/** minLift per cell, kept aside while attempts are added; see `rollUp`. */
|
|
582
441
|
const minLifts = /* @__PURE__ */ new WeakMap();
|
|
583
442
|
/** Cells whose fingerprint came from an attempt that ran, not one that was skipped. */
|
|
@@ -635,13 +494,7 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
|
|
|
635
494
|
sandbox: beforeCell.sandbox,
|
|
636
495
|
role: beforeCell.role,
|
|
637
496
|
...beforeCell.variant !== void 0 ? { variant: beforeCell.variant } : {}
|
|
638
|
-
}) ??
|
|
639
|
-
...UNKNOWN_CELL,
|
|
640
|
-
key: kept.cell,
|
|
641
|
-
harness: kept.cell.split("/")[0] ?? "unknown",
|
|
642
|
-
environment: kept.cell.split("/")[1] ?? "unknown",
|
|
643
|
-
sandbox: kept.cell.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
|
|
644
|
-
};
|
|
497
|
+
}) ?? cellOfKey(kept.cell);
|
|
645
498
|
const sources = before?.sources ?? [];
|
|
646
499
|
const { source, revision } = e.attempt ?? before ?? {};
|
|
647
500
|
const stub = {
|
|
@@ -655,7 +508,9 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
|
|
|
655
508
|
durationMs: kept.durationMs,
|
|
656
509
|
errors: [],
|
|
657
510
|
formattedErrors: [],
|
|
658
|
-
meta: void 0
|
|
511
|
+
meta: void 0,
|
|
512
|
+
...before?.todo ? { todo: true } : {},
|
|
513
|
+
...before?.skip ? { skip: true } : {}
|
|
659
514
|
};
|
|
660
515
|
seen.add(null, cell, stub);
|
|
661
516
|
seen.addKept(previous, kept, cell, beforeCell?.environmentDigest ?? null, e.attempt !== void 0);
|
|
@@ -663,6 +518,7 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
|
|
|
663
518
|
source,
|
|
664
519
|
revision
|
|
665
520
|
});
|
|
521
|
+
if (!e.attempt && !entry.excluded && before?.excluded) entry.excluded = before.excluded;
|
|
666
522
|
const created = !entry.cells.some((c) => c.key === cell.key);
|
|
667
523
|
const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? beforeCell?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway, e.attempt ? e.attempt.bestOf : beforeCell?.bestOf);
|
|
668
524
|
if (created && beforeCell && !e.attempt) {
|
|
@@ -732,7 +588,7 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
|
|
|
732
588
|
} else {
|
|
733
589
|
const parts = {
|
|
734
590
|
sources: sourcesPart(source, revision, sources),
|
|
735
|
-
harness:
|
|
591
|
+
harness: harnessPart(e.attempt?.harness),
|
|
736
592
|
environment: cellRecord.environmentDigest,
|
|
737
593
|
image: kept.imageId,
|
|
738
594
|
agentCli: null,
|
|
@@ -745,16 +601,22 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousA
|
|
|
745
601
|
cellRecord.fingerprint = fingerprintOf(parts);
|
|
746
602
|
}
|
|
747
603
|
}
|
|
604
|
+
/** The earlier infra errors of an attempt that ran again, less the try its record still is. */
|
|
605
|
+
function retriesOf(retries, record) {
|
|
606
|
+
const earlier = (retries ?? []).filter((r) => !(record?.outcome === "infra_error" && r.at === (record.startedAt ?? null) && r.error.message === record.error?.message));
|
|
607
|
+
return earlier.length ? { retries: earlier } : {};
|
|
608
|
+
}
|
|
748
609
|
/** Reads one attempt's worker record, files its artifacts and adds it to its eval and cell. */
|
|
749
|
-
async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts) {
|
|
750
|
-
const cell = e.attempt?.cell ?? UNKNOWN_CELL;
|
|
751
|
-
const n = e.attempt?.n ?? 1;
|
|
610
|
+
async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts, unplanned) {
|
|
611
|
+
const cell = e.attempt?.cell ?? unplanned?.cell ?? UNKNOWN_CELL;
|
|
612
|
+
const n = e.attempt?.n ?? unplanned?.n ?? 1;
|
|
752
613
|
const kept = previousAttempt(previous, e.key);
|
|
753
|
-
|
|
614
|
+
const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
|
|
615
|
+
const rerun = kept?.outcome === "infra_error" && !!record && isKeptAttempt(record.outcome, record.error);
|
|
616
|
+
if (previous && e.state === "skipped" && kept && !rerun && isCompletedAttempt(kept.outcome, kept.error)) {
|
|
754
617
|
await addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts);
|
|
755
618
|
return;
|
|
756
619
|
}
|
|
757
|
-
const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
|
|
758
620
|
const { outcome, error } = mergeOutcome(e, record);
|
|
759
621
|
if (!seen.sources.has(e.file)) seen.sources.set(e.file, await evalFile(ctx.config.root, e.file));
|
|
760
622
|
const sources = seen.sources.get(e.file) ?? [];
|
|
@@ -776,9 +638,9 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
776
638
|
if (record) measured.add(cellRecord);
|
|
777
639
|
const parts = {
|
|
778
640
|
sources: sourcesPart(source, revision, sources),
|
|
779
|
-
harness:
|
|
641
|
+
harness: harnessPart(e.attempt?.harness),
|
|
780
642
|
environment: environment?.digest ?? null,
|
|
781
|
-
image: record?.image?.id ?? null,
|
|
643
|
+
image: record?.image?.id ?? record?.vmImage?.digest ?? null,
|
|
782
644
|
agentCli: record?.image?.agentCli ?? null,
|
|
783
645
|
judge,
|
|
784
646
|
context: sha256(JSON.stringify(context)),
|
|
@@ -796,6 +658,7 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
796
658
|
seq: null,
|
|
797
659
|
startedAt: record?.startedAt ?? null,
|
|
798
660
|
imageId: record?.image?.id ?? null,
|
|
661
|
+
...record?.vmImage ? { vmImage: record.vmImage } : {},
|
|
799
662
|
env: record?.env ?? {
|
|
800
663
|
vars: [],
|
|
801
664
|
secrets: [],
|
|
@@ -812,6 +675,7 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
812
675
|
outcome,
|
|
813
676
|
error,
|
|
814
677
|
durationMs: e.durationMs,
|
|
678
|
+
...retriesOf(seen.retries.get(e.key), record),
|
|
815
679
|
...await attemptEvidence(record, `${entry.slug}/${evalSlug(cell.key)}/${n}`, out, join(ctx.stagingDir, "evals", recordFileName(e.key)))
|
|
816
680
|
});
|
|
817
681
|
}
|
|
@@ -926,6 +790,7 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
926
790
|
...snapshots.length ? { snapshots } : {},
|
|
927
791
|
...record?.serves?.length ? { serves: record.serves } : {},
|
|
928
792
|
...record?.assertions?.length ? { assertions: record.assertions } : {},
|
|
793
|
+
...record?.observations?.length ? { observations: record.observations } : {},
|
|
929
794
|
checks: (record?.checks ?? []).map((c, i) => ({
|
|
930
795
|
command: c.command,
|
|
931
796
|
exitCode: c.exitCode,
|
|
@@ -978,45 +843,6 @@ function liftOf(control, variant, pooled) {
|
|
|
978
843
|
};
|
|
979
844
|
}
|
|
980
845
|
/**
|
|
981
|
-
* Each cell's checks: every label any attempt of the eval recorded, tallied over the cell's
|
|
982
|
-
* counted attempts (pass or fail). An attempt that counted without reaching a label, because
|
|
983
|
-
* an `expect.require` or a thrown error before it ended the body, counts it `notRun`.
|
|
984
|
-
*/
|
|
985
|
-
function checkTallies(entry) {
|
|
986
|
-
const labels = [...new Set(entry.attempts.flatMap((a) => (a.assertions ?? []).map((x) => x.label)))];
|
|
987
|
-
if (!labels.length) return;
|
|
988
|
-
for (const c of entry.cells) {
|
|
989
|
-
const counted = entry.attempts.filter((a) => a.cell === c.key && (a.outcome === "pass" || a.outcome === "fail"));
|
|
990
|
-
if (!counted.length) continue;
|
|
991
|
-
c.checks = labels.map((label) => {
|
|
992
|
-
const tally = {
|
|
993
|
-
label,
|
|
994
|
-
pass: 0,
|
|
995
|
-
fail: 0,
|
|
996
|
-
error: 0,
|
|
997
|
-
notRun: 0
|
|
998
|
-
};
|
|
999
|
-
for (const a of counted) {
|
|
1000
|
-
const status = a.assertions?.find((x) => x.label === label)?.status;
|
|
1001
|
-
if (status === void 0) tally.notRun++;
|
|
1002
|
-
else tally[status]++;
|
|
1003
|
-
}
|
|
1004
|
-
return tally;
|
|
1005
|
-
});
|
|
1006
|
-
}
|
|
1007
|
-
}
|
|
1008
|
-
/** Puts the control's tally beside each of the variant's checks, with whether it discriminates. */
|
|
1009
|
-
function discrimination(control, variant) {
|
|
1010
|
-
for (const check of variant.checks ?? []) {
|
|
1011
|
-
const k = control.checks?.find((x) => x.label === check.label);
|
|
1012
|
-
if (!k) continue;
|
|
1013
|
-
const { label: _label, ...tally } = k;
|
|
1014
|
-
check.control = tally;
|
|
1015
|
-
const verdict = checkVerdict(check, k);
|
|
1016
|
-
if (verdict) check.verdict = verdict;
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
/**
|
|
1020
846
|
* Marks a best-of cell's attempts past its majority's first: `added` for one that ran, `skipped`
|
|
1021
847
|
* for one the majority before it left unrun.
|
|
1022
848
|
*/
|
|
@@ -1034,6 +860,8 @@ function markBestOf(cell, attempts) {
|
|
|
1034
860
|
if (a.outcome === "fail") fail++;
|
|
1035
861
|
}
|
|
1036
862
|
}
|
|
863
|
+
/** A case of an eval with several cases, which pools its gates over them. */
|
|
864
|
+
const pooled = (entry) => (entry.family?.cases ?? 0) >= 2;
|
|
1037
865
|
/** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
|
|
1038
866
|
function rollUp(evals) {
|
|
1039
867
|
for (const entry of evals) {
|
|
@@ -1053,12 +881,14 @@ function rollUp(evals) {
|
|
|
1053
881
|
if (c.role !== "variant") continue;
|
|
1054
882
|
const control = entry.cells.find((x) => x.role === "control" && x.harness === c.harness && x.sandbox === c.sandbox);
|
|
1055
883
|
if (control) discrimination(control, c);
|
|
1056
|
-
const lift = control && liftOf(control, c, entry
|
|
884
|
+
const lift = control && liftOf(control, c, pooled(entry));
|
|
1057
885
|
if (!lift) continue;
|
|
1058
886
|
c.lift = lift;
|
|
1059
887
|
if (lift.minLift !== void 0 && lift.interval.lo < lift.minLift && c.outcome === "pass") c.outcome = "fail";
|
|
1060
888
|
}
|
|
1061
889
|
entry.outcome = worstOutcome(entry.cells.map((c) => c.gate === "none" && c.outcome === "fail" ? "pass" : c.outcome));
|
|
890
|
+
if (pooled(entry) && entry.cells.some((c) => pooledPassRate(c) !== void 0)) entry.verdict = "family";
|
|
891
|
+
else delete entry.verdict;
|
|
1062
892
|
const gated = new Set(entry.cells.filter((c) => c.gate !== "none").map((c) => c.key));
|
|
1063
893
|
entry.passRate = passRateOf(entry.attempts.filter((a) => gated.has(a.cell)).map((a) => a.outcome));
|
|
1064
894
|
entry.fingerprint = sha256(JSON.stringify(entry.cells.map((c) => [c.key, c.fingerprint]).sort()));
|
|
@@ -1091,6 +921,15 @@ function gitState(input, out) {
|
|
|
1091
921
|
}
|
|
1092
922
|
return state;
|
|
1093
923
|
}
|
|
924
|
+
/** A retried run's git state as it ran, with its `worktree.patch` carried over byte-exact. */
|
|
925
|
+
async function keptGit(previous, from, out) {
|
|
926
|
+
const ref = previous.git?.worktreePatch;
|
|
927
|
+
if (ref && from) {
|
|
928
|
+
const bytes = await readFile(join(from, ...ref.path.split("/"))).catch(() => null);
|
|
929
|
+
if (bytes) out.files.set(ref.path, bytes);
|
|
930
|
+
}
|
|
931
|
+
return previous.git;
|
|
932
|
+
}
|
|
1094
933
|
const byName = (key) => (a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0;
|
|
1095
934
|
function stackOf(input, seen, judgeModel, threshold) {
|
|
1096
935
|
const packages = packageVersions(input.enginePackages);
|
|
@@ -1121,6 +960,19 @@ function stackOf(input, seen, judgeModel, threshold) {
|
|
|
1121
960
|
fingerprintScope: FINGERPRINT_SCOPE
|
|
1122
961
|
};
|
|
1123
962
|
}
|
|
963
|
+
/** The judge's part of every cell's fingerprint, and the judge model's name (null for none). */
|
|
964
|
+
function judgePart(config, profile, reeval, enginePackages) {
|
|
965
|
+
const runJudge = profile ? mergeJudge(config.judge, profileJudge(profile)) : config.judge;
|
|
966
|
+
const model = runJudge.model === void 0 ? null : judgeModelName(runJudge.model);
|
|
967
|
+
return {
|
|
968
|
+
part: sha256(JSON.stringify({
|
|
969
|
+
model,
|
|
970
|
+
threshold: runJudge.threshold,
|
|
971
|
+
version: judgeVersion(reeval, packageVersions(enginePackages), model ?? "")
|
|
972
|
+
})),
|
|
973
|
+
model
|
|
974
|
+
};
|
|
975
|
+
}
|
|
1124
976
|
async function assembleRun(input) {
|
|
1125
977
|
const { ctx, engine } = input;
|
|
1126
978
|
const { config, runId, profile } = ctx;
|
|
@@ -1131,18 +983,15 @@ async function assembleRun(input) {
|
|
|
1131
983
|
include: config.include,
|
|
1132
984
|
harness: config.harness.map(withoutCaps),
|
|
1133
985
|
sandbox: withoutCaps(config.sandbox),
|
|
1134
|
-
environment: config.
|
|
986
|
+
environment: config.environments.length > 1 ? config.environments : config.environments[0] ?? null,
|
|
1135
987
|
runs: config.runs,
|
|
1136
988
|
env: config.env,
|
|
1137
989
|
judge: withoutCaps(config.judge),
|
|
1138
990
|
timeoutMs: config.timeoutMs
|
|
1139
991
|
}));
|
|
1140
992
|
const seen = new Seen(input.sandbox);
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
threshold: runJudge.threshold,
|
|
1144
|
-
version: judgeVersion(input.reeval, packageVersions(input.enginePackages), judgeModel ?? "")
|
|
1145
|
-
}));
|
|
993
|
+
seen.retries = readRetries(ctx.stagingDir);
|
|
994
|
+
const { part: judge } = judgePart(config, profile, input.reeval, input.enginePackages);
|
|
1146
995
|
for (const e of engine.evals) await addAttempt(e, ctx, seen, out, judge, input.previous, input.previousArtifacts);
|
|
1147
996
|
if (input.previous) {
|
|
1148
997
|
const have = new Set([...seen.byEval.values()].flatMap((entry) => entry.attempts.map((a) => a.key)));
|
|
@@ -1163,13 +1012,37 @@ async function assembleRun(input) {
|
|
|
1163
1012
|
}, a, seen, out, judge, input.previous, input.previousArtifacts);
|
|
1164
1013
|
}
|
|
1165
1014
|
}
|
|
1015
|
+
const added = new Set([...seen.byEval.values()].flatMap((e) => e.attempts.map((a) => a.key)));
|
|
1016
|
+
for (const l of input.leftOut ?? []) {
|
|
1017
|
+
if (added.has(l.key)) continue;
|
|
1018
|
+
const [, ...names] = l.evalId.split(" › ");
|
|
1019
|
+
const stub = {
|
|
1020
|
+
id: l.evalId,
|
|
1021
|
+
key: l.key,
|
|
1022
|
+
file: l.file,
|
|
1023
|
+
group: names.slice(0, -1),
|
|
1024
|
+
name: l.name,
|
|
1025
|
+
attempt: void 0,
|
|
1026
|
+
state: "skipped",
|
|
1027
|
+
durationMs: 0,
|
|
1028
|
+
errors: [],
|
|
1029
|
+
formattedErrors: [],
|
|
1030
|
+
meta: void 0
|
|
1031
|
+
};
|
|
1032
|
+
const was = previousAttempt(input.previous, l.key);
|
|
1033
|
+
if (input.previous && was) await addPreviousAttempt(stub, was, seen, out, judge, input.previous, input.previousArtifacts);
|
|
1034
|
+
else await addAttempt(stub, ctx, seen, out, judge, null, null, {
|
|
1035
|
+
cell: cellOfKey(l.cell),
|
|
1036
|
+
n: l.n
|
|
1037
|
+
});
|
|
1038
|
+
}
|
|
1166
1039
|
const evals = [...seen.byEval.values()];
|
|
1167
1040
|
for (const entry of evals) {
|
|
1168
1041
|
const before = input.previous?.evals.find((p) => p.id === entry.id)?.family;
|
|
1169
1042
|
if (!entry.family && before) entry.family = before;
|
|
1170
1043
|
}
|
|
1171
1044
|
rollUp(evals);
|
|
1172
|
-
const families = familiesOf(evals, (cell) => minLifts.get(cell));
|
|
1045
|
+
const families = familiesOf(evals, (cell) => minLifts.get(cell), (id) => seen.familyExcluded.get(id) ?? input.previous?.families?.find((f) => f.id === id)?.excluded);
|
|
1173
1046
|
sequence(evals);
|
|
1174
1047
|
for (const e of evals) {
|
|
1175
1048
|
const asked = input.answers?.evals.get(e.id);
|
|
@@ -1191,73 +1064,87 @@ async function assembleRun(input) {
|
|
|
1191
1064
|
const attempts = evals.flatMap((e) => e.attempts);
|
|
1192
1065
|
const count = (o) => attempts.filter((a) => a.outcome === o).length;
|
|
1193
1066
|
const notGated = evals.flatMap((e) => e.attempts.filter((a) => e.cells.find((c) => c.key === a.cell)?.gate === "none")).filter((a) => a.outcome === "fail").length;
|
|
1067
|
+
const retried = input.retry && input.previous ? input.previous : null;
|
|
1068
|
+
const stack = stackOf(input, seen, judgeModel, runJudge.threshold);
|
|
1194
1069
|
const ids = (list) => list === void 0 ? void 0 : [list].flat().map(harnessId);
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
environment: profile.defaults.environment?.name,
|
|
1216
|
-
sandbox: (profile.defaults.environment || profile.defaults.sandbox) && sandboxId(profileSandbox(config, profile.defaults)),
|
|
1217
|
-
...profile.defaults.runs !== void 0 ? runsOf(profile.defaults.runs) : {},
|
|
1218
|
-
timeout: profile.defaults.timeout,
|
|
1219
|
-
env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
|
|
1220
|
-
})),
|
|
1221
|
-
limit: profile.limit
|
|
1222
|
-
} : null,
|
|
1223
|
-
plan: {
|
|
1224
|
-
evals: input.plan.evals,
|
|
1225
|
-
cells: input.plan.cells,
|
|
1226
|
-
attempts: input.plan.attempts,
|
|
1227
|
-
estimateMs: Math.round(input.plan.estimateMs),
|
|
1228
|
-
sandboxWarnings: input.plan.sandboxWarnings
|
|
1229
|
-
},
|
|
1230
|
-
budget: Object.keys(input.plan.budget).length ? input.plan.budget : null,
|
|
1231
|
-
config: {
|
|
1232
|
-
sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
|
|
1233
|
-
environment: config.environment?.name ?? null,
|
|
1234
|
-
...config.runs !== void 0 ? runsOf(config.runs) : { runs: 1 },
|
|
1235
|
-
judgeModel,
|
|
1236
|
-
threshold: runJudge.threshold,
|
|
1237
|
-
timeoutMs: config.timeoutMs,
|
|
1238
|
-
concurrency: input.ctx.concurrency,
|
|
1239
|
-
configHash
|
|
1240
|
-
},
|
|
1241
|
-
cost: costOf(attempts),
|
|
1242
|
-
artifacts: "all",
|
|
1243
|
-
redaction: { version: 1 },
|
|
1244
|
-
summary: {
|
|
1245
|
-
evals: evals.length,
|
|
1246
|
-
attempts: attempts.length,
|
|
1247
|
-
pass: count("pass"),
|
|
1248
|
-
fail: count("fail"),
|
|
1249
|
-
notGated,
|
|
1250
|
-
infraError: count("infra_error"),
|
|
1251
|
-
skipped: count("skipped")
|
|
1070
|
+
const record = {
|
|
1071
|
+
kind: "reeval-run",
|
|
1072
|
+
schemaVersion: 1,
|
|
1073
|
+
runId,
|
|
1074
|
+
status: input.interrupted ? "interrupted" : input.budgetReached ? "budget" : input.leftOut?.some((l) => !l.done) ? input.previous?.status === "budget" ? "budget" : "interrupted" : "complete",
|
|
1075
|
+
startedAt: input.startedAt.toISOString(),
|
|
1076
|
+
endedAt: input.endedAt.toISOString(),
|
|
1077
|
+
durationMs: retried ? retried.durationMs + input.endedAt.getTime() - (input.retry?.startedAt ?? input.endedAt).getTime() : input.endedAt.getTime() - input.startedAt.getTime(),
|
|
1078
|
+
...retried && input.retry ? { retriedAt: [...retried.retriedAt ?? [], ...!input.retry.resume ? [input.retry.startedAt.toISOString()] : input.retry.began ? [input.retry.began.toISOString()] : []] } : {},
|
|
1079
|
+
author: input.author,
|
|
1080
|
+
git: retried ? await keptGit(retried, input.previousArtifacts, out) : gitState(input, out),
|
|
1081
|
+
runner: input.runner,
|
|
1082
|
+
dependencies: retried ? retried.dependencies : { lockfiles: input.lockfiles },
|
|
1083
|
+
stack: retried?.stack ? {
|
|
1084
|
+
...retried.stack,
|
|
1085
|
+
images: stack.images,
|
|
1086
|
+
harnesses: stack.harnesses,
|
|
1087
|
+
judge: {
|
|
1088
|
+
...retried.stack.judge,
|
|
1089
|
+
resolvedModels: stack.judge.resolvedModels
|
|
1252
1090
|
},
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1091
|
+
context: stack.context
|
|
1092
|
+
} : stack,
|
|
1093
|
+
environments: [...seen.environments.values()].sort(byName((e) => e.name)),
|
|
1094
|
+
profile: profile ? {
|
|
1095
|
+
name: profile.name,
|
|
1096
|
+
reason: profile.reason,
|
|
1097
|
+
defaults: JSON.parse(JSON.stringify({
|
|
1098
|
+
harness: ids(profile.defaults.harnesses),
|
|
1099
|
+
environment: profile.defaults.environments && [profile.defaults.environments].flat().map((e) => e.name).join(", "),
|
|
1100
|
+
sandbox: (profile.defaults.environments || profile.defaults.sandbox) && sandboxId(profileSandbox(config, profile.defaults)),
|
|
1101
|
+
...profile.defaults.runs !== void 0 ? runsOf(profile.defaults.runs) : {},
|
|
1102
|
+
timeout: profile.defaults.timeout,
|
|
1103
|
+
env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
|
|
1104
|
+
})),
|
|
1105
|
+
limit: profile.limit
|
|
1106
|
+
} : null,
|
|
1107
|
+
...input.invocation ? { invocation: input.invocation } : {},
|
|
1108
|
+
plan: retried ? retried.plan : {
|
|
1109
|
+
evals: input.plan.evals,
|
|
1110
|
+
cells: input.plan.cells,
|
|
1111
|
+
attempts: input.plan.attempts,
|
|
1112
|
+
estimateMs: Math.round(input.plan.estimateMs),
|
|
1113
|
+
sandboxWarnings: input.plan.sandboxWarnings
|
|
1256
1114
|
},
|
|
1115
|
+
budget: Object.keys(input.plan.budget).length ? input.plan.budget : null,
|
|
1116
|
+
config: {
|
|
1117
|
+
sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
|
|
1118
|
+
environment: config.environments.map((e) => e.name).join(", ") || null,
|
|
1119
|
+
...config.runs !== void 0 ? runsOf(config.runs) : { runs: 1 },
|
|
1120
|
+
judgeModel,
|
|
1121
|
+
threshold: runJudge.threshold,
|
|
1122
|
+
timeoutMs: config.timeoutMs,
|
|
1123
|
+
concurrency: input.ctx.concurrency,
|
|
1124
|
+
configHash
|
|
1125
|
+
},
|
|
1126
|
+
cost: costOf(attempts),
|
|
1127
|
+
artifacts: "all",
|
|
1128
|
+
redaction: { version: 1 },
|
|
1129
|
+
summary: {
|
|
1130
|
+
evals: evals.length,
|
|
1131
|
+
attempts: attempts.length,
|
|
1132
|
+
pass: count("pass"),
|
|
1133
|
+
fail: count("fail"),
|
|
1134
|
+
notGated,
|
|
1135
|
+
infraError: count("infra_error"),
|
|
1136
|
+
skipped: count("skipped")
|
|
1137
|
+
},
|
|
1138
|
+
evals,
|
|
1139
|
+
...families.length ? { families } : {},
|
|
1140
|
+
...engine.unhandledErrors.length ? { errors: engine.unhandledErrors.map((e) => redactSecrets(e)) } : {}
|
|
1141
|
+
};
|
|
1142
|
+
return {
|
|
1143
|
+
record: foldCases(record),
|
|
1257
1144
|
artifacts: out.files,
|
|
1258
1145
|
moves: out.moves
|
|
1259
1146
|
};
|
|
1260
1147
|
}
|
|
1261
|
-
export { gatewayPart as a,
|
|
1148
|
+
export { gatewayPart as a, judgmentKey as c, transcriptLines as d, turnError as f, fingerprintOf as i, mergeOutcome as l, assembleRun as n, harnessPart as o, turnText as p, capText as r, judgePart as s, OUTPUT_CAP as t, rawLines as u };
|
|
1262
1149
|
|
|
1263
|
-
//# sourceMappingURL=assemble-
|
|
1150
|
+
//# sourceMappingURL=assemble-t6h018An.js.map
|