reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { a as plural } from "./text-B8-W3Cvn.js";
|
|
2
|
+
import { n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
|
|
1
3
|
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
|
|
2
|
-
import {
|
|
3
|
-
import { a as rescore,
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { u as readableTranscriptLine } from "./record-
|
|
4
|
+
import { l as sampleReasons } from "./pairs-BGb134Vu.js";
|
|
5
|
+
import { a as rescore, n as gateLabel } from "./gate-CgYdaprG.js";
|
|
6
|
+
import { T as cellEffort, w as cellContrast } from "./cases-z9E6DCd5.js";
|
|
7
|
+
import { m as uniqueJudgeLabels, n as cellUsage, t as cellJudgeRates } from "./judgments-CzgpsTqv.js";
|
|
8
|
+
import { u as readableTranscriptLine } from "./record-6IUzwP9K.js";
|
|
7
9
|
// src/view/compare.ts
|
|
8
10
|
const STACK_FIELDS = [
|
|
9
11
|
{
|
|
@@ -223,7 +225,7 @@ function canceledSides(outcomes, cells) {
|
|
|
223
225
|
const sides = ["base", "head"].filter((s) => canceledOn(outcomes[s], cells[s] ?? []));
|
|
224
226
|
return sides.length ? { canceled: sides } : {};
|
|
225
227
|
}
|
|
226
|
-
/** An eval only head has. */
|
|
228
|
+
/** An eval only head has, with each of its cases. */
|
|
227
229
|
const addedRow = (h) => ({
|
|
228
230
|
id: h.id,
|
|
229
231
|
base: null,
|
|
@@ -232,8 +234,41 @@ const addedRow = (h) => ({
|
|
|
232
234
|
...canceledSides({
|
|
233
235
|
base: null,
|
|
234
236
|
head: h.outcome
|
|
235
|
-
}, { head: h.cells })
|
|
237
|
+
}, { head: h.cells }),
|
|
238
|
+
...verdictOf(h),
|
|
239
|
+
...h.cases ? { cases: caseEvals(h).map(addedRow) } : {},
|
|
240
|
+
...casesGate(null, h)
|
|
236
241
|
});
|
|
242
|
+
/** An eval only base has, with each of its cases. */
|
|
243
|
+
const removedRow = (b) => ({
|
|
244
|
+
id: b.id,
|
|
245
|
+
base: b.outcome,
|
|
246
|
+
head: null,
|
|
247
|
+
status: "removed",
|
|
248
|
+
...canceledSides({
|
|
249
|
+
base: b.outcome,
|
|
250
|
+
head: null
|
|
251
|
+
}, { base: b.cells }),
|
|
252
|
+
...verdictOf(b),
|
|
253
|
+
...b.cases ? { cases: caseEvals(b).map(removedRow) } : {},
|
|
254
|
+
...casesGate(b, null)
|
|
255
|
+
});
|
|
256
|
+
/** A case its eval's rate gates, on the side that decides: head's, or base's when only it ran. */
|
|
257
|
+
const verdictOf = (e) => e?.verdict ? { verdict: e.verdict } : {};
|
|
258
|
+
/** An eval's cases, each as an eval of its own keyed by the case key. */
|
|
259
|
+
const caseEvals = (e) => (e.cases ?? []).map((k) => ({
|
|
260
|
+
id: k.key,
|
|
261
|
+
fingerprint: k.fingerprint,
|
|
262
|
+
outcome: k.outcome,
|
|
263
|
+
...k.verdict ? { verdict: k.verdict } : {},
|
|
264
|
+
sources: e.sources,
|
|
265
|
+
...e.source ? { source: {
|
|
266
|
+
...e.source,
|
|
267
|
+
...k.hash ? { case: k.hash } : {}
|
|
268
|
+
} } : {},
|
|
269
|
+
...e.revision !== void 0 ? { revision: e.revision } : {},
|
|
270
|
+
cells: k.cells
|
|
271
|
+
}));
|
|
237
272
|
/**
|
|
238
273
|
* When two runs of an eval differ only in which cells ran (a filtered run, a harness added), its
|
|
239
274
|
* outcome on the cells both have, and the cells only one has. Null when no cell is shared or a
|
|
@@ -276,13 +311,34 @@ function compareCells(base, head) {
|
|
|
276
311
|
}];
|
|
277
312
|
});
|
|
278
313
|
}
|
|
279
|
-
/**
|
|
280
|
-
|
|
314
|
+
/**
|
|
315
|
+
* An eval whose cases pool a gate: some lift has a `minLift`, or some cell's rate over the cases
|
|
316
|
+
* a `passRate`. Its outcome is then over the cases that ran, so a run of other cases can't be
|
|
317
|
+
* compared with it.
|
|
318
|
+
*/
|
|
319
|
+
const pooledGate = (e) => !!e && ((e.lifts ?? []).some((l) => l.minLift !== void 0) || !!e.rates?.length);
|
|
281
320
|
const liftSide = ({ variant: _, vs: __, ...side }) => side;
|
|
282
|
-
/**
|
|
283
|
-
|
|
321
|
+
/**
|
|
322
|
+
* When both runs pooled lifts over different variants: the variants only head or only base has,
|
|
323
|
+
* and those both have, over which the eval's outcome is then read.
|
|
324
|
+
*/
|
|
325
|
+
function liftScope(b, h) {
|
|
326
|
+
const of = (e) => (e.lifts ?? []).map((l) => l.variant);
|
|
327
|
+
const [base, head] = [of(b), of(h)];
|
|
328
|
+
if (!base.length || !head.length) return null;
|
|
329
|
+
const added = head.filter((v) => !base.includes(v));
|
|
330
|
+
const removed = base.filter((v) => !head.includes(v));
|
|
331
|
+
if (!added.length && !removed.length) return null;
|
|
332
|
+
return {
|
|
333
|
+
added,
|
|
334
|
+
removed,
|
|
335
|
+
shared: head.filter((v) => base.includes(v))
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/** Each variant's lift paired by case on both sides, head's order first. */
|
|
339
|
+
function casesLifts(b, h) {
|
|
284
340
|
const out = /* @__PURE__ */ new Map();
|
|
285
|
-
for (const [side,
|
|
341
|
+
for (const [side, e] of [["head", h], ["base", b]]) for (const l of e?.lifts ?? []) {
|
|
286
342
|
const row = out.get(l.variant) ?? {
|
|
287
343
|
key: l.variant,
|
|
288
344
|
vs: l.vs,
|
|
@@ -294,131 +350,142 @@ function familyLifts(b, h) {
|
|
|
294
350
|
}
|
|
295
351
|
return [...out.values()];
|
|
296
352
|
}
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
const
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
353
|
+
/** Each gated cell's rate over the cases on both sides, head's order first. */
|
|
354
|
+
function casesRates(b, h) {
|
|
355
|
+
const out = /* @__PURE__ */ new Map();
|
|
356
|
+
for (const [side, e] of [["head", h], ["base", b]]) for (const { cell, ...rate } of e?.rates ?? []) {
|
|
357
|
+
const row = out.get(cell) ?? {
|
|
358
|
+
key: cell,
|
|
359
|
+
base: null,
|
|
360
|
+
head: null
|
|
361
|
+
};
|
|
362
|
+
row[side] = rate;
|
|
363
|
+
out.set(cell, row);
|
|
364
|
+
}
|
|
365
|
+
return [...out.values()];
|
|
366
|
+
}
|
|
367
|
+
/** The lifts and gated rates of an eval with `cases`, on both sides; unset when it has none. */
|
|
368
|
+
function casesGate(b, h) {
|
|
369
|
+
const lifts = casesLifts(b, h);
|
|
370
|
+
const rates = casesRates(b, h);
|
|
371
|
+
return {
|
|
372
|
+
...lifts.length ? { lifts } : {},
|
|
373
|
+
...rates.length ? { rates } : {}
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
/** The rows of a run with nothing to compare it with: every eval is new. */
|
|
377
|
+
const newRows = (head) => head.evals.map(addedRow);
|
|
378
|
+
/** Each eval's row, an eval with `cases` with a row for each of its cases. */
|
|
379
|
+
function compare(base, head) {
|
|
380
|
+
const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
|
|
381
|
+
return evalRows(base.evals, head.evals, graderChanged);
|
|
382
|
+
}
|
|
383
|
+
/** A case's outcome as its eval's verdict reads it: a failure its eval's rate gates passes. */
|
|
384
|
+
const caseOutcome = (o, verdict) => verdict === "family" && o === "fail" ? "pass" : o;
|
|
309
385
|
/**
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
386
|
+
* An eval with `cases` in both runs. Over the same cases, its outcome is the eval's; over
|
|
387
|
+
* different ones, the worst of the cases both ran, as their eval reads them, unless its cases
|
|
388
|
+
* pool a gate, which a run of other cases can't be compared with. A case that changed or hit an
|
|
389
|
+
* infrastructure error makes the eval `drift` or `infra` too, and a case that regressed outside
|
|
390
|
+
* a pooled gate makes it `regressed`.
|
|
315
391
|
*/
|
|
316
|
-
function
|
|
317
|
-
|
|
318
|
-
const
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
lifts
|
|
392
|
+
function casesRow(b, h, graderChanged, row) {
|
|
393
|
+
const caseRows = evalRows(caseEvals(b), caseEvals(h), graderChanged);
|
|
394
|
+
const shared = caseRows.filter((r) => r.base !== null && r.head !== null);
|
|
395
|
+
const added = caseRows.filter((r) => r.base === null).map((r) => r.id);
|
|
396
|
+
const removed = caseRows.filter((r) => r.head === null).map((r) => r.id);
|
|
397
|
+
const out = {
|
|
398
|
+
...row,
|
|
399
|
+
cases: caseRows,
|
|
400
|
+
...casesGate(b, h)
|
|
326
401
|
};
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
402
|
+
const changes = [.../* @__PURE__ */ new Set([...row.changes ?? [], ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])])];
|
|
403
|
+
const failing = caseRows.filter((r) => r.base === null && caseOutcome(r.head, r.verdict) === "fail" && !r.canceled?.includes("head"));
|
|
404
|
+
const over = (side) => side === "head" ? [...shared, ...failing] : shared;
|
|
405
|
+
if (added.length || removed.length) {
|
|
406
|
+
if (pooledGate(b) || pooledGate(h) || !shared.length) changes.push(...added.map((k) => `case ${k} added`), ...removed.map((k) => `case ${k} removed`));
|
|
407
|
+
else {
|
|
408
|
+
const worst = (side) => outcomeOf(over(side).map((r) => ({
|
|
409
|
+
gate: "all",
|
|
410
|
+
outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
|
|
411
|
+
})));
|
|
412
|
+
out.base = worst("base");
|
|
413
|
+
out.head = worst("head");
|
|
414
|
+
out.caseScope = {
|
|
415
|
+
added,
|
|
416
|
+
removed
|
|
417
|
+
};
|
|
418
|
+
changes.push(...failing.map((r) => `case ${r.id} added`));
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
const variants = liftScope(b, h);
|
|
422
|
+
if (variants) {
|
|
423
|
+
out.cells = {
|
|
424
|
+
added: variants.added,
|
|
425
|
+
removed: variants.removed
|
|
426
|
+
};
|
|
427
|
+
if (!variants.shared.length) return {
|
|
428
|
+
...out,
|
|
429
|
+
...changes.length ? { changes } : {},
|
|
430
|
+
status: "drift"
|
|
431
|
+
};
|
|
432
|
+
const worst = (side, e) => outcomeOf([
|
|
433
|
+
...(shared.length ? over(side) : caseRows).map((r) => ({
|
|
434
|
+
gate: "all",
|
|
435
|
+
outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
|
|
436
|
+
})),
|
|
437
|
+
...(e.lifts ?? []).filter((l) => variants.shared.includes(l.variant)).map((l) => ({
|
|
438
|
+
gate: "all",
|
|
439
|
+
outcome: l.outcome
|
|
440
|
+
})),
|
|
441
|
+
...(e.rates ?? []).filter((r) => !variants.added.includes(r.cell) && !variants.removed.includes(r.cell)).map((r) => ({
|
|
442
|
+
gate: "all",
|
|
443
|
+
outcome: r.outcome
|
|
444
|
+
}))
|
|
445
|
+
]);
|
|
446
|
+
out.base = worst("base", b);
|
|
447
|
+
out.head = worst("head", h);
|
|
448
|
+
}
|
|
449
|
+
if (changes.length) out.changes = changes;
|
|
450
|
+
if (out.base === "infra_error" || out.head === "infra_error") return {
|
|
451
|
+
...out,
|
|
452
|
+
status: "infra"
|
|
343
453
|
};
|
|
344
|
-
const prefix = id.slice(0, id.lastIndexOf(" › ") + 3);
|
|
345
|
-
const name = (caseId) => caseId.startsWith(prefix) ? caseId.slice(prefix.length) : caseId;
|
|
346
|
-
const before = new Set(b.caseIds);
|
|
347
|
-
const after = new Set(h.caseIds);
|
|
348
|
-
const caseRows = [.../* @__PURE__ */ new Set([...h.caseIds, ...b.caseIds])].flatMap((c) => ctx.rows.get(c) ?? []);
|
|
349
|
-
const changes = [.../* @__PURE__ */ new Set([
|
|
350
|
-
...h.caseIds.filter((c) => !before.has(c)).map((c) => `case ${name(c)} added`),
|
|
351
|
-
...b.caseIds.filter((c) => !after.has(c)).map((c) => `case ${name(c)} removed`),
|
|
352
|
-
...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])
|
|
353
|
-
])];
|
|
354
|
-
const canceled = ["base", "head"].filter((s) => row[s] === "fail" && ctx.canceledIn(s, s === "base" ? b.caseIds : h.caseIds));
|
|
355
|
-
if (canceled.length) row.canceled = canceled;
|
|
356
454
|
if (caseRows.some((r) => r.status === "infra")) return {
|
|
357
|
-
...
|
|
455
|
+
...out,
|
|
358
456
|
status: "infra"
|
|
359
457
|
};
|
|
360
|
-
if (changes.
|
|
361
|
-
...
|
|
362
|
-
status: "drift",
|
|
363
|
-
changes
|
|
364
|
-
};
|
|
365
|
-
if (disjoint || row.canceled || caseRows.some((r) => r.status === "drift")) return {
|
|
366
|
-
...row,
|
|
458
|
+
if (changes.some((c) => c !== "gate changed") || out.canceled || caseRows.some((r) => r.status === "drift" || r.canceled)) return {
|
|
459
|
+
...out,
|
|
367
460
|
status: "drift"
|
|
368
461
|
};
|
|
369
|
-
|
|
370
|
-
|
|
462
|
+
const from = out.gate?.rescored ?? out.base;
|
|
463
|
+
if (graderChanged && from !== out.head) return {
|
|
464
|
+
...out,
|
|
371
465
|
status: "drift"
|
|
372
466
|
};
|
|
373
|
-
|
|
374
|
-
|
|
467
|
+
const caseRegressed = caseRows.some((r) => r.status === "regressed" && r.verdict !== "family");
|
|
468
|
+
if (from === "pass" && out.head === "fail" || caseRegressed) return {
|
|
469
|
+
...out,
|
|
375
470
|
status: "regressed"
|
|
376
471
|
};
|
|
377
|
-
if (
|
|
378
|
-
...
|
|
472
|
+
if (from === "fail" && out.head === "pass") return {
|
|
473
|
+
...out,
|
|
379
474
|
status: "fixed"
|
|
380
475
|
};
|
|
381
476
|
return {
|
|
382
|
-
...
|
|
383
|
-
status: "same"
|
|
477
|
+
...out,
|
|
478
|
+
status: out.base === out.head ? "same" : "gate"
|
|
384
479
|
};
|
|
385
480
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
const before = new Map((base.families ?? []).map((f) => [f.id, f]));
|
|
391
|
-
const after = new Map((head.families ?? []).map((f) => [f.id, f]));
|
|
392
|
-
const families = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((id) => gated(before.get(id)) || gated(after.get(id)));
|
|
393
|
-
const runs = {
|
|
394
|
-
base,
|
|
395
|
-
head
|
|
396
|
-
};
|
|
397
|
-
const ctx = {
|
|
398
|
-
rows: new Map(rows.map((r) => [r.id, r])),
|
|
399
|
-
graderChanged,
|
|
400
|
-
canceledIn: (side, caseIds) => runs[side].evals.some((e) => caseIds.includes(e.id) && e.cells.some((c) => (c.canceled ?? 0) > 0))
|
|
401
|
-
};
|
|
402
|
-
return [...rows, ...families.map((id) => familyRow(id, before.get(id), after.get(id), ctx))];
|
|
403
|
-
}
|
|
404
|
-
/** The key that tells a family's row from an eval's, whose ids can coincide. */
|
|
405
|
-
const rowKey = (row) => row.kind === "family" ? `family\0${row.id}` : row.id;
|
|
406
|
-
function evalRows(base, head, graderChanged) {
|
|
407
|
-
return [.../* @__PURE__ */ new Set([...head.evals.map((e) => e.id), ...base.evals.map((e) => e.id)])].map((id) => {
|
|
408
|
-
const b = base.evals.find((e) => e.id === id);
|
|
409
|
-
const h = head.evals.find((e) => e.id === id);
|
|
481
|
+
function evalRows(baseEvals, headEvals, graderChanged) {
|
|
482
|
+
return [.../* @__PURE__ */ new Set([...headEvals.map((e) => e.id), ...baseEvals.map((e) => e.id)])].map((id) => {
|
|
483
|
+
const b = baseEvals.find((e) => e.id === id);
|
|
484
|
+
const h = headEvals.find((e) => e.id === id);
|
|
410
485
|
if (!b) return addedRow(h);
|
|
411
|
-
if (!h) return
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
head: null,
|
|
415
|
-
status: "removed",
|
|
416
|
-
...canceledSides({
|
|
417
|
-
base: b.outcome,
|
|
418
|
-
head: null
|
|
419
|
-
}, { base: b.cells })
|
|
420
|
-
};
|
|
421
|
-
const comparable = sameEval(b, h);
|
|
486
|
+
if (!h) return removedRow(b);
|
|
487
|
+
const withCases = !!(b.cases || h.cases);
|
|
488
|
+
const comparable = withCases || sameEval(b, h);
|
|
422
489
|
const scoped = comparable ? null : sharedScope(b, h);
|
|
423
490
|
const outcomes = scoped ? {
|
|
424
491
|
base: scoped.base,
|
|
@@ -434,19 +501,22 @@ function evalRows(base, head, graderChanged) {
|
|
|
434
501
|
...canceledSides(outcomes, scoped?.over ?? {
|
|
435
502
|
base: b.cells,
|
|
436
503
|
head: h.cells
|
|
437
|
-
})
|
|
504
|
+
}),
|
|
505
|
+
...verdictOf(h)
|
|
438
506
|
};
|
|
439
507
|
const perCell = compareCells(b, h);
|
|
440
508
|
if (perCell.length) row.perCell = perCell;
|
|
441
|
-
if (
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
509
|
+
if (!withCases) {
|
|
510
|
+
if (row.base === "infra_error" || row.head === "infra_error") return {
|
|
511
|
+
...row,
|
|
512
|
+
status: "infra"
|
|
513
|
+
};
|
|
514
|
+
if (!comparable && !scoped) return {
|
|
515
|
+
...row,
|
|
516
|
+
status: "drift",
|
|
517
|
+
changes: whyChanged(b, h)
|
|
518
|
+
};
|
|
519
|
+
}
|
|
450
520
|
const over = scoped?.over ?? {
|
|
451
521
|
base: b.cells ?? [],
|
|
452
522
|
head: h.cells ?? []
|
|
@@ -456,6 +526,7 @@ function evalRows(base, head, graderChanged) {
|
|
|
456
526
|
changes: ["gate changed"],
|
|
457
527
|
gate
|
|
458
528
|
});
|
|
529
|
+
if (withCases) return casesRow(b, h, graderChanged, row);
|
|
459
530
|
const from = gate?.rescored ?? row.base;
|
|
460
531
|
if (row.canceled) return {
|
|
461
532
|
...row,
|
|
@@ -479,426 +550,6 @@ function evalRows(base, head, graderChanged) {
|
|
|
479
550
|
};
|
|
480
551
|
});
|
|
481
552
|
}
|
|
482
|
-
// src/view/effort.ts
|
|
483
|
-
/** Below this two-sided p, an effort difference is an effect. */
|
|
484
|
-
const EFFORT_ALPHA = .05;
|
|
485
|
-
/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
|
|
486
|
-
const TOKENS_UNREPORTED = "some cells did not report usage";
|
|
487
|
-
const effortSample = (a) => ({
|
|
488
|
-
tools: a.activity.tools,
|
|
489
|
-
agentMs: a.timings.agentMs,
|
|
490
|
-
fileEdits: a.activity.fileEdits,
|
|
491
|
-
filesChanged: filesChanged(a.changedFiles),
|
|
492
|
-
...a.usage ? { tokens: totalTokens(a.usage) } : {}
|
|
493
|
-
});
|
|
494
|
-
/** The files the workspace diff changed, a collapsed directory counting each of its files. */
|
|
495
|
-
const filesChanged = (files) => files.reduce((n, f) => n + (f.count ?? 1), 0);
|
|
496
|
-
const counted$1 = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
497
|
-
const spread = (values) => ({
|
|
498
|
-
median: median(values) ?? 0,
|
|
499
|
-
q1: quantile(values, .25) ?? 0,
|
|
500
|
-
q3: quantile(values, .75) ?? 0
|
|
501
|
-
});
|
|
502
|
-
/** Token totals from the attempts that reported usage; empty when none did. */
|
|
503
|
-
const tokenValues = (samples) => samples.flatMap((s) => s.tokens === void 0 ? [] : [s.tokens]);
|
|
504
|
-
function effortOf(samples) {
|
|
505
|
-
if (samples.length === 0) return null;
|
|
506
|
-
const tokens = tokenValues(samples);
|
|
507
|
-
return {
|
|
508
|
-
n: samples.length,
|
|
509
|
-
tools: spread(samples.map((s) => s.tools)),
|
|
510
|
-
agentMs: spread(samples.map((s) => s.agentMs)),
|
|
511
|
-
fileEdits: median(samples.map((s) => s.fileEdits)) ?? 0,
|
|
512
|
-
filesChanged: median(samples.map((s) => s.filesChanged)) ?? 0,
|
|
513
|
-
...tokens.length ? { tokens: spread(tokens) } : {}
|
|
514
|
-
};
|
|
515
|
-
}
|
|
516
|
-
/**
|
|
517
|
-
* The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
|
|
518
|
-
* the test's (which side ranks lower), not the medians', which can be equal or disagree.
|
|
519
|
-
*/
|
|
520
|
-
function effortVerdict(xs, ys) {
|
|
521
|
-
const test = mannWhitney(xs, ys);
|
|
522
|
-
const p = test?.p ?? null;
|
|
523
|
-
if (test === null || test.floor >= .05) return {
|
|
524
|
-
p,
|
|
525
|
-
verdict: "insufficient"
|
|
526
|
-
};
|
|
527
|
-
if (test.p >= .05) return {
|
|
528
|
-
p,
|
|
529
|
-
verdict: "noise"
|
|
530
|
-
};
|
|
531
|
-
return {
|
|
532
|
-
p,
|
|
533
|
-
verdict: test.sign < 0 ? "fewer" : "more"
|
|
534
|
-
};
|
|
535
|
-
}
|
|
536
|
-
const ratio = (a, b) => b === 0 ? null : a / b;
|
|
537
|
-
/** Every value on one side is below every value on the other. */
|
|
538
|
-
const apart = (xs, ys) => Math.max(...xs) < Math.min(...ys) || Math.min(...xs) > Math.max(...ys);
|
|
539
|
-
/**
|
|
540
|
-
* The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
|
|
541
|
-
* ten separates by chance; at 1 against 1 every pair of unequal values does.
|
|
542
|
-
*/
|
|
543
|
-
const DIRECTIONAL_MIN = 3;
|
|
544
|
-
/**
|
|
545
|
-
* The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
|
|
546
|
-
*/
|
|
547
|
-
const EFFORT_MIN_RUNS = (() => {
|
|
548
|
-
let n = 1;
|
|
549
|
-
while (minimumP(n, n) >= EFFORT_ALPHA) n++;
|
|
550
|
-
return n;
|
|
551
|
-
})();
|
|
552
|
-
/**
|
|
553
|
-
* The metrics whose test is `insufficient` but whose ranges don't overlap at
|
|
554
|
-
* `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
|
|
555
|
-
*/
|
|
556
|
-
function directional(c) {
|
|
557
|
-
if (Math.min(c.variant.n, c.control.n) < 3) return [];
|
|
558
|
-
const out = [];
|
|
559
|
-
if (c.separated && c.verdict === "insufficient") out.push("tools");
|
|
560
|
-
if (c.timeSeparated && c.timeVerdict === "insufficient") out.push("agentS");
|
|
561
|
-
if (c.tokensSeparated && c.tokensVerdict === "insufficient") out.push("tokens");
|
|
562
|
-
return out;
|
|
563
|
-
}
|
|
564
|
-
function contrastEffort(variant, control) {
|
|
565
|
-
const v = effortOf(variant);
|
|
566
|
-
const c = effortOf(control);
|
|
567
|
-
if (!v || !c) return null;
|
|
568
|
-
const tools = effortVerdict(variant.map((s) => s.tools), control.map((s) => s.tools));
|
|
569
|
-
const time = effortVerdict(variant.map((s) => s.agentMs), control.map((s) => s.agentMs));
|
|
570
|
-
const vTok = tokenValues(variant);
|
|
571
|
-
const cTok = tokenValues(control);
|
|
572
|
-
const tokens = vTok.length && cTok.length ? effortVerdict(vTok, cTok) : null;
|
|
573
|
-
const reported = vTok.length + cTok.length;
|
|
574
|
-
const unreported = reported > 0 && reported < variant.length + control.length;
|
|
575
|
-
return {
|
|
576
|
-
variant: v,
|
|
577
|
-
control: c,
|
|
578
|
-
toolsRatio: ratio(v.tools.median, c.tools.median),
|
|
579
|
-
agentMsRatio: ratio(v.agentMs.median, c.agentMs.median),
|
|
580
|
-
agentMsDelta: v.agentMs.median - c.agentMs.median,
|
|
581
|
-
toolsP: tools.p,
|
|
582
|
-
agentMsP: time.p,
|
|
583
|
-
separated: apart(variant.map((s) => s.tools), control.map((s) => s.tools)),
|
|
584
|
-
timeSeparated: apart(variant.map((s) => s.agentMs), control.map((s) => s.agentMs)),
|
|
585
|
-
tokensSeparated: tokens ? apart(vTok, cTok) : null,
|
|
586
|
-
verdict: tools.verdict,
|
|
587
|
-
timeVerdict: time.verdict,
|
|
588
|
-
tokensRatio: tokens && v.tokens && c.tokens ? ratio(v.tokens.median, c.tokens.median) : null,
|
|
589
|
-
tokensP: tokens?.p ?? null,
|
|
590
|
-
tokensVerdict: tokens?.verdict ?? null,
|
|
591
|
-
...unreported ? { tokensNote: TOKENS_UNREPORTED } : {}
|
|
592
|
-
};
|
|
593
|
-
}
|
|
594
|
-
/** The counted attempts of one cell of an eval, as effort samples. */
|
|
595
|
-
const cellSamples = (e, cellKey) => e.attempts.filter((a) => a.cell === cellKey && counted$1(a)).map(effortSample);
|
|
596
|
-
const cellEffort = (e, cellKey) => effortOf(cellSamples(e, cellKey));
|
|
597
|
-
/** A variant cell's effort against its control's, in the same run; null for controls. */
|
|
598
|
-
function cellContrast(e, cellKey) {
|
|
599
|
-
const cell = e.cells.find((c) => c.key === cellKey);
|
|
600
|
-
if (!cell?.lift) return null;
|
|
601
|
-
return contrastEffort(cellSamples(e, cellKey), cellSamples(e, cell.lift.vs));
|
|
602
|
-
}
|
|
603
|
-
/** `4`, `4.5`: medians of counts are whole or halves. */
|
|
604
|
-
const formatCount = (n) => Number.isInteger(n) ? String(n) : n.toFixed(1);
|
|
605
|
-
const formatRatio = (r) => r === null ? "×–" : `×${r.toFixed(2)}`;
|
|
606
|
-
/** Below this, agent time rounds under a tenth of a second and prints `<0.1s`. */
|
|
607
|
-
const TENTH_MS = 50;
|
|
608
|
-
/** Two agent times that both print `<0.1s` have no ratio worth printing. */
|
|
609
|
-
const bothUnderATenth = (a, b) => a < TENTH_MS && b < TENTH_MS;
|
|
610
|
-
/** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
|
|
611
|
-
function formatSeconds(ms) {
|
|
612
|
-
if (ms < 9950) return ms < TENTH_MS ? "<0.1s" : `${(ms / 1e3).toFixed(1)}s`;
|
|
613
|
-
const s = Math.round(ms / 1e3);
|
|
614
|
-
return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, "0")}s`;
|
|
615
|
-
}
|
|
616
|
-
const formatP = (p) => p === null ? "" : ` (p = ${p < .001 ? "<0.001" : p.toFixed(3)})`;
|
|
617
|
-
const VERDICT_TEXT = {
|
|
618
|
-
fewer: (what) => `fewer ${what}`,
|
|
619
|
-
more: (what) => `more ${what}`,
|
|
620
|
-
noise: () => "within noise",
|
|
621
|
-
insufficient: () => "too few attempts to test"
|
|
622
|
-
};
|
|
623
|
-
const toolsVerdictText = (c) => c.verdict === "fewer" || c.verdict === "more" ? `${VERDICT_TEXT[c.verdict]("tool calls")}${formatP(c.toolsP)}` : VERDICT_TEXT[c.verdict]("");
|
|
624
|
-
/** Token verdict after the tools one; empty when omitted or already covered as insufficient. */
|
|
625
|
-
const tokensVerdictText = (c) => {
|
|
626
|
-
if (c.tokensVerdict === null) return "";
|
|
627
|
-
if (c.verdict === "insufficient" && c.tokensVerdict === "insufficient") return "";
|
|
628
|
-
if (c.tokensVerdict === "fewer" || c.tokensVerdict === "more") return `; ${VERDICT_TEXT[c.tokensVerdict]("tokens")}${formatP(c.tokensP)}`;
|
|
629
|
-
if (c.tokensVerdict === "noise") return "; tokens within noise";
|
|
630
|
-
return `; ${VERDICT_TEXT[c.tokensVerdict]("")}`;
|
|
631
|
-
};
|
|
632
|
-
/**
|
|
633
|
-
* `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
|
|
634
|
-
* against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
|
|
635
|
-
* when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
|
|
636
|
-
* A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`. Agent
|
|
637
|
-
* times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
|
|
638
|
-
*/
|
|
639
|
-
function effortLabel(c) {
|
|
640
|
-
const [va, ca] = [c.variant.agentMs.median, c.control.agentMs.median];
|
|
641
|
-
const tiny = bothUnderATenth(va, ca);
|
|
642
|
-
const apartOn = new Set(directional(c).filter((m) => !(tiny && m === "agentS")));
|
|
643
|
-
const r = (metric, value) => `(${formatRatio(value)}${apartOn.has(metric) ? ", no overlap" : ""})`;
|
|
644
|
-
const parts = [`tools ${formatCount(c.variant.tools.median)} vs ${formatCount(c.control.tools.median)} ${r("tools", c.toolsRatio)}`, `agent ${formatSeconds(va)} vs ${formatSeconds(ca)}${tiny ? "" : ` ${r("agentS", c.agentMsRatio)}`}`];
|
|
645
|
-
if (c.tokensVerdict !== null && c.variant.tokens && c.control.tokens) parts.push(`tokens ${formatTokens(c.variant.tokens.median)} vs ${formatTokens(c.control.tokens.median)} ${r("tokens", c.tokensRatio)}`);
|
|
646
|
-
const n = apartOn.size ? ` (n = ${c.variant.n} vs ${c.control.n})` : "";
|
|
647
|
-
return `${parts.join(", ")}: ${toolsVerdictText(c)}${tokensVerdictText(c)}${n}`;
|
|
648
|
-
}
|
|
649
|
-
/** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
|
|
650
|
-
const effortSummary = (e) => e.tokens ? `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)} · tokens ${formatTokens(e.tokens.median)}` : `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)}`;
|
|
651
|
-
/** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
|
|
652
|
-
function effortChangeLabel(base, head) {
|
|
653
|
-
const t = ratio(head.tools.median, base.tools.median);
|
|
654
|
-
const a = ratio(head.agentMs.median, base.agentMs.median);
|
|
655
|
-
const [ba, ha] = [base.agentMs.median, head.agentMs.median];
|
|
656
|
-
const agent = `agent ${formatSeconds(ba)} → ${formatSeconds(ha)}${bothUnderATenth(ba, ha) ? "" : ` (${formatRatio(a)})`}`;
|
|
657
|
-
const core = `tools ${formatCount(base.tools.median)} → ${formatCount(head.tools.median)} (${formatRatio(t)}) · ${agent}`;
|
|
658
|
-
if (!base.tokens || !head.tokens) return core;
|
|
659
|
-
const tok = ratio(head.tokens.median, base.tokens.median);
|
|
660
|
-
return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
|
|
661
|
-
}
|
|
662
|
-
// src/view/judgments.ts
|
|
663
|
-
/** Show each key until this many; then the minimum score plus a count. */
|
|
664
|
-
const COLLAPSE_AFTER = 2;
|
|
665
|
-
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
666
|
-
/** Short names stay at this many characters; a collision takes the distinctive tail instead. */
|
|
667
|
-
const LABEL_MAX = 20;
|
|
668
|
-
const judgeText = (question) => question.replace(/\?+$/, "").trim();
|
|
669
|
-
/** Last `max` characters, cut at a word boundary where there is one, marked with a leading `…`. */
|
|
670
|
-
function clipTail(text, max) {
|
|
671
|
-
if (text.length <= max) return text;
|
|
672
|
-
const cut = text.slice(-(max - 1));
|
|
673
|
-
const space = cut.search(/\s/);
|
|
674
|
-
return `…${(space >= 0 && space < max / 2 ? cut.slice(space + 1) : cut).trimStart()}`;
|
|
675
|
-
}
|
|
676
|
-
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
677
|
-
function judgeLabel(question) {
|
|
678
|
-
return clip(judgeText(question), LABEL_MAX);
|
|
679
|
-
}
|
|
680
|
-
/**
|
|
681
|
-
* One short label per key. Shared prefixes take the distinctive tail so
|
|
682
|
-
* `Does this change…` does not collapse several checks; a tail that still
|
|
683
|
-
* collides falls back to the key.
|
|
684
|
-
*/
|
|
685
|
-
function uniqueJudgeLabels(items) {
|
|
686
|
-
const first = /* @__PURE__ */ new Map();
|
|
687
|
-
for (const item of items) if (!first.has(item.key)) first.set(item.key, item);
|
|
688
|
-
const labels = /* @__PURE__ */ new Map();
|
|
689
|
-
const used = /* @__PURE__ */ new Map();
|
|
690
|
-
const setLabel = (key, label) => {
|
|
691
|
-
const prev = labels.get(key);
|
|
692
|
-
if (prev !== void 0) {
|
|
693
|
-
const group = used.get(prev);
|
|
694
|
-
if (group) {
|
|
695
|
-
const next = group.filter((k) => k !== key);
|
|
696
|
-
if (next.length) used.set(prev, next);
|
|
697
|
-
else used.delete(prev);
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
labels.set(key, label);
|
|
701
|
-
const group = used.get(label) ?? [];
|
|
702
|
-
group.push(key);
|
|
703
|
-
used.set(label, group);
|
|
704
|
-
};
|
|
705
|
-
for (const row of first.values()) setLabel(row.key, judgeLabel(row.question));
|
|
706
|
-
for (const [label, keys] of [...used]) {
|
|
707
|
-
if (keys.length < 2) continue;
|
|
708
|
-
const tails = /* @__PURE__ */ new Map();
|
|
709
|
-
for (const key of keys) {
|
|
710
|
-
const row = first.get(key);
|
|
711
|
-
if (!row) continue;
|
|
712
|
-
const tail = clipTail(judgeText(row.question), LABEL_MAX);
|
|
713
|
-
const group = tails.get(tail) ?? [];
|
|
714
|
-
group.push(key);
|
|
715
|
-
tails.set(tail, group);
|
|
716
|
-
}
|
|
717
|
-
for (const [tail, tkeys] of tails) {
|
|
718
|
-
const [only] = tkeys;
|
|
719
|
-
const taken = (used.get(tail) ?? []).some((k) => !tkeys.includes(k));
|
|
720
|
-
if (only && tkeys.length === 1 && tail !== label && !taken) setLabel(only, tail);
|
|
721
|
-
else for (const key of tkeys) setLabel(key, key);
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
for (const [, keys] of [...used]) {
|
|
725
|
-
if (keys.length < 2) continue;
|
|
726
|
-
for (const key of keys) setLabel(key, key);
|
|
727
|
-
}
|
|
728
|
-
return labels;
|
|
729
|
-
}
|
|
730
|
-
function keyTallies(e, cellKey) {
|
|
731
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
732
|
-
const expectLabels = /* @__PURE__ */ new Map();
|
|
733
|
-
for (const a of e.attempts) {
|
|
734
|
-
if (a.cell !== cellKey || !counted(a)) continue;
|
|
735
|
-
for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
|
|
736
|
-
const row = byKey.get(j.key) ?? {
|
|
737
|
-
label: "",
|
|
738
|
-
labelled: false,
|
|
739
|
-
question: j.question,
|
|
740
|
-
pass: 0,
|
|
741
|
-
counted: 0
|
|
742
|
-
};
|
|
743
|
-
row.counted++;
|
|
744
|
-
if (j.verdict === "pass") row.pass++;
|
|
745
|
-
byKey.set(j.key, row);
|
|
746
|
-
if (labeled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
|
|
750
|
-
key,
|
|
751
|
-
question: row.question
|
|
752
|
-
})));
|
|
753
|
-
for (const [key, row] of byKey) {
|
|
754
|
-
const label = expectLabels.get(key);
|
|
755
|
-
row.labelled = label !== void 0;
|
|
756
|
-
row.label = label ?? labels.get(key) ?? judgeLabel(row.question);
|
|
757
|
-
}
|
|
758
|
-
return byKey;
|
|
759
|
-
}
|
|
760
|
-
function pairedControl(e, cellKey) {
|
|
761
|
-
const cell = e.cells.find((c) => c.key === cellKey);
|
|
762
|
-
if (cell?.role !== "variant") return void 0;
|
|
763
|
-
return e.cells.find((x) => x.role === "control" && x.harness === cell.harness && x.sandbox === cell.sandbox);
|
|
764
|
-
}
|
|
765
|
-
function liftForKey(variant, control) {
|
|
766
|
-
if (!control || variant.counted < 4 || control.counted < 4) return null;
|
|
767
|
-
const { points, low, high } = liftInterval({
|
|
768
|
-
pass: variant.pass,
|
|
769
|
-
n: variant.counted
|
|
770
|
-
}, {
|
|
771
|
-
pass: control.pass,
|
|
772
|
-
n: control.counted
|
|
773
|
-
});
|
|
774
|
-
return {
|
|
775
|
-
points: round3(points),
|
|
776
|
-
control: {
|
|
777
|
-
pass: control.pass,
|
|
778
|
-
counted: control.counted
|
|
779
|
-
},
|
|
780
|
-
variant: {
|
|
781
|
-
pass: variant.pass,
|
|
782
|
-
counted: variant.counted
|
|
783
|
-
},
|
|
784
|
-
withinNoise: containsZero(low, high)
|
|
785
|
-
};
|
|
786
|
-
}
|
|
787
|
-
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
|
|
788
|
-
function cellJudgeRates(e, cellKey) {
|
|
789
|
-
const tallies = keyTallies(e, cellKey);
|
|
790
|
-
const control = pairedControl(e, cellKey);
|
|
791
|
-
const controlTallies = control ? keyTallies(e, control.key) : void 0;
|
|
792
|
-
return [...tallies].map(([key, row]) => ({
|
|
793
|
-
key,
|
|
794
|
-
...row,
|
|
795
|
-
lift: liftForKey(row, controlTallies?.get(key))
|
|
796
|
-
}));
|
|
797
|
-
}
|
|
798
|
-
/**
|
|
799
|
-
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
800
|
-
* of the cell's checks against its control is already there.
|
|
801
|
-
*/
|
|
802
|
-
function summaryJudgeRates(e, cell) {
|
|
803
|
-
const checked = new Set((cell.checks ?? []).filter((k) => k.control && k.verdict).map((k) => k.label));
|
|
804
|
-
return cellJudgeRates(e, cell.key).filter((r) => !(r.labelled && checked.has(r.label)));
|
|
805
|
-
}
|
|
806
|
-
const signedPoints = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
|
|
807
|
-
/** `login 4/5 · checkout 0/5`, with `(+80 pts)` on a key that has lift. */
|
|
808
|
-
function judgeRatesLabel(rates) {
|
|
809
|
-
return rates.map((r) => {
|
|
810
|
-
const n = `${r.label} ${r.pass}/${r.counted}`;
|
|
811
|
-
return r.lift ? `${n} (${signedPoints(r.lift.points)} pts)` : n;
|
|
812
|
-
}).join(" · ");
|
|
813
|
-
}
|
|
814
|
-
/** Each key, or the lowest score plus how many judges there were. */
|
|
815
|
-
function collapsedJudges(judgments) {
|
|
816
|
-
const labels = uniqueJudgeLabels(judgments);
|
|
817
|
-
const items = judgments.map((j) => ({
|
|
818
|
-
verdict: j.verdict,
|
|
819
|
-
score: j.score,
|
|
820
|
-
threshold: j.threshold,
|
|
821
|
-
label: labels.get(j.key) ?? judgeLabel(j.question),
|
|
822
|
-
key: j.key
|
|
823
|
-
}));
|
|
824
|
-
if (items.length <= COLLAPSE_AFTER) return {
|
|
825
|
-
shown: items,
|
|
826
|
-
extra: 0
|
|
827
|
-
};
|
|
828
|
-
return {
|
|
829
|
-
shown: [items.reduce((a, b) => a.score <= b.score ? a : b)],
|
|
830
|
-
extra: items.length
|
|
831
|
-
};
|
|
832
|
-
}
|
|
833
|
-
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
834
|
-
function collapsedJudgeLabel(judgments) {
|
|
835
|
-
if (!judgments.length) return null;
|
|
836
|
-
const { shown, extra } = collapsedJudges(judgments);
|
|
837
|
-
const parts = shown.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold.toFixed(2)}`);
|
|
838
|
-
if (extra) parts.push(`${extra} judges`);
|
|
839
|
-
return parts.join(" · ");
|
|
840
|
-
}
|
|
841
|
-
const JUDGE_ERROR = /^judge failed for /;
|
|
842
|
-
/**
|
|
843
|
-
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
844
|
-
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
845
|
-
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
846
|
-
* reaches is named by its question.
|
|
847
|
-
*/
|
|
848
|
-
function labeledJudgments(a) {
|
|
849
|
-
const passes = (a.assertions ?? []).filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
|
|
850
|
-
const zip = passes.length === a.judgements.length;
|
|
851
|
-
return a.judgements.map((judgment, i) => {
|
|
852
|
-
const x = zip ? passes[i] : void 0;
|
|
853
|
-
return {
|
|
854
|
-
judgment,
|
|
855
|
-
label: x?.labelled ? x.label : judgeText(judgment.question),
|
|
856
|
-
labelled: x?.labelled === true,
|
|
857
|
-
...x ? { check: x.label } : {}
|
|
858
|
-
};
|
|
859
|
-
});
|
|
860
|
-
}
|
|
861
|
-
/** Sum of counted attempts' usage; null when none of them reported any. */
|
|
862
|
-
function cellUsage(e, cellKey) {
|
|
863
|
-
return sumUsage(e.attempts.filter((a) => a.cell === cellKey && counted(a)).map((a) => a.usage));
|
|
864
|
-
}
|
|
865
|
-
/**
|
|
866
|
-
* Each `c.exec` check's `expect` label, by position: the attempt's `toSucceed` assertions in call
|
|
867
|
-
* order, when they are exactly as many as its checks; undefined where no label reaches.
|
|
868
|
-
*/
|
|
869
|
-
function checkLabels(a) {
|
|
870
|
-
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
871
|
-
const zip = succeeds.length === a.checks.length;
|
|
872
|
-
return a.checks.map((_, i) => {
|
|
873
|
-
const x = zip ? succeeds[i] : void 0;
|
|
874
|
-
return x?.labelled ? x.label : void 0;
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
/**
|
|
878
|
-
* Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
|
|
879
|
-
* are exactly as many as its checks; empty otherwise.
|
|
880
|
-
*/
|
|
881
|
-
function checksByLabel(a) {
|
|
882
|
-
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
883
|
-
if (succeeds.length !== a.checks.length) return /* @__PURE__ */ new Map();
|
|
884
|
-
return new Map(succeeds.map((x, i) => [x.label, a.checks[i]]));
|
|
885
|
-
}
|
|
886
|
-
/**
|
|
887
|
-
* How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
|
|
888
|
-
* when it had none or passed on its first try.
|
|
889
|
-
*/
|
|
890
|
-
function triesNote(exitCode, tried, of) {
|
|
891
|
-
if (of <= 1 || exitCode === 0 && tried <= 1) return null;
|
|
892
|
-
if (exitCode === 0) return `passed on try ${tried} of ${of}`;
|
|
893
|
-
return tried >= of ? `failed all ${of} tries` : `failed on try ${tried} of ${of}`;
|
|
894
|
-
}
|
|
895
|
-
/** {@link triesNote} for a recorded check. */
|
|
896
|
-
const checkTries = (c) => c.retries && c.tries ? triesNote(c.exitCode, c.tries.length, c.retries + 1) : null;
|
|
897
|
-
/** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
|
|
898
|
-
function commandLine(command, max = 80) {
|
|
899
|
-
const lines = command.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
900
|
-
return clip(lines.length > 1 ? `${lines[0]} …` : lines[0] ?? "", max);
|
|
901
|
-
}
|
|
902
553
|
// src/view/model.ts
|
|
903
554
|
/** Splits `file › group › … › name` back into its parts. */
|
|
904
555
|
function splitEvalId(id) {
|
|
@@ -1006,28 +657,31 @@ const shortSha = (sha) => sha ? sha.slice(0, 7) : "no git";
|
|
|
1006
657
|
/** Reasoning time, or `n/a` when the harness couldn't measure it: never `0s` for unknown. */
|
|
1007
658
|
const reasoningLabel = (ms) => ms === null ? "n/a" : formatDuration(ms);
|
|
1008
659
|
/**
|
|
1009
|
-
* An attempt's effort,
|
|
1010
|
-
*
|
|
1011
|
-
* With the workspace diff's count, `files changed` follows: an edit made from the shell is in it,
|
|
1012
|
-
* never in the file edits.
|
|
660
|
+
* An attempt's effort: tool calls, then the files the workspace diff changed, which counts a file
|
|
661
|
+
* the shell wrote, then the calls to the agent's edit tools, which doesn't.
|
|
1013
662
|
*/
|
|
1014
663
|
function activityLabel(a, changed) {
|
|
1015
|
-
|
|
1016
|
-
return `${plural(a.tools, "tool call")} · ${plural(a.fileEdits, "file edit")}${files}`;
|
|
664
|
+
return `${plural(a.tools, "tool call")} · ${plural(changed, "file")} changed · ${plural(a.fileEdits, "edit tool call")}`;
|
|
1017
665
|
}
|
|
1018
666
|
/**
|
|
1019
667
|
* A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
|
|
1020
668
|
* failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
|
|
1021
|
-
* default) don't. An interrupted run is `skipped`, whatever ran
|
|
669
|
+
* default) don't. An interrupted run is `skipped`, whatever ran; a resume its filters left
|
|
670
|
+
* `interrupted` exits on its outcome as if it were complete.
|
|
1022
671
|
*/
|
|
1023
672
|
function runOutcome(run) {
|
|
1024
673
|
if (run.status === "interrupted") return "skipped";
|
|
1025
674
|
if (run.errors?.length) return "fail";
|
|
1026
|
-
const outcomes =
|
|
675
|
+
const outcomes = run.evals.map((e) => e.outcome);
|
|
1027
676
|
if (outcomes.includes("fail") || run.status === "budget") return "fail";
|
|
1028
677
|
if (outcomes.includes("infra_error")) return "infra_error";
|
|
1029
678
|
return "pass";
|
|
1030
679
|
}
|
|
680
|
+
/**
|
|
681
|
+
* The eval hasn't run yet: its run stopped (interrupted, or out of budget) or a filtered resume
|
|
682
|
+
* left it out, and it's `skipped` without being a todo or declared skipped. A resume runs it.
|
|
683
|
+
*/
|
|
684
|
+
const notRunYet = (run, e) => run.status !== "complete" && e.outcome === "skipped" && !e.todo && !e.skip;
|
|
1031
685
|
function andList(items) {
|
|
1032
686
|
if (items.length <= 1) return items[0] ?? "";
|
|
1033
687
|
if (items.length === 2) return `${items[0]} and ${items[1]}`;
|
|
@@ -1325,7 +979,7 @@ function gatedPassAt1(cells) {
|
|
|
1325
979
|
/** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
|
|
1326
980
|
function compareView(base, head, records = {}) {
|
|
1327
981
|
const rows = compare(base, head).map((row) => {
|
|
1328
|
-
const evalOf = (run) =>
|
|
982
|
+
const evalOf = (run) => run.evals.find((e) => e.id === row.id);
|
|
1329
983
|
const b = evalOf(base);
|
|
1330
984
|
const h = evalOf(head);
|
|
1331
985
|
const baseKeys = new Set(b?.cells.map((c) => c.key));
|
|
@@ -1388,6 +1042,6 @@ const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
|
|
|
1388
1042
|
const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
|
|
1389
1043
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
1390
1044
|
const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
|
|
1391
|
-
export {
|
|
1045
|
+
export { runRows as A, stackChanges as B, isUngatedMiss as C, reasoningLabel as D, parseTranscript as E, addedRow as F, cellsChanged as I, compare as L, shortSha as M, splitEvalId as N, runLead as O, transcriptItems as P, newRows as R, isNotGatedPass as S, notRunYet as T, whyChanged as V, formatUsage as _, buildIndex as a, indexFiles as b, defaultAttempt as c, evalTree as d, failedJudgeLabels as f, formatPercent as g, formatDuration as h, attemptAnswerRows as i, runtimeLabel as j, runOutcome as k, evalHistory as l, findEval as m, activityLabel as n, cellRows as o, findAttempt as p, answerRows as r, compareView as s, COMPARE_ORDER as t, evalMarkOutcome as u, harnessLabel as v, languageLabel as w, isMatrix as x, hasUngatedCells as y, sourceChanges as z };
|
|
1392
1046
|
|
|
1393
|
-
//# sourceMappingURL=model-
|
|
1047
|
+
//# sourceMappingURL=model-V3mMXp6x.js.map
|