reeval 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-CptK032x.js → adapter-C60z3wLq.js} +5 -4
- package/dist/adapter-C60z3wLq.js.map +1 -0
- package/dist/{agent-B63OzUX9.js → agent-BmsMXZXx.js} +30 -10
- package/dist/{agent-B63OzUX9.js.map → agent-BmsMXZXx.js.map} +1 -1
- package/dist/agents-DXuFxtGC.js +1151 -0
- package/dist/agents-DXuFxtGC.js.map +1 -0
- package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-CE4pHKIa.js +1 -0
- package/dist/{assemble-Me7Whp-y.js → assemble-DM2PNU4R.js} +152 -129
- package/dist/assemble-DM2PNU4R.js.map +1 -0
- package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
- package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-Cik0SoIn.js → attempt-host-7VviAJrl.js} +6 -6
- package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-7VviAJrl.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-DSNgZt-t.js +322 -0
- package/dist/boat-DSNgZt-t.js.map +1 -0
- package/dist/bridge/index.mjs +401 -79
- package/dist/capture-BzotD2hs.js +1 -0
- package/dist/{capture-C3RmjPph.js → capture-Ccf0Hmyx.js} +27 -25
- package/dist/capture-Ccf0Hmyx.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-BBOlXKPG.js → cli-DTuLSlmW.js} +442 -707
- package/dist/cli-DTuLSlmW.js.map +1 -0
- package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-DhiwsZl1.js → command-C2C7P5qo.js} +9 -9
- package/dist/{command-DhiwsZl1.js.map → command-C2C7P5qo.js.map} +1 -1
- package/dist/{command-CFQ9c1hk.js → command-D8oMB2fa.js} +16 -16
- package/dist/command-D8oMB2fa.js.map +1 -0
- package/dist/{command-fO0BFsTq.js → command-DSL7I-SI.js} +37 -37
- package/dist/command-DSL7I-SI.js.map +1 -0
- package/dist/{command-qGS802uX.js → command-Ff0T-1Hi.js} +21 -80
- package/dist/command-Ff0T-1Hi.js.map +1 -0
- package/dist/{commands-xYTMNTPv.js → commands-Bk9NG1s_.js} +1 -1
- package/dist/{commands-CC60TOT5.js → commands-xAOi_VEk.js} +186 -85
- package/dist/commands-xAOi_VEk.js.map +1 -0
- package/dist/{config-C2UtS9vy.js → config-SvK8pjEK.js} +261 -1708
- package/dist/config-SvK8pjEK.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-BdgCao9Q.js} +3 -2
- package/dist/context-BdgCao9Q.js.map +1 -0
- package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
- package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-DEzan4IT.js} +3 -3
- package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-DEzan4IT.js.map} +1 -1
- package/dist/{db-C5SZ0510.js → db-BzOJ4Js4.js} +130 -84
- package/dist/db-BzOJ4Js4.js.map +1 -0
- package/dist/db-CQRZ3URX.js +2 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-FEidNAIi.js → default-effort-BBDPbJ4D.js} +1 -1
- package/dist/{default-effort-BVmx7r7L.js → default-effort-BDKIxJyC.js} +4 -4
- package/dist/{default-effort-BVmx7r7L.js.map → default-effort-BDKIxJyC.js.map} +1 -1
- package/dist/{dist-ByYk_wAs.js → dist-Db7ifchA.js} +37 -8
- package/dist/dist-Db7ifchA.js.map +1 -0
- package/dist/{docker-BlSopHpZ.js → docker--mukta3L.js} +1 -1
- package/dist/{docker-B2F8-PGC.js → docker-DFrjoNs-.js} +3 -3
- package/dist/{docker-B2F8-PGC.js.map → docker-DFrjoNs-.js.map} +1 -1
- package/dist/{docker-Dto8ROCq.js → docker-DG_uFgn2.js} +109 -77
- package/dist/docker-DG_uFgn2.js.map +1 -0
- package/dist/{doctor-BcvfKpBX.js → doctor-DWK4ttex.js} +84 -27
- package/dist/doctor-DWK4ttex.js.map +1 -0
- package/dist/{environment-CC1RV21n.js → environment-cPMCOk-h.js} +79 -39
- package/dist/environment-cPMCOk-h.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-CzxdgFBW.js → eval-source-C9b1h8L6.js} +10 -10
- package/dist/{eval-source-CzxdgFBW.js.map → eval-source-C9b1h8L6.js.map} +1 -1
- package/dist/{evaluate-NTNJ8T9H.js → evaluate-B12Tosvq.js} +377 -142
- package/dist/evaluate-B12Tosvq.js.map +1 -0
- package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
- package/dist/{files-aAzoGNCB.js → files-BOBdC0uI.js} +5 -5
- package/dist/{files-aAzoGNCB.js.map → files-BOBdC0uI.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-DMQ9-Vv5.js +1 -0
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/files-Di0xDbCc.js +1 -0
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/{files-DsXULUi5.js → files-F5EG0WPT.js} +5 -5
- package/dist/{files-DsXULUi5.js.map → files-F5EG0WPT.js.map} +1 -1
- package/dist/files-G5jf8wBp.js +1 -0
- package/dist/{files-CZXrkPja.js → files-s28a6czq.js} +54 -13
- package/dist/{files-CZXrkPja.js.map → files-s28a6czq.js.map} +1 -1
- package/dist/files-tiNWe8Ih.js +64 -0
- package/dist/files-tiNWe8Ih.js.map +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-C1MB5bjr.js → git-BpoqEgDV.js} +2 -2
- package/dist/{git-C1MB5bjr.js.map → git-BpoqEgDV.js.map} +1 -1
- package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
- package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
- package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-CNayXCp2.js → images-D2jFze2v.js} +98 -10
- package/dist/images-D2jFze2v.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/{init-D5HtmyMI.js → init-Bi1Q3lRi.js} +7 -6
- package/dist/init-Bi1Q3lRi.js.map +1 -0
- package/dist/init-CUw65o36.js +1 -0
- package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
- package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
- package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
- package/dist/kinds-Bh2I35id.js +698 -0
- package/dist/kinds-Bh2I35id.js.map +1 -0
- package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
- package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
- package/dist/{live-uRURFDMZ.js → live-DL0se4Zd.js} +7 -7
- package/dist/{live-uRURFDMZ.js.map → live-DL0se4Zd.js.map} +1 -1
- package/dist/live-DZLmj0iG.js +1 -0
- package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
- package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-CXVVB4RK.js → model-CkNVAxob.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{model-BxBWFOZn.js → model-cgt5uvu8.js} +5 -5
- package/dist/{model-BxBWFOZn.js.map → model-cgt5uvu8.js.map} +1 -1
- package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-DPUmk7v_.js +1 -0
- package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-ug5HpMOj.js → patch-DAi1nCuH.js} +2 -2
- package/dist/{patch-ug5HpMOj.js.map → patch-DAi1nCuH.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-DqnzfDH1.js → plan-CsPilzpk.js} +110 -39
- package/dist/plan-CsPilzpk.js.map +1 -0
- package/dist/{plugins-B2KQ0y3P.js → plugins-BfnhazOv.js} +10 -267
- package/dist/plugins-BfnhazOv.js.map +1 -0
- package/dist/{preflight-DyzmvK7i.js → preflight-BixaE4BA.js} +16 -9
- package/dist/preflight-BixaE4BA.js.map +1 -0
- package/dist/preflight-Bn3ianye.js +1 -0
- package/dist/{preflight-DVXEvzFU.js → preflight-D1C-iH0U.js} +1 -1
- package/dist/preflight-GkpgrBxP.js +68 -0
- package/dist/preflight-GkpgrBxP.js.map +1 -0
- package/dist/{prepare-BcSRa7-3.js → prepare-C--rM_wO.js} +2 -2
- package/dist/{prepare-21WMQZyt.js → prepare-DQEDXoDp.js} +9 -9
- package/dist/{prepare-21WMQZyt.js.map → prepare-DQEDXoDp.js.map} +1 -1
- package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
- package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-iRxjp4Pu.js → provenance-BXyYwUC8.js} +4 -3
- package/dist/provenance-BXyYwUC8.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-vXMXuwn-.js → providers-1p4HKYIp.js} +152 -9
- package/dist/providers-1p4HKYIp.js.map +1 -0
- package/dist/providers-B-ijCktf.js +1 -0
- package/dist/query-BYhyf0LR.js +1 -0
- package/dist/{query-BhaKQ07P.js → query-DhhU9ADI.js} +2 -2
- package/dist/{query-BhaKQ07P.js.map → query-DhhU9ADI.js.map} +1 -1
- package/dist/{questions-BzdLdV3C.js → questions-CMAq4pr8.js} +9 -9
- package/dist/{questions-BzdLdV3C.js.map → questions-CMAq4pr8.js.map} +1 -1
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{remarks-CauU408t.js → remarks-BNABzemt.js} +115 -35
- package/dist/remarks-BNABzemt.js.map +1 -0
- package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remote-Dwy3fyo3.js → remote-U7fNt0UP.js} +6 -6
- package/dist/{remote-Dwy3fyo3.js.map → remote-U7fNt0UP.js.map} +1 -1
- package/dist/resolve-C14Pmtej.js +1 -0
- package/dist/{resolve-B1zMnPwK.js → resolve-Cx3a9UQ2.js} +10 -9
- package/dist/resolve-Cx3a9UQ2.js.map +1 -0
- package/dist/{results-CfC0id_u.js → results-CQKXmHym.js} +6 -6
- package/dist/{results-CfC0id_u.js.map → results-CQKXmHym.js.map} +1 -1
- package/dist/{resume-Dkhrpjow.js → resume-POpZa1ge.js} +169 -27
- package/dist/resume-POpZa1ge.js.map +1 -0
- package/dist/retry-CmG809Mp.js +105 -0
- package/dist/retry-CmG809Mp.js.map +1 -0
- package/dist/{runtime-DcvQcr36.js → runtime-Xm9oeC7o.js} +26 -7
- package/dist/runtime-Xm9oeC7o.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-8zCrtuCf.js → sandbox-DrlL4yqb.js} +31 -3
- package/dist/sandbox-DrlL4yqb.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/{server-Dplq8g0r.js → server-C00zd6zl.js} +9 -9
- package/dist/{server-Dplq8g0r.js.map → server-C00zd6zl.js.map} +1 -1
- package/dist/server-C1UJcpUh.js +1 -0
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/{source-NWHqn5-o.js → source-DDgr48Gn.js} +2 -2
- package/dist/{source-NWHqn5-o.js.map → source-DDgr48Gn.js.map} +1 -1
- package/dist/source-DJHmQv_j.js +1 -0
- package/dist/src-1dBuXzs4.js +3 -0
- package/dist/{src-CUpoVVb8.js → src-BKpioayC.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BY1cCaoJ.js +2 -0
- package/dist/src-BbZYaQ8b.js +2 -0
- package/dist/src-CDFH2vVN.js +2 -0
- package/dist/src-CMW5R9Q0.js +1 -0
- package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-B9oE_mNS.js → src-CrV2WA2i.js} +2 -2
- package/dist/src-CsFaawcZ.js +2 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-wKwznPgt.js +2 -0
- package/dist/{src-BQ6BiXoc.js → src-y0nOeRRE.js} +610 -72
- package/dist/src-y0nOeRRE.js.map +1 -0
- package/dist/{state-DDCGUhva.js → state-328JpnHA.js} +17 -17
- package/dist/{state-DDCGUhva.js.map → state-328JpnHA.js.map} +1 -1
- package/dist/state-Ca55Gk_O.js +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DDy9DfyJ.js +148 -0
- package/dist/summary-DDy9DfyJ.js.map +1 -0
- package/dist/tart-cli-C-GsbVsQ.js +1 -0
- package/dist/tart-cli-CU0nFS-L.js +55 -0
- package/dist/tart-cli-CU0nFS-L.js.map +1 -0
- package/dist/tart-dWoJvWHc.js +586 -0
- package/dist/tart-dWoJvWHc.js.map +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +4 -4
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
- package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
- package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
- package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +101 -58
- package/dist/view/index.js +9 -8
- package/dist/views-B0Yt_Q1S.js +2 -0
- package/dist/{views-BISrsIjw.js → views-BROacjBv.js} +7 -6
- package/dist/views-BROacjBv.js.map +1 -0
- package/dist/{worker-entry-DJD_nHkS.js → worker-entry-Smb4bNDc.js} +11 -8
- package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-Smb4bNDc.js.map} +1 -1
- package/dist/workspace-C0dDP75l.js +1 -0
- package/dist/{workspace-BO1VhUR9.js → workspace-DNDTEk8v.js} +6 -6
- package/dist/{workspace-BO1VhUR9.js.map → workspace-DNDTEk8v.js.map} +1 -1
- package/package.json +24 -11
- package/skills/eval-rubric/SKILL.md +15 -7
- package/dist/adapter-CptK032x.js.map +0 -1
- package/dist/app-files-sgqwfjtn.js.map +0 -1
- package/dist/assemble-CxBHkavM.js +0 -1
- package/dist/assemble-Me7Whp-y.js.map +0 -1
- package/dist/capture-C3RmjPph.js.map +0 -1
- package/dist/capture-YPwxp_iK.js +0 -1
- package/dist/cli-BBOlXKPG.js.map +0 -1
- package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
- package/dist/client-tZ2WBDxf.js.map +0 -1
- package/dist/command-CFQ9c1hk.js.map +0 -1
- package/dist/command-fO0BFsTq.js.map +0 -1
- package/dist/command-qGS802uX.js.map +0 -1
- package/dist/commands-CC60TOT5.js.map +0 -1
- package/dist/config-C2UtS9vy.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-BjOg7wyr.js +0 -101
- package/dist/contrasts-BjOg7wyr.js.map +0 -1
- package/dist/cost-BJU0MAji.js +0 -1
- package/dist/db-C5SZ0510.js.map +0 -1
- package/dist/db-puzLzX4k.js +0 -2
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker-Dto8ROCq.js.map +0 -1
- package/dist/doctor-BcvfKpBX.js.map +0 -1
- package/dist/environment-CC1RV21n.js.map +0 -1
- package/dist/evaluate-NTNJ8T9H.js.map +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-DFtuRQJx.js +0 -1
- package/dist/files-oOYNR7ZH.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-Cp_pQoEe.js.map +0 -1
- package/dist/harness-DYSC4zMh.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-CNayXCp2.js.map +0 -1
- package/dist/init-8o6W3_fn.js +0 -1
- package/dist/init-D5HtmyMI.js.map +0 -1
- package/dist/ink-list-Ca-5dunO.js.map +0 -1
- package/dist/judge-DYOF69Oe.js.map +0 -1
- package/dist/list-CM_YSJXq.js +0 -374
- package/dist/list-CM_YSJXq.js.map +0 -1
- package/dist/live-DQc8jr6o.js +0 -1
- package/dist/model-Ce9l9_rF.js.map +0 -1
- package/dist/model-Cxzo4cjw.js +0 -1
- package/dist/node-DhIRSrrK.js.map +0 -1
- package/dist/opencode-providers-Co7plCea.js.map +0 -1
- package/dist/plan-DqnzfDH1.js.map +0 -1
- package/dist/plugins-B2KQ0y3P.js.map +0 -1
- package/dist/preflight-DyzmvK7i.js.map +0 -1
- package/dist/provenance-iRxjp4Pu.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-D0ob_tra.js +0 -1
- package/dist/providers-vXMXuwn-.js.map +0 -1
- package/dist/query-CvgUsvep.js +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/remarks-CauU408t.js.map +0 -1
- package/dist/remarks-NMgPd30U.js.map +0 -1
- package/dist/resolve-B1zMnPwK.js.map +0 -1
- package/dist/resolve-BIIeVT12.js +0 -1
- package/dist/resume-Dkhrpjow.js.map +0 -1
- package/dist/runtime-DcvQcr36.js.map +0 -1
- package/dist/sandbox-8zCrtuCf.js.map +0 -1
- package/dist/server-0tuj8VSa.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-llwLewOU.js +0 -1
- package/dist/src-BQ6BiXoc.js.map +0 -1
- package/dist/src-BXnv0ViU.js +0 -2
- package/dist/src-BiS8YNN_.js +0 -3
- package/dist/src-BkH4GOxu.js +0 -1
- package/dist/src-C-tH5ebd.js +0 -2
- package/dist/src-Cg1bFs9g.js +0 -2
- package/dist/src-DV_UorZv.js +0 -2
- package/dist/src-XxUpr2sa.js +0 -2
- package/dist/state-DZbAO95a.js +0 -1
- package/dist/suite-M8_7xio7.js.map +0 -1
- package/dist/suite-u2KJnGpY.js +0 -2
- package/dist/summary-Uzc9osfV.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-h_peaZTN.js +0 -1
- package/dist/views-BISrsIjw.js.map +0 -1
- package/dist/views-Bfab8fM9.js +0 -2
- package/dist/workspace-B8EOCuxl.js +0 -1
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
|
|
2
1
|
import { a as plural } from "./text-B8-W3Cvn.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
2
|
+
import { n as formatTokens, r as promptTokens } from "./usage-DJiDnh2D.js";
|
|
3
|
+
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-Dwmi7L4X.js";
|
|
4
|
+
import { l as sampleReasons } from "./pairs-BGb134Vu.js";
|
|
5
|
+
import { a as rescore, n as gateLabel } from "./gate-CgYdaprG.js";
|
|
6
|
+
import { T as cellEffort, w as cellContrast } from "./cases-z9E6DCd5.js";
|
|
7
|
+
import { m as uniqueJudgeLabels, n as cellUsage, t as cellJudgeRates } from "./judgments-CzgpsTqv.js";
|
|
8
|
+
import { u as readableTranscriptLine } from "./record-6IUzwP9K.js";
|
|
8
9
|
// src/view/compare.ts
|
|
9
10
|
const STACK_FIELDS = [
|
|
10
11
|
{
|
|
@@ -224,7 +225,7 @@ function canceledSides(outcomes, cells) {
|
|
|
224
225
|
const sides = ["base", "head"].filter((s) => canceledOn(outcomes[s], cells[s] ?? []));
|
|
225
226
|
return sides.length ? { canceled: sides } : {};
|
|
226
227
|
}
|
|
227
|
-
/** An eval only head has. */
|
|
228
|
+
/** An eval only head has, with each of its cases. */
|
|
228
229
|
const addedRow = (h) => ({
|
|
229
230
|
id: h.id,
|
|
230
231
|
base: null,
|
|
@@ -233,8 +234,41 @@ const addedRow = (h) => ({
|
|
|
233
234
|
...canceledSides({
|
|
234
235
|
base: null,
|
|
235
236
|
head: h.outcome
|
|
236
|
-
}, { head: h.cells })
|
|
237
|
+
}, { head: h.cells }),
|
|
238
|
+
...verdictOf(h),
|
|
239
|
+
...h.cases ? { cases: caseEvals(h).map(addedRow) } : {},
|
|
240
|
+
...casesGate(null, h)
|
|
241
|
+
});
|
|
242
|
+
/** An eval only base has, with each of its cases. */
|
|
243
|
+
const removedRow = (b) => ({
|
|
244
|
+
id: b.id,
|
|
245
|
+
base: b.outcome,
|
|
246
|
+
head: null,
|
|
247
|
+
status: "removed",
|
|
248
|
+
...canceledSides({
|
|
249
|
+
base: b.outcome,
|
|
250
|
+
head: null
|
|
251
|
+
}, { base: b.cells }),
|
|
252
|
+
...verdictOf(b),
|
|
253
|
+
...b.cases ? { cases: caseEvals(b).map(removedRow) } : {},
|
|
254
|
+
...casesGate(b, null)
|
|
237
255
|
});
|
|
256
|
+
/** A case its eval's rate gates, on the side that decides: head's, or base's when only it ran. */
|
|
257
|
+
const verdictOf = (e) => e?.verdict ? { verdict: e.verdict } : {};
|
|
258
|
+
/** An eval's cases, each as an eval of its own keyed by the case key. */
|
|
259
|
+
const caseEvals = (e) => (e.cases ?? []).map((k) => ({
|
|
260
|
+
id: k.key,
|
|
261
|
+
fingerprint: k.fingerprint,
|
|
262
|
+
outcome: k.outcome,
|
|
263
|
+
...k.verdict ? { verdict: k.verdict } : {},
|
|
264
|
+
sources: e.sources,
|
|
265
|
+
...e.source ? { source: {
|
|
266
|
+
...e.source,
|
|
267
|
+
...k.hash ? { case: k.hash } : {}
|
|
268
|
+
} } : {},
|
|
269
|
+
...e.revision !== void 0 ? { revision: e.revision } : {},
|
|
270
|
+
cells: k.cells
|
|
271
|
+
}));
|
|
238
272
|
/**
|
|
239
273
|
* When two runs of an eval differ only in which cells ran (a filtered run, a harness added), its
|
|
240
274
|
* outcome on the cells both have, and the cells only one has. Null when no cell is shared or a
|
|
@@ -277,13 +311,34 @@ function compareCells(base, head) {
|
|
|
277
311
|
}];
|
|
278
312
|
});
|
|
279
313
|
}
|
|
280
|
-
/**
|
|
281
|
-
|
|
314
|
+
/**
|
|
315
|
+
* An eval whose cases pool a gate: some lift has a `minLift`, or some cell's rate over the cases
|
|
316
|
+
* a `passRate`. Its outcome is then over the cases that ran, so a run of other cases can't be
|
|
317
|
+
* compared with it.
|
|
318
|
+
*/
|
|
319
|
+
const pooledGate = (e) => !!e && ((e.lifts ?? []).some((l) => l.minLift !== void 0) || !!e.rates?.length);
|
|
282
320
|
const liftSide = ({ variant: _, vs: __, ...side }) => side;
|
|
283
|
-
/**
|
|
284
|
-
|
|
321
|
+
/**
|
|
322
|
+
* When both runs pooled lifts over different variants: the variants only head or only base has,
|
|
323
|
+
* and those both have, over which the eval's outcome is then read.
|
|
324
|
+
*/
|
|
325
|
+
function liftScope(b, h) {
|
|
326
|
+
const of = (e) => (e.lifts ?? []).map((l) => l.variant);
|
|
327
|
+
const [base, head] = [of(b), of(h)];
|
|
328
|
+
if (!base.length || !head.length) return null;
|
|
329
|
+
const added = head.filter((v) => !base.includes(v));
|
|
330
|
+
const removed = base.filter((v) => !head.includes(v));
|
|
331
|
+
if (!added.length && !removed.length) return null;
|
|
332
|
+
return {
|
|
333
|
+
added,
|
|
334
|
+
removed,
|
|
335
|
+
shared: head.filter((v) => base.includes(v))
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/** Each variant's lift paired by case on both sides, head's order first. */
|
|
339
|
+
function casesLifts(b, h) {
|
|
285
340
|
const out = /* @__PURE__ */ new Map();
|
|
286
|
-
for (const [side,
|
|
341
|
+
for (const [side, e] of [["head", h], ["base", b]]) for (const l of e?.lifts ?? []) {
|
|
287
342
|
const row = out.get(l.variant) ?? {
|
|
288
343
|
key: l.variant,
|
|
289
344
|
vs: l.vs,
|
|
@@ -295,131 +350,142 @@ function familyLifts(b, h) {
|
|
|
295
350
|
}
|
|
296
351
|
return [...out.values()];
|
|
297
352
|
}
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
353
|
+
/** Each gated cell's rate over the cases on both sides, head's order first. */
|
|
354
|
+
function casesRates(b, h) {
|
|
355
|
+
const out = /* @__PURE__ */ new Map();
|
|
356
|
+
for (const [side, e] of [["head", h], ["base", b]]) for (const { cell, ...rate } of e?.rates ?? []) {
|
|
357
|
+
const row = out.get(cell) ?? {
|
|
358
|
+
key: cell,
|
|
359
|
+
base: null,
|
|
360
|
+
head: null
|
|
361
|
+
};
|
|
362
|
+
row[side] = rate;
|
|
363
|
+
out.set(cell, row);
|
|
364
|
+
}
|
|
365
|
+
return [...out.values()];
|
|
366
|
+
}
|
|
367
|
+
/** The lifts and gated rates of an eval with `cases`, on both sides; unset when it has none. */
|
|
368
|
+
function casesGate(b, h) {
|
|
369
|
+
const lifts = casesLifts(b, h);
|
|
370
|
+
const rates = casesRates(b, h);
|
|
371
|
+
return {
|
|
372
|
+
...lifts.length ? { lifts } : {},
|
|
373
|
+
...rates.length ? { rates } : {}
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
/** The rows of a run with nothing to compare it with: every eval is new. */
|
|
377
|
+
const newRows = (head) => head.evals.map(addedRow);
|
|
378
|
+
/** Each eval's row, an eval with `cases` with a row for each of its cases. */
|
|
379
|
+
function compare(base, head) {
|
|
380
|
+
const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
|
|
381
|
+
return evalRows(base.evals, head.evals, graderChanged);
|
|
382
|
+
}
|
|
383
|
+
/** A case's outcome as its eval's verdict reads it: a failure its eval's rate gates passes. */
|
|
384
|
+
const caseOutcome = (o, verdict) => verdict === "family" && o === "fail" ? "pass" : o;
|
|
310
385
|
/**
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
386
|
+
* An eval with `cases` in both runs. Over the same cases, its outcome is the eval's; over
|
|
387
|
+
* different ones, the worst of the cases both ran, as their eval reads them, unless its cases
|
|
388
|
+
* pool a gate, which a run of other cases can't be compared with. A case that changed or hit an
|
|
389
|
+
* infrastructure error makes the eval `drift` or `infra` too, and a case that regressed outside
|
|
390
|
+
* a pooled gate makes it `regressed`.
|
|
316
391
|
*/
|
|
317
|
-
function
|
|
318
|
-
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
lifts
|
|
392
|
+
function casesRow(b, h, graderChanged, row) {
|
|
393
|
+
const caseRows = evalRows(caseEvals(b), caseEvals(h), graderChanged);
|
|
394
|
+
const shared = caseRows.filter((r) => r.base !== null && r.head !== null);
|
|
395
|
+
const added = caseRows.filter((r) => r.base === null).map((r) => r.id);
|
|
396
|
+
const removed = caseRows.filter((r) => r.head === null).map((r) => r.id);
|
|
397
|
+
const out = {
|
|
398
|
+
...row,
|
|
399
|
+
cases: caseRows,
|
|
400
|
+
...casesGate(b, h)
|
|
327
401
|
};
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
402
|
+
const changes = [.../* @__PURE__ */ new Set([...row.changes ?? [], ...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])])];
|
|
403
|
+
const failing = caseRows.filter((r) => r.base === null && caseOutcome(r.head, r.verdict) === "fail" && !r.canceled?.includes("head"));
|
|
404
|
+
const over = (side) => side === "head" ? [...shared, ...failing] : shared;
|
|
405
|
+
if (added.length || removed.length) {
|
|
406
|
+
if (pooledGate(b) || pooledGate(h) || !shared.length) changes.push(...added.map((k) => `case ${k} added`), ...removed.map((k) => `case ${k} removed`));
|
|
407
|
+
else {
|
|
408
|
+
const worst = (side) => outcomeOf(over(side).map((r) => ({
|
|
409
|
+
gate: "all",
|
|
410
|
+
outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
|
|
411
|
+
})));
|
|
412
|
+
out.base = worst("base");
|
|
413
|
+
out.head = worst("head");
|
|
414
|
+
out.caseScope = {
|
|
415
|
+
added,
|
|
416
|
+
removed
|
|
417
|
+
};
|
|
418
|
+
changes.push(...failing.map((r) => `case ${r.id} added`));
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
const variants = liftScope(b, h);
|
|
422
|
+
if (variants) {
|
|
423
|
+
out.cells = {
|
|
424
|
+
added: variants.added,
|
|
425
|
+
removed: variants.removed
|
|
426
|
+
};
|
|
427
|
+
if (!variants.shared.length) return {
|
|
428
|
+
...out,
|
|
429
|
+
...changes.length ? { changes } : {},
|
|
430
|
+
status: "drift"
|
|
431
|
+
};
|
|
432
|
+
const worst = (side, e) => outcomeOf([
|
|
433
|
+
...(shared.length ? over(side) : caseRows).map((r) => ({
|
|
434
|
+
gate: "all",
|
|
435
|
+
outcome: caseOutcome(r[side], r.verdict) ?? "skipped"
|
|
436
|
+
})),
|
|
437
|
+
...(e.lifts ?? []).filter((l) => variants.shared.includes(l.variant)).map((l) => ({
|
|
438
|
+
gate: "all",
|
|
439
|
+
outcome: l.outcome
|
|
440
|
+
})),
|
|
441
|
+
...(e.rates ?? []).filter((r) => !variants.added.includes(r.cell) && !variants.removed.includes(r.cell)).map((r) => ({
|
|
442
|
+
gate: "all",
|
|
443
|
+
outcome: r.outcome
|
|
444
|
+
}))
|
|
445
|
+
]);
|
|
446
|
+
out.base = worst("base", b);
|
|
447
|
+
out.head = worst("head", h);
|
|
448
|
+
}
|
|
449
|
+
if (changes.length) out.changes = changes;
|
|
450
|
+
if (out.base === "infra_error" || out.head === "infra_error") return {
|
|
451
|
+
...out,
|
|
452
|
+
status: "infra"
|
|
344
453
|
};
|
|
345
|
-
const prefix = id.slice(0, id.lastIndexOf(" › ") + 3);
|
|
346
|
-
const name = (caseId) => caseId.startsWith(prefix) ? caseId.slice(prefix.length) : caseId;
|
|
347
|
-
const before = new Set(b.caseIds);
|
|
348
|
-
const after = new Set(h.caseIds);
|
|
349
|
-
const caseRows = [.../* @__PURE__ */ new Set([...h.caseIds, ...b.caseIds])].flatMap((c) => ctx.rows.get(c) ?? []);
|
|
350
|
-
const changes = [.../* @__PURE__ */ new Set([
|
|
351
|
-
...h.caseIds.filter((c) => !before.has(c)).map((c) => `case ${name(c)} added`),
|
|
352
|
-
...b.caseIds.filter((c) => !after.has(c)).map((c) => `case ${name(c)} removed`),
|
|
353
|
-
...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])
|
|
354
|
-
])];
|
|
355
|
-
const canceled = ["base", "head"].filter((s) => row[s] === "fail" && ctx.canceledIn(s, s === "base" ? b.caseIds : h.caseIds));
|
|
356
|
-
if (canceled.length) row.canceled = canceled;
|
|
357
454
|
if (caseRows.some((r) => r.status === "infra")) return {
|
|
358
|
-
...
|
|
455
|
+
...out,
|
|
359
456
|
status: "infra"
|
|
360
457
|
};
|
|
361
|
-
if (changes.
|
|
362
|
-
...
|
|
363
|
-
status: "drift",
|
|
364
|
-
changes
|
|
365
|
-
};
|
|
366
|
-
if (disjoint || row.canceled || caseRows.some((r) => r.status === "drift")) return {
|
|
367
|
-
...row,
|
|
458
|
+
if (changes.some((c) => c !== "gate changed") || out.canceled || caseRows.some((r) => r.status === "drift" || r.canceled)) return {
|
|
459
|
+
...out,
|
|
368
460
|
status: "drift"
|
|
369
461
|
};
|
|
370
|
-
|
|
371
|
-
|
|
462
|
+
const from = out.gate?.rescored ?? out.base;
|
|
463
|
+
if (graderChanged && from !== out.head) return {
|
|
464
|
+
...out,
|
|
372
465
|
status: "drift"
|
|
373
466
|
};
|
|
374
|
-
|
|
375
|
-
|
|
467
|
+
const caseRegressed = caseRows.some((r) => r.status === "regressed" && r.verdict !== "family");
|
|
468
|
+
if (from === "pass" && out.head === "fail" || caseRegressed) return {
|
|
469
|
+
...out,
|
|
376
470
|
status: "regressed"
|
|
377
471
|
};
|
|
378
|
-
if (
|
|
379
|
-
...
|
|
472
|
+
if (from === "fail" && out.head === "pass") return {
|
|
473
|
+
...out,
|
|
380
474
|
status: "fixed"
|
|
381
475
|
};
|
|
382
476
|
return {
|
|
383
|
-
...
|
|
384
|
-
status: "same"
|
|
385
|
-
};
|
|
386
|
-
}
|
|
387
|
-
/** Each eval's row, then each family's that either run gated. */
|
|
388
|
-
function compare(base, head) {
|
|
389
|
-
const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
|
|
390
|
-
const rows = evalRows(base, head, graderChanged);
|
|
391
|
-
const before = new Map((base.families ?? []).map((f) => [f.id, f]));
|
|
392
|
-
const after = new Map((head.families ?? []).map((f) => [f.id, f]));
|
|
393
|
-
const families = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((id) => gated(before.get(id)) || gated(after.get(id)));
|
|
394
|
-
const runs = {
|
|
395
|
-
base,
|
|
396
|
-
head
|
|
477
|
+
...out,
|
|
478
|
+
status: out.base === out.head ? "same" : "gate"
|
|
397
479
|
};
|
|
398
|
-
const ctx = {
|
|
399
|
-
rows: new Map(rows.map((r) => [r.id, r])),
|
|
400
|
-
graderChanged,
|
|
401
|
-
canceledIn: (side, caseIds) => runs[side].evals.some((e) => caseIds.includes(e.id) && e.cells.some((c) => (c.canceled ?? 0) > 0))
|
|
402
|
-
};
|
|
403
|
-
return [...rows, ...families.map((id) => familyRow(id, before.get(id), after.get(id), ctx))];
|
|
404
480
|
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
const b = base.evals.find((e) => e.id === id);
|
|
410
|
-
const h = head.evals.find((e) => e.id === id);
|
|
481
|
+
function evalRows(baseEvals, headEvals, graderChanged) {
|
|
482
|
+
return [.../* @__PURE__ */ new Set([...headEvals.map((e) => e.id), ...baseEvals.map((e) => e.id)])].map((id) => {
|
|
483
|
+
const b = baseEvals.find((e) => e.id === id);
|
|
484
|
+
const h = headEvals.find((e) => e.id === id);
|
|
411
485
|
if (!b) return addedRow(h);
|
|
412
|
-
if (!h) return
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
head: null,
|
|
416
|
-
status: "removed",
|
|
417
|
-
...canceledSides({
|
|
418
|
-
base: b.outcome,
|
|
419
|
-
head: null
|
|
420
|
-
}, { base: b.cells })
|
|
421
|
-
};
|
|
422
|
-
const comparable = sameEval(b, h);
|
|
486
|
+
if (!h) return removedRow(b);
|
|
487
|
+
const withCases = !!(b.cases || h.cases);
|
|
488
|
+
const comparable = withCases || sameEval(b, h);
|
|
423
489
|
const scoped = comparable ? null : sharedScope(b, h);
|
|
424
490
|
const outcomes = scoped ? {
|
|
425
491
|
base: scoped.base,
|
|
@@ -435,19 +501,22 @@ function evalRows(base, head, graderChanged) {
|
|
|
435
501
|
...canceledSides(outcomes, scoped?.over ?? {
|
|
436
502
|
base: b.cells,
|
|
437
503
|
head: h.cells
|
|
438
|
-
})
|
|
504
|
+
}),
|
|
505
|
+
...verdictOf(h)
|
|
439
506
|
};
|
|
440
507
|
const perCell = compareCells(b, h);
|
|
441
508
|
if (perCell.length) row.perCell = perCell;
|
|
442
|
-
if (
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
509
|
+
if (!withCases) {
|
|
510
|
+
if (row.base === "infra_error" || row.head === "infra_error") return {
|
|
511
|
+
...row,
|
|
512
|
+
status: "infra"
|
|
513
|
+
};
|
|
514
|
+
if (!comparable && !scoped) return {
|
|
515
|
+
...row,
|
|
516
|
+
status: "drift",
|
|
517
|
+
changes: whyChanged(b, h)
|
|
518
|
+
};
|
|
519
|
+
}
|
|
451
520
|
const over = scoped?.over ?? {
|
|
452
521
|
base: b.cells ?? [],
|
|
453
522
|
head: h.cells ?? []
|
|
@@ -457,6 +526,7 @@ function evalRows(base, head, graderChanged) {
|
|
|
457
526
|
changes: ["gate changed"],
|
|
458
527
|
gate
|
|
459
528
|
});
|
|
529
|
+
if (withCases) return casesRow(b, h, graderChanged, row);
|
|
460
530
|
const from = gate?.rescored ?? row.base;
|
|
461
531
|
if (row.canceled) return {
|
|
462
532
|
...row,
|
|
@@ -480,186 +550,6 @@ function evalRows(base, head, graderChanged) {
|
|
|
480
550
|
};
|
|
481
551
|
});
|
|
482
552
|
}
|
|
483
|
-
// src/view/effort.ts
|
|
484
|
-
/** Below this two-sided p, an effort difference is an effect. */
|
|
485
|
-
const EFFORT_ALPHA = .05;
|
|
486
|
-
/** Some counted attempts reported no usage. Not a verdict; never printed as one. */
|
|
487
|
-
const TOKENS_UNREPORTED = "some cells did not report usage";
|
|
488
|
-
const effortSample = (a) => ({
|
|
489
|
-
tools: a.activity.tools,
|
|
490
|
-
agentMs: a.timings.agentMs,
|
|
491
|
-
fileEdits: a.activity.fileEdits,
|
|
492
|
-
filesChanged: filesChanged(a.changedFiles),
|
|
493
|
-
...a.usage ? { tokens: totalTokens(a.usage) } : {}
|
|
494
|
-
});
|
|
495
|
-
/** The files the workspace diff changed, a collapsed directory counting each of its files. */
|
|
496
|
-
const filesChanged = (files) => files.reduce((n, f) => n + (f.count ?? 1), 0);
|
|
497
|
-
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
498
|
-
const spread = (values) => ({
|
|
499
|
-
median: median(values) ?? 0,
|
|
500
|
-
q1: quantile(values, .25) ?? 0,
|
|
501
|
-
q3: quantile(values, .75) ?? 0
|
|
502
|
-
});
|
|
503
|
-
/** Token totals from the attempts that reported usage; empty when none did. */
|
|
504
|
-
const tokenValues = (samples) => samples.flatMap((s) => s.tokens === void 0 ? [] : [s.tokens]);
|
|
505
|
-
function effortOf(samples) {
|
|
506
|
-
if (samples.length === 0) return null;
|
|
507
|
-
const tokens = tokenValues(samples);
|
|
508
|
-
return {
|
|
509
|
-
n: samples.length,
|
|
510
|
-
tools: spread(samples.map((s) => s.tools)),
|
|
511
|
-
agentMs: spread(samples.map((s) => s.agentMs)),
|
|
512
|
-
fileEdits: median(samples.map((s) => s.fileEdits)) ?? 0,
|
|
513
|
-
filesChanged: median(samples.map((s) => s.filesChanged)) ?? 0,
|
|
514
|
-
...tokens.length ? { tokens: spread(tokens) } : {}
|
|
515
|
-
};
|
|
516
|
-
}
|
|
517
|
-
/**
|
|
518
|
-
* The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
|
|
519
|
-
* the test's (which side ranks lower), not the medians', which can be equal or disagree.
|
|
520
|
-
*/
|
|
521
|
-
function effortVerdict(xs, ys) {
|
|
522
|
-
const test = mannWhitney(xs, ys);
|
|
523
|
-
const p = test?.p ?? null;
|
|
524
|
-
if (test === null || test.floor >= .05) return {
|
|
525
|
-
p,
|
|
526
|
-
verdict: "insufficient"
|
|
527
|
-
};
|
|
528
|
-
if (test.p >= .05) return {
|
|
529
|
-
p,
|
|
530
|
-
verdict: "noise"
|
|
531
|
-
};
|
|
532
|
-
return {
|
|
533
|
-
p,
|
|
534
|
-
verdict: test.sign < 0 ? "fewer" : "more"
|
|
535
|
-
};
|
|
536
|
-
}
|
|
537
|
-
const ratio = (a, b) => b === 0 ? null : a / b;
|
|
538
|
-
/** Every value on one side is below every value on the other. */
|
|
539
|
-
const apart = (xs, ys) => Math.max(...xs) < Math.min(...ys) || Math.min(...xs) > Math.max(...ys);
|
|
540
|
-
/**
|
|
541
|
-
* The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
|
|
542
|
-
* ten separates by chance; at 1 against 1 every pair of unequal values does.
|
|
543
|
-
*/
|
|
544
|
-
const DIRECTIONAL_MIN = 3;
|
|
545
|
-
/**
|
|
546
|
-
* The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
|
|
547
|
-
*/
|
|
548
|
-
const EFFORT_MIN_RUNS = (() => {
|
|
549
|
-
let n = 1;
|
|
550
|
-
while (minimumP(n, n) >= EFFORT_ALPHA) n++;
|
|
551
|
-
return n;
|
|
552
|
-
})();
|
|
553
|
-
/**
|
|
554
|
-
* The metrics whose test is `insufficient` but whose ranges don't overlap at
|
|
555
|
-
* `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
|
|
556
|
-
*/
|
|
557
|
-
function directional(c) {
|
|
558
|
-
if (Math.min(c.variant.n, c.control.n) < 3) return [];
|
|
559
|
-
const out = [];
|
|
560
|
-
if (c.separated && c.verdict === "insufficient") out.push("tools");
|
|
561
|
-
if (c.timeSeparated && c.timeVerdict === "insufficient") out.push("agentS");
|
|
562
|
-
if (c.tokensSeparated && c.tokensVerdict === "insufficient") out.push("tokens");
|
|
563
|
-
return out;
|
|
564
|
-
}
|
|
565
|
-
function contrastEffort(variant, control) {
|
|
566
|
-
const v = effortOf(variant);
|
|
567
|
-
const c = effortOf(control);
|
|
568
|
-
if (!v || !c) return null;
|
|
569
|
-
const tools = effortVerdict(variant.map((s) => s.tools), control.map((s) => s.tools));
|
|
570
|
-
const time = effortVerdict(variant.map((s) => s.agentMs), control.map((s) => s.agentMs));
|
|
571
|
-
const vTok = tokenValues(variant);
|
|
572
|
-
const cTok = tokenValues(control);
|
|
573
|
-
const tokens = vTok.length && cTok.length ? effortVerdict(vTok, cTok) : null;
|
|
574
|
-
const reported = vTok.length + cTok.length;
|
|
575
|
-
const unreported = reported > 0 && reported < variant.length + control.length;
|
|
576
|
-
return {
|
|
577
|
-
variant: v,
|
|
578
|
-
control: c,
|
|
579
|
-
toolsRatio: ratio(v.tools.median, c.tools.median),
|
|
580
|
-
agentMsRatio: ratio(v.agentMs.median, c.agentMs.median),
|
|
581
|
-
agentMsDelta: v.agentMs.median - c.agentMs.median,
|
|
582
|
-
toolsP: tools.p,
|
|
583
|
-
agentMsP: time.p,
|
|
584
|
-
separated: apart(variant.map((s) => s.tools), control.map((s) => s.tools)),
|
|
585
|
-
timeSeparated: apart(variant.map((s) => s.agentMs), control.map((s) => s.agentMs)),
|
|
586
|
-
tokensSeparated: tokens ? apart(vTok, cTok) : null,
|
|
587
|
-
verdict: tools.verdict,
|
|
588
|
-
timeVerdict: time.verdict,
|
|
589
|
-
tokensRatio: tokens && v.tokens && c.tokens ? ratio(v.tokens.median, c.tokens.median) : null,
|
|
590
|
-
tokensP: tokens?.p ?? null,
|
|
591
|
-
tokensVerdict: tokens?.verdict ?? null,
|
|
592
|
-
...unreported ? { tokensNote: TOKENS_UNREPORTED } : {}
|
|
593
|
-
};
|
|
594
|
-
}
|
|
595
|
-
/** The counted attempts of one cell of an eval, as effort samples. */
|
|
596
|
-
const cellSamples = (e, cellKey) => e.attempts.filter((a) => a.cell === cellKey && counted(a)).map(effortSample);
|
|
597
|
-
const cellEffort = (e, cellKey) => effortOf(cellSamples(e, cellKey));
|
|
598
|
-
/** A variant cell's effort against its control's, in the same run; null for controls. */
|
|
599
|
-
function cellContrast(e, cellKey) {
|
|
600
|
-
const cell = e.cells.find((c) => c.key === cellKey);
|
|
601
|
-
if (!cell?.lift) return null;
|
|
602
|
-
return contrastEffort(cellSamples(e, cellKey), cellSamples(e, cell.lift.vs));
|
|
603
|
-
}
|
|
604
|
-
/** `4`, `4.5`: medians of counts are whole or halves. */
|
|
605
|
-
const formatCount = (n) => Number.isInteger(n) ? String(n) : n.toFixed(1);
|
|
606
|
-
const formatRatio = (r) => r === null ? "×–" : `×${r.toFixed(2)}`;
|
|
607
|
-
/** Below this, agent time rounds under a tenth of a second and prints `<0.1s`. */
|
|
608
|
-
const TENTH_MS = 50;
|
|
609
|
-
/** Two agent times that both print `<0.1s` have no ratio worth printing. */
|
|
610
|
-
const bothUnderATenth = (a, b) => a < TENTH_MS && b < TENTH_MS;
|
|
611
|
-
/** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
|
|
612
|
-
function formatSeconds(ms) {
|
|
613
|
-
if (ms < 9950) return ms < TENTH_MS ? "<0.1s" : `${(ms / 1e3).toFixed(1)}s`;
|
|
614
|
-
const s = Math.round(ms / 1e3);
|
|
615
|
-
return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, "0")}s`;
|
|
616
|
-
}
|
|
617
|
-
const formatP = (p) => p === null ? "" : ` (p = ${p < .001 ? "<0.001" : p.toFixed(3)})`;
|
|
618
|
-
const VERDICT_TEXT = {
|
|
619
|
-
fewer: (what) => `fewer ${what}`,
|
|
620
|
-
more: (what) => `more ${what}`,
|
|
621
|
-
noise: () => "within noise",
|
|
622
|
-
insufficient: () => "too few attempts to test"
|
|
623
|
-
};
|
|
624
|
-
const toolsVerdictText = (c) => c.verdict === "fewer" || c.verdict === "more" ? `${VERDICT_TEXT[c.verdict]("tool calls")}${formatP(c.toolsP)}` : VERDICT_TEXT[c.verdict]("");
|
|
625
|
-
/** Token verdict after the tools one; empty when omitted or already covered as insufficient. */
|
|
626
|
-
const tokensVerdictText = (c) => {
|
|
627
|
-
if (c.tokensVerdict === null) return "";
|
|
628
|
-
if (c.verdict === "insufficient" && c.tokensVerdict === "insufficient") return "";
|
|
629
|
-
if (c.tokensVerdict === "fewer" || c.tokensVerdict === "more") return `; ${VERDICT_TEXT[c.tokensVerdict]("tokens")}${formatP(c.tokensP)}`;
|
|
630
|
-
if (c.tokensVerdict === "noise") return "; tokens within noise";
|
|
631
|
-
return `; ${VERDICT_TEXT[c.tokensVerdict]("")}`;
|
|
632
|
-
};
|
|
633
|
-
/**
|
|
634
|
-
* `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
|
|
635
|
-
* against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
|
|
636
|
-
* when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
|
|
637
|
-
* A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`. Agent
|
|
638
|
-
* times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
|
|
639
|
-
*/
|
|
640
|
-
function effortLabel(c) {
|
|
641
|
-
const [va, ca] = [c.variant.agentMs.median, c.control.agentMs.median];
|
|
642
|
-
const tiny = bothUnderATenth(va, ca);
|
|
643
|
-
const apartOn = new Set(directional(c).filter((m) => !(tiny && m === "agentS")));
|
|
644
|
-
const r = (metric, value) => `(${formatRatio(value)}${apartOn.has(metric) ? ", no overlap" : ""})`;
|
|
645
|
-
const parts = [`tools ${formatCount(c.variant.tools.median)} vs ${formatCount(c.control.tools.median)} ${r("tools", c.toolsRatio)}`, `agent ${formatSeconds(va)} vs ${formatSeconds(ca)}${tiny ? "" : ` ${r("agentS", c.agentMsRatio)}`}`];
|
|
646
|
-
if (c.tokensVerdict !== null && c.variant.tokens && c.control.tokens) parts.push(`tokens ${formatTokens(c.variant.tokens.median)} vs ${formatTokens(c.control.tokens.median)} ${r("tokens", c.tokensRatio)}`);
|
|
647
|
-
const n = apartOn.size ? ` (n = ${c.variant.n} vs ${c.control.n})` : "";
|
|
648
|
-
return `${parts.join(", ")}: ${toolsVerdictText(c)}${tokensVerdictText(c)}${n}`;
|
|
649
|
-
}
|
|
650
|
-
/** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
|
|
651
|
-
const effortSummary = (e) => e.tokens ? `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)} · tokens ${formatTokens(e.tokens.median)}` : `tools ${formatCount(e.tools.median)} · agent ${formatSeconds(e.agentMs.median)}`;
|
|
652
|
-
/** The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`. */
|
|
653
|
-
function effortChangeLabel(base, head) {
|
|
654
|
-
const t = ratio(head.tools.median, base.tools.median);
|
|
655
|
-
const a = ratio(head.agentMs.median, base.agentMs.median);
|
|
656
|
-
const [ba, ha] = [base.agentMs.median, head.agentMs.median];
|
|
657
|
-
const agent = `agent ${formatSeconds(ba)} → ${formatSeconds(ha)}${bothUnderATenth(ba, ha) ? "" : ` (${formatRatio(a)})`}`;
|
|
658
|
-
const core = `tools ${formatCount(base.tools.median)} → ${formatCount(head.tools.median)} (${formatRatio(t)}) · ${agent}`;
|
|
659
|
-
if (!base.tokens || !head.tokens) return core;
|
|
660
|
-
const tok = ratio(head.tokens.median, base.tokens.median);
|
|
661
|
-
return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
|
|
662
|
-
}
|
|
663
553
|
// src/view/model.ts
|
|
664
554
|
/** Splits `file › group › … › name` back into its parts. */
|
|
665
555
|
function splitEvalId(id) {
|
|
@@ -782,7 +672,7 @@ function activityLabel(a, changed) {
|
|
|
782
672
|
function runOutcome(run) {
|
|
783
673
|
if (run.status === "interrupted") return "skipped";
|
|
784
674
|
if (run.errors?.length) return "fail";
|
|
785
|
-
const outcomes =
|
|
675
|
+
const outcomes = run.evals.map((e) => e.outcome);
|
|
786
676
|
if (outcomes.includes("fail") || run.status === "budget") return "fail";
|
|
787
677
|
if (outcomes.includes("infra_error")) return "infra_error";
|
|
788
678
|
return "pass";
|
|
@@ -1089,7 +979,7 @@ function gatedPassAt1(cells) {
|
|
|
1089
979
|
/** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
|
|
1090
980
|
function compareView(base, head, records = {}) {
|
|
1091
981
|
const rows = compare(base, head).map((row) => {
|
|
1092
|
-
const evalOf = (run) =>
|
|
982
|
+
const evalOf = (run) => run.evals.find((e) => e.id === row.id);
|
|
1093
983
|
const b = evalOf(base);
|
|
1094
984
|
const h = evalOf(head);
|
|
1095
985
|
const baseKeys = new Set(b?.cells.map((c) => c.key));
|
|
@@ -1152,6 +1042,6 @@ const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
|
|
|
1152
1042
|
const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
|
|
1153
1043
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
1154
1044
|
const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
|
|
1155
|
-
export {
|
|
1045
|
+
export { runRows as A, stackChanges as B, isUngatedMiss as C, reasoningLabel as D, parseTranscript as E, addedRow as F, cellsChanged as I, compare as L, shortSha as M, splitEvalId as N, runLead as O, transcriptItems as P, newRows as R, isNotGatedPass as S, notRunYet as T, whyChanged as V, formatUsage as _, buildIndex as a, indexFiles as b, defaultAttempt as c, evalTree as d, failedJudgeLabels as f, formatPercent as g, formatDuration as h, attemptAnswerRows as i, runtimeLabel as j, runOutcome as k, evalHistory as l, findEval as m, activityLabel as n, cellRows as o, findAttempt as p, answerRows as r, compareView as s, COMPARE_ORDER as t, evalMarkOutcome as u, harnessLabel as v, languageLabel as w, isMatrix as x, hasUngatedCells as y, sourceChanges as z };
|
|
1156
1046
|
|
|
1157
|
-
//# sourceMappingURL=model-
|
|
1047
|
+
//# sourceMappingURL=model-V3mMXp6x.js.map
|