reeval 0.17.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-DyvDj4ZN.js → adapter-C8xPuXYq.js} +5 -4
- package/dist/adapter-C8xPuXYq.js.map +1 -0
- package/dist/{agent-v036GEgG.js → agent-CBoLndM7.js} +30 -10
- package/dist/{agent-v036GEgG.js.map → agent-CBoLndM7.js.map} +1 -1
- package/dist/agents-C6zhkxp3.js +1151 -0
- package/dist/agents-C6zhkxp3.js.map +1 -0
- package/dist/{analysis-adKPz-UJ.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-adKPz-UJ.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-hpRT5lj_.js → app-files-DT5mJyV4.js} +4 -4
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-UPCRrtOz.js +1 -0
- package/dist/{assemble-Dlh6Xv3U.js → assemble-t6h018An.js} +194 -307
- package/dist/assemble-t6h018An.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-CAdlwlNu.js} +3 -3
- package/dist/{attempt-CtKGvaQR.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-DErH-DYV.js → attempt-host-CIm1-11m.js} +18 -10
- package/dist/{attempt-host-DErH-DYV.js.map → attempt-host-CIm1-11m.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-CJCtzfB1.js +322 -0
- package/dist/boat-CJCtzfB1.js.map +1 -0
- package/dist/bridge/index.mjs +300 -78
- package/dist/capture-DoxFUN3J.js +1 -0
- package/dist/{capture-Bdcx9Gap.js → capture-iBzFiC0j.js} +406 -625
- package/dist/capture-iBzFiC0j.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-CGd51thd.js → cli-20Fpvs5H.js} +929 -576
- package/dist/cli-20Fpvs5H.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-Dsr79Qy1.js} +46 -18
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-KM4C5r5_.js} +111 -14
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-CsfcwMyc.js → command-B5OyHpOu.js} +24 -81
- package/dist/command-B5OyHpOu.js.map +1 -0
- package/dist/{command-CtTdCF3B.js → command-BLNVDiIg.js} +10 -10
- package/dist/{command-CtTdCF3B.js.map → command-BLNVDiIg.js.map} +1 -1
- package/dist/{command-CyIb9BSC.js → command-CRh-cb6Z.js} +120 -55
- package/dist/command-CRh-cb6Z.js.map +1 -0
- package/dist/{command-TLmY-rYp.js → command-aUJvENwy.js} +18 -18
- package/dist/command-aUJvENwy.js.map +1 -0
- package/dist/{commands-C54S45d7.js → commands-CWy-WndG.js} +239 -92
- package/dist/commands-CWy-WndG.js.map +1 -0
- package/dist/{commands-jO9Jj35r.js → commands-CfdBX-mc.js} +1 -1
- package/dist/{config-ImY6AlRE.js → config-B9mB1kF5.js} +924 -1729
- package/dist/config-B9mB1kF5.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-Bkv3ndjq.js} +3 -2
- package/dist/context-Bkv3ndjq.js.map +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BdRgySQl.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-Bw6-pJjR.js → cursor-catalog-BSn9xjmo.js} +3 -3
- package/dist/{cursor-catalog-Bw6-pJjR.js.map → cursor-catalog-BSn9xjmo.js.map} +1 -1
- package/dist/db-P2fXw4nx.js +2 -0
- package/dist/{db-sHGpaLo9.js → db-zgC1tvKB.js} +178 -74
- package/dist/db-zgC1tvKB.js.map +1 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-fpdbYVKm.js → default-effort-BAy8ixd1.js} +1 -1
- package/dist/{default-effort-BZRa-SQP.js → default-effort-DGhQmu95.js} +8 -5
- package/dist/default-effort-DGhQmu95.js.map +1 -0
- package/dist/{dist-ByYk_wAs.js → dist-CAFD1yfv.js} +28 -6
- package/dist/dist-CAFD1yfv.js.map +1 -0
- package/dist/{docker-CWuSRj7a.js → docker-Cqw-CyWz.js} +4 -4
- package/dist/{docker-CWuSRj7a.js.map → docker-Cqw-CyWz.js.map} +1 -1
- package/dist/{docker--2ytOHcu.js → docker-DAMnm3L9.js} +103 -73
- package/dist/docker-DAMnm3L9.js.map +1 -0
- package/dist/{docker-CIzHzwlC.js → docker-Dr4XeDc8.js} +1 -1
- package/dist/{doctor-eE65YpEv.js → doctor-B2PohneB.js} +86 -29
- package/dist/doctor-B2PohneB.js.map +1 -0
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Cy3FVLR5.js → environment-DB8o3hGi.js} +80 -40
- package/dist/environment-DB8o3hGi.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-BTsTRiBW.js → eval-source-D3sbAzt-.js} +23 -15
- package/dist/eval-source-D3sbAzt-.js.map +1 -0
- package/dist/evaluate-COdzf7Hl.d.ts +92 -0
- package/dist/{evaluate-CFFdyYZX.js → evaluate-CwKeN_fo.js} +472 -159
- package/dist/evaluate-CwKeN_fo.js.map +1 -0
- package/dist/files-B4KQUiM7.js +1 -0
- package/dist/files-BEwfjD8J.js +64 -0
- package/dist/files-BEwfjD8J.js.map +1 -0
- package/dist/{files-CQU08uh4.js → files-BRzV7aPh.js} +5 -5
- package/dist/{files-CQU08uh4.js.map → files-BRzV7aPh.js.map} +1 -1
- package/dist/{files-CIX-pLl4.js → files-BiFU16BW.js} +6 -6
- package/dist/{files-CIX-pLl4.js.map → files-BiFU16BW.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-C3mCC8Mn.js +1 -0
- package/dist/{files-tnMgu7-X.js → files-Cl7BQ5gE.js} +55 -14
- package/dist/{files-tnMgu7-X.js.map → files-Cl7BQ5gE.js.map} +1 -1
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/files-DnlA1A7N.js +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BWVkG4Yl.js → git-fDiJ8UIV.js} +3 -3
- package/dist/{git-BWVkG4Yl.js.map → git-fDiJ8UIV.js.map} +1 -1
- package/dist/harness-C2berMiM.js +105 -0
- package/dist/harness-C2berMiM.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DXv2gnaT.js} +68 -111
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-BVZHDf_z.js → images-BNgcR77K.js} +98 -10
- package/dist/images-BNgcR77K.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/init-4RPeD-K-.js +1 -0
- package/dist/{init-BWnMbUnc.js → init-DbuZVRrW.js} +8 -7
- package/dist/init-DbuZVRrW.js.map +1 -0
- package/dist/ink-list-BpfidQ1h.js +274 -0
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-BGRs1r_C.js → invocations-CxGNOlh0.js} +4 -4
- package/dist/{invocations-BGRs1r_C.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-CzgpsTqv.js +246 -0
- package/dist/judgments-CzgpsTqv.js.map +1 -0
- package/dist/kinds-C1oY3qyC.js +697 -0
- package/dist/kinds-C1oY3qyC.js.map +1 -0
- package/dist/{summary-zP3cCXDN.js → layout-Cm85tQYk.js} +77 -154
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/live-CJCK11nh.js +1 -0
- package/dist/{live-d_5a5iUv.js → live-CgYbO4Te.js} +24 -4
- package/dist/live-CgYbO4Te.js.map +1 -0
- package/dist/{live-ColieOai.js → live-CnGrRM6j.js} +8 -8
- package/dist/{live-ColieOai.js.map → live-CnGrRM6j.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-CXLn-1jm.js} +4 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-DOXnWH65.js → model-047Dcd-a.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-CFSjV-HJ.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-DBpdRXtO.js → model-DJjLw-0r.js} +43 -19
- package/dist/{model-DBpdRXtO.js.map → model-DJjLw-0r.js.map} +1 -1
- package/dist/{model-DOiCiMCD.js → model-V3mMXp6x.js} +213 -559
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{node-CwP1G178.js → node-D7pAU--0.js} +34 -25
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-BGiXt8_B.js} +107 -49
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-CHubslQf.js +1 -0
- package/dist/{gate-BoDKeuJu.js → pairs-BGb134Vu.js} +3 -49
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-DvLr8wCZ.js → patch-DH4e_Gkd.js} +3 -3
- package/dist/{patch-DvLr8wCZ.js.map → patch-DH4e_Gkd.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-CIfOFhth.js → plan-Cgq9NCEX.js} +117 -45
- package/dist/plan-Cgq9NCEX.js.map +1 -0
- package/dist/{plugins-BEs2Am3L.js → plugins-BkmPxDiY.js} +11 -268
- package/dist/plugins-BkmPxDiY.js.map +1 -0
- package/dist/preflight-4gWSy0Pe.js +1 -0
- package/dist/{preflight-50Qw5e5K.js → preflight-BElE_q0u.js} +16 -9
- package/dist/preflight-BElE_q0u.js.map +1 -0
- package/dist/preflight-CtlloeOQ.js +68 -0
- package/dist/preflight-CtlloeOQ.js.map +1 -0
- package/dist/{preflight-BAIRsTpl.js → preflight-R-InZCQb.js} +1 -1
- package/dist/{prepare-8XhzhLHX.js → prepare-C7f5mPl5.js} +9 -9
- package/dist/{prepare-8XhzhLHX.js.map → prepare-C7f5mPl5.js.map} +1 -1
- package/dist/{prepare-8nvfDFQx.js → prepare-CzkAdbXm.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-5m3SzMZ3.js} +3 -3
- package/dist/{prices-C4BRuSe8.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-Cc8fFE7Z.js → provenance-zsc4--dP.js} +6 -5
- package/dist/provenance-zsc4--dP.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-DZSTlC9R.js → providers-B8xYMeah.js} +152 -9
- package/dist/providers-B8xYMeah.js.map +1 -0
- package/dist/providers-CVu8e0xS.js +1 -0
- package/dist/{query-DXpxaaTH.js → query-Bb3SYICk.js} +3 -3
- package/dist/{query-DXpxaaTH.js.map → query-Bb3SYICk.js.map} +1 -1
- package/dist/query-DIfkUcPh.js +1 -0
- package/dist/{questions-C_Y-oXW6.js → questions-CWlR9ffu.js} +41 -11
- package/dist/questions-CWlR9ffu.js.map +1 -0
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/remarks-DPCONFxZ.js +138 -0
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remarks-E4rpi8Ym.js → remarks-vtkPmQb9.js} +327 -41
- package/dist/remarks-vtkPmQb9.js.map +1 -0
- package/dist/{remote-DZ00A8V_.js → remote-BgYWqogI.js} +6 -6
- package/dist/{remote-DZ00A8V_.js.map → remote-BgYWqogI.js.map} +1 -1
- package/dist/{resolve-B_JlQ0La.js → resolve-Do0ZaYH6.js} +11 -10
- package/dist/resolve-Do0ZaYH6.js.map +1 -0
- package/dist/resolve-amw_2Odz.js +1 -0
- package/dist/{results-BMJHaQbi.js → results-JyuM64gy.js} +7 -7
- package/dist/{results-BMJHaQbi.js.map → results-JyuM64gy.js.map} +1 -1
- package/dist/resume-AW5oXzUc.js +691 -0
- package/dist/resume-AW5oXzUc.js.map +1 -0
- package/dist/retry-ChzISxmy.js +105 -0
- package/dist/retry-ChzISxmy.js.map +1 -0
- package/dist/{runtime-CSZ1hjlN.js → runtime-BpW1f1Mx.js} +26 -7
- package/dist/runtime-BpW1f1Mx.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-B5W8lLL1.js → sandbox-kSgr7vo5.js} +31 -3
- package/dist/sandbox-kSgr7vo5.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/server-2_gUnVvf.js +1 -0
- package/dist/{server-BfN6q2lH.js → server-Cn8VqO9S.js} +10 -10
- package/dist/{server-BfN6q2lH.js.map → server-Cn8VqO9S.js.map} +1 -1
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/source-CY3etKzw.js +1 -0
- package/dist/{source-BD6Bg0Tr.js → source-CYOhR98B.js} +2 -2
- package/dist/{source-BD6Bg0Tr.js.map → source-CYOhR98B.js.map} +1 -1
- package/dist/src-6Zzyg4Qd.js +2 -0
- package/dist/{src-BC2WXULN.js → src-BDhcPtnI.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BRMsk9nh.js +2 -0
- package/dist/{src-Dp-fWOfG.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-Dp-fWOfG.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-Cpy9MNS3.js → src-Ck2r_EMT.js} +617 -74
- package/dist/src-Ck2r_EMT.js.map +1 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/src-DB7N8TYJ.js +2 -0
- package/dist/src-DPDaAYwh.js +2 -0
- package/dist/{src-Cx0HJIJ-.js → src-Dbk19BiP.js} +2 -2
- package/dist/src-DtodMT59.js +1 -0
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/src-T-OeuJLr.js +2 -0
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-zntEDlnx.js +3 -0
- package/dist/{state-CdSjXIfE.js → state-BOLt8pJg.js} +17 -17
- package/dist/{state-CdSjXIfE.js.map → state-BOLt8pJg.js.map} +1 -1
- package/dist/state-kimtIhBP.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/suite-COglUhPd.js +1272 -0
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DU5zaibx.js +148 -0
- package/dist/summary-DU5zaibx.js.map +1 -0
- package/dist/tart-BTVXp8rz.js +373 -0
- package/dist/tart-BTVXp8rz.js.map +1 -0
- package/dist/tart-cli-BIC9HqPg.js +37 -0
- package/dist/tart-cli-BIC9HqPg.js.map +1 -0
- package/dist/tart-cli-BMrkmBjI.js +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +47 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-C_IpYMRk.js → transcript-CCf07UmU.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-BHR1lGPh.d.ts → types-DBwFsGem.d.ts} +420 -136
- package/dist/{judge-ByEUlzOM.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-6uSpZ5L8.js → vertex-Eb2wqahk.js} +5 -2
- package/dist/vertex-Eb2wqahk.js.map +1 -0
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +238 -90
- package/dist/view/index.js +11 -9
- package/dist/{views-CKoN5NAx.js → views-DAIiLiV3.js} +8 -7
- package/dist/views-DAIiLiV3.js.map +1 -0
- package/dist/views-DtRhj2Tn.js +2 -0
- package/dist/{worker-entry-D5dB8cgT.js → worker-entry-BIROTPX1.js} +11 -8
- package/dist/{worker-entry-D5dB8cgT.js.map → worker-entry-BIROTPX1.js.map} +1 -1
- package/dist/workspace-CcQRT31C.js +1 -0
- package/dist/{workspace-Cho30qlM.js → workspace-uuE51x35.js} +7 -7
- package/dist/{workspace-Cho30qlM.js.map → workspace-uuE51x35.js.map} +1 -1
- package/package.json +26 -11
- package/skills/eval-rubric/SKILL.md +31 -4
- package/dist/adapter-DyvDj4ZN.js.map +0 -1
- package/dist/app-files-hpRT5lj_.js.map +0 -1
- package/dist/assemble-DTXwCL3m.js +0 -1
- package/dist/assemble-Dlh6Xv3U.js.map +0 -1
- package/dist/capture-BT2fAmpZ.js +0 -1
- package/dist/capture-Bdcx9Gap.js.map +0 -1
- package/dist/cli-CGd51thd.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-CsfcwMyc.js.map +0 -1
- package/dist/command-CyIb9BSC.js.map +0 -1
- package/dist/command-TLmY-rYp.js.map +0 -1
- package/dist/commands-C54S45d7.js.map +0 -1
- package/dist/config-ImY6AlRE.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-DtSX8E_V.js +0 -101
- package/dist/contrasts-DtSX8E_V.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-ZIk6S2CD.js +0 -2
- package/dist/db-sHGpaLo9.js.map +0 -1
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/default-effort-BZRa-SQP.js.map +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker--2ytOHcu.js.map +0 -1
- package/dist/doctor-eE65YpEv.js.map +0 -1
- package/dist/environment-Cy3FVLR5.js.map +0 -1
- package/dist/eval-source-BTsTRiBW.js.map +0 -1
- package/dist/evaluate-CFFdyYZX.js.map +0 -1
- package/dist/evaluate-DnGOaGQc.d.ts +0 -43
- package/dist/files-B3d-8Fe0.js +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-Do4Rp3yZ.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-BoDKeuJu.js.map +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-BVZHDf_z.js.map +0 -1
- package/dist/init-BWnMbUnc.js.map +0 -1
- package/dist/init-s1jN-PCI.js +0 -1
- package/dist/judge-ByEUlzOM.js.map +0 -1
- package/dist/live-DmJix4c8.js +0 -1
- package/dist/live-d_5a5iUv.js.map +0 -1
- package/dist/model-8uZBYqV5.js +0 -1
- package/dist/model-DOiCiMCD.js.map +0 -1
- package/dist/node-CwP1G178.js.map +0 -1
- package/dist/opencode-providers-yNKyXD49.js.map +0 -1
- package/dist/plan-CIfOFhth.js.map +0 -1
- package/dist/plugins-BEs2Am3L.js.map +0 -1
- package/dist/preflight-50Qw5e5K.js.map +0 -1
- package/dist/provenance-Cc8fFE7Z.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-CP_bBtYm.js +0 -1
- package/dist/providers-DZSTlC9R.js.map +0 -1
- package/dist/query-kFPuqBv1.js +0 -1
- package/dist/questions-C_Y-oXW6.js.map +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-DeZZT8PK.js +0 -116
- package/dist/remarks-DeZZT8PK.js.map +0 -1
- package/dist/remarks-E4rpi8Ym.js.map +0 -1
- package/dist/resolve-BJSay360.js +0 -1
- package/dist/resolve-B_JlQ0La.js.map +0 -1
- package/dist/resume-CexsWuGK.js +0 -434
- package/dist/resume-CexsWuGK.js.map +0 -1
- package/dist/runtime-CSZ1hjlN.js.map +0 -1
- package/dist/sandbox-B5W8lLL1.js.map +0 -1
- package/dist/server-SBzrf35T.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-Dn1uqPon.js +0 -1
- package/dist/src-BKDZO3ra.js +0 -2
- package/dist/src-BxrHipV0.js +0 -3
- package/dist/src-C9elednc.js +0 -2
- package/dist/src-CM7xdJFA.js +0 -2
- package/dist/src-CWEGknZo.js +0 -2
- package/dist/src-Cpy9MNS3.js.map +0 -1
- package/dist/src-DVaAKCsH.js +0 -1
- package/dist/src-DurP--_D.js +0 -2
- package/dist/state-Cicc9Oa5.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BYJ2Q0S-.js +0 -2
- package/dist/suite-BtHPBFYL.js +0 -544
- package/dist/suite-BtHPBFYL.js.map +0 -1
- package/dist/summary-zP3cCXDN.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CKoN5NAx.js.map +0 -1
- package/dist/views-CmyX255j.js +0 -2
- package/dist/workspace-Dw9DEZox.js +0 -1
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { a as VERCEL_KEY, d as OPENROUTER_KEY, l as vercelCredentialProblem, o as VERCEL_OIDC } from "./vercel-CGlSBv63.js";
|
|
2
|
+
import { v as vertexSettings } from "./vertex-Eb2wqahk.js";
|
|
3
|
+
import { h as redactSecrets, r as badCharacterAt } from "./redact-BhjtnjrU.js";
|
|
4
|
+
import { setTimeout } from "node:timers/promises";
|
|
5
|
+
// src/judge/retry.ts
|
|
6
|
+
/** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
|
|
7
|
+
const JUDGE_RETRY = {
|
|
8
|
+
attempts: 6,
|
|
9
|
+
baseMs: 2e3,
|
|
10
|
+
maxMs: 3e4
|
|
11
|
+
};
|
|
12
|
+
const TRANSIENT_STATUS = /* @__PURE__ */ new Set([
|
|
13
|
+
408,
|
|
14
|
+
409,
|
|
15
|
+
425,
|
|
16
|
+
429
|
|
17
|
+
]);
|
|
18
|
+
const TRANSIENT_MESSAGE = /high demand|overloaded|unavailable|rate.?limit|resource.?exhausted|try again|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed/i;
|
|
19
|
+
/** The AI SDK wraps the provider's error once it has retried itself; look through that. */
|
|
20
|
+
const innermost = (error) => {
|
|
21
|
+
const e = error;
|
|
22
|
+
return e?.name === "AI_RetryError" && e.lastError ? innermost(e.lastError) : error;
|
|
23
|
+
};
|
|
24
|
+
/** An error from the provider's API (an AI SDK `APICallError`), not one reeval raised itself. */
|
|
25
|
+
const isApiError = (error) => typeof innermost(error)?.statusCode === "number";
|
|
26
|
+
/** Rate limits, 5xx, overload and network blips. Never a bad request, bad key, abort or timeout. */
|
|
27
|
+
function isTransient(error) {
|
|
28
|
+
const e = innermost(error);
|
|
29
|
+
if (!e || e.name === "AbortError" || e.name === "TimeoutError") return false;
|
|
30
|
+
if (typeof e.statusCode === "number") return e.statusCode >= 500 || TRANSIENT_STATUS.has(e.statusCode) || e.isRetryable === true;
|
|
31
|
+
return e.isRetryable === true || TRANSIENT_MESSAGE.test(e.message ?? String(e));
|
|
32
|
+
}
|
|
33
|
+
function retryAfterMs(error) {
|
|
34
|
+
const header = innermost(error)?.responseHeaders?.["retry-after"];
|
|
35
|
+
const seconds = Number(header);
|
|
36
|
+
return Number.isFinite(seconds) && seconds > 0 ? seconds * 1e3 : 0;
|
|
37
|
+
}
|
|
38
|
+
/** Exponential backoff with equal jitter, at least any `retry-after`, never above `maxMs`. */
|
|
39
|
+
function backoffMs(policy, retry, error) {
|
|
40
|
+
const cap = Math.min(policy.maxMs, policy.baseMs * 2 ** retry);
|
|
41
|
+
const jittered = cap / 2 + Math.random() * (cap / 2);
|
|
42
|
+
return Math.round(Math.min(policy.maxMs, Math.max(jittered, retryAfterMs(error))));
|
|
43
|
+
}
|
|
44
|
+
/** Runs `call` until it succeeds, fails for good, or runs out of tries. */
|
|
45
|
+
async function withRetries(call, policy, signal) {
|
|
46
|
+
const retries = [];
|
|
47
|
+
for (;;) try {
|
|
48
|
+
return {
|
|
49
|
+
value: await call(),
|
|
50
|
+
retries
|
|
51
|
+
};
|
|
52
|
+
} catch (error) {
|
|
53
|
+
if (signal.aborted || retries.length + 1 >= policy.attempts || !isTransient(error)) {
|
|
54
|
+
const failed = error instanceof Error ? error : new Error(String(error));
|
|
55
|
+
failed.retries = retries;
|
|
56
|
+
throw failed;
|
|
57
|
+
}
|
|
58
|
+
const delayMs = backoffMs(policy, retries.length, error);
|
|
59
|
+
const e = innermost(error);
|
|
60
|
+
retries.push({
|
|
61
|
+
error: e?.message ?? String(e),
|
|
62
|
+
delayMs
|
|
63
|
+
});
|
|
64
|
+
await setTimeout(delayMs, void 0, { signal });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
// src/judge/index.ts
|
|
68
|
+
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
69
|
+
const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
|
|
70
|
+
/** The same judge as OpenRouter names it. */
|
|
71
|
+
const OPENROUTER_JUDGE_MODEL = "typesafe/jev-1.14";
|
|
72
|
+
/** The language model `reeval init` sets as `analysis.model` and `reeval analyze` uses. */
|
|
73
|
+
const RECOMMENDED_ANALYSIS_MODEL = "google/gemini-3.8-flash";
|
|
74
|
+
const DEFAULT_THRESHOLD = .8;
|
|
75
|
+
/** Each routing gateway's keys, in the order reeval prefers them. */
|
|
76
|
+
const ROUTED_KEYS = {
|
|
77
|
+
vercel: [VERCEL_KEY, VERCEL_OIDC],
|
|
78
|
+
openrouter: [OPENROUTER_KEY]
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* Providers `direct()` calls, each through its own AI SDK package and key. Any other prefix needs
|
|
82
|
+
* `vercel()` in the config's gateway list.
|
|
83
|
+
*/
|
|
84
|
+
const DIRECT_PROVIDERS = {
|
|
85
|
+
anthropic: {
|
|
86
|
+
key: "ANTHROPIC_API_KEY",
|
|
87
|
+
pkg: "@ai-sdk/anthropic",
|
|
88
|
+
name: "Anthropic"
|
|
89
|
+
},
|
|
90
|
+
google: {
|
|
91
|
+
key: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
92
|
+
pkg: "@ai-sdk/google",
|
|
93
|
+
name: "Google"
|
|
94
|
+
},
|
|
95
|
+
openai: {
|
|
96
|
+
key: "OPENAI_API_KEY",
|
|
97
|
+
pkg: "@ai-sdk/openai",
|
|
98
|
+
name: "OpenAI"
|
|
99
|
+
},
|
|
100
|
+
"typesafe-ai": {
|
|
101
|
+
key: "TYPESAFE_AI_API_KEY",
|
|
102
|
+
pkg: "@ai-sdk/typesafe-ai",
|
|
103
|
+
name: "TypeSafe AI"
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
const isFakeJudge = (m) => typeof m === "object" && m !== null && m.kind === "fake-judge";
|
|
107
|
+
function judgeModelName(model) {
|
|
108
|
+
if (typeof model === "string") return model;
|
|
109
|
+
if (isFakeJudge(model)) return "fake";
|
|
110
|
+
const m = model;
|
|
111
|
+
return m.provider && m.modelId ? `${m.provider}/${m.modelId}` : "custom";
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Checks a model string's shape; returns an error message naming it as `what` (with where it was
|
|
115
|
+
* set), or undefined when it's valid.
|
|
116
|
+
*/
|
|
117
|
+
function checkModelString(model, what = "The judge model", examples = [RECOMMENDED_JUDGE_MODEL, `google/${model}`]) {
|
|
118
|
+
if (model.includes("/")) return void 0;
|
|
119
|
+
return `${what} needs a provider prefix, e.g. ${examples.map((e) => `"${e}"`).join(" or ")} (got "${model}")`;
|
|
120
|
+
}
|
|
121
|
+
const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
122
|
+
/**
|
|
123
|
+
* Environment variables a model needs on `gateway`, in the order reeval prefers them; any one is
|
|
124
|
+
* enough. On direct, its provider's own key (none for a prefix reeval can't call directly); on
|
|
125
|
+
* vertex none, since its token comes from Application Default Credentials; on another gateway,
|
|
126
|
+
* that gateway's.
|
|
127
|
+
*/
|
|
128
|
+
function judgeCredentials(model, gateway = "direct") {
|
|
129
|
+
if (isFakeJudge(model) || gateway === "vertex") return [];
|
|
130
|
+
if (gateway !== "direct") return [...ROUTED_KEYS[gateway]];
|
|
131
|
+
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
132
|
+
return direct ? [direct.key] : [];
|
|
133
|
+
}
|
|
134
|
+
/** The keys that would work for a model on `on`: `ANTHROPIC_API_KEY (for direct())`. */
|
|
135
|
+
function modelKeyOptions(model, on = "direct") {
|
|
136
|
+
if (on === "vertex") return "Application Default Credentials (for vertex())";
|
|
137
|
+
if (on !== "direct") return `${ROUTED_KEYS[on].join(" or ")} (for ${on}())`;
|
|
138
|
+
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
139
|
+
return direct ? `${direct.key} (for direct())` : `vercel() in gateway (direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models)`;
|
|
140
|
+
}
|
|
141
|
+
/** A gateway's credential set in `env`, if any, with the gateway it's for. */
|
|
142
|
+
function setGatewayKey(env) {
|
|
143
|
+
for (const [gateway, keys] of Object.entries(ROUTED_KEYS)) {
|
|
144
|
+
const key = keys.find((k) => env[k]);
|
|
145
|
+
if (key) return {
|
|
146
|
+
key,
|
|
147
|
+
gateway
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* ` AI_GATEWAY_API_KEY is set, but …` when a gateway's credential is set and the model goes
|
|
153
|
+
* direct, so the key says how to use it.
|
|
154
|
+
*/
|
|
155
|
+
function unusedGatewayKey(env, gateway = "direct") {
|
|
156
|
+
if (gateway !== "direct") return "";
|
|
157
|
+
const set = setGatewayKey(env);
|
|
158
|
+
if (!set) return "";
|
|
159
|
+
const name = set.gateway === "vercel" ? "Vercel AI Gateway" : "OpenRouter";
|
|
160
|
+
return ` ${set.key} is set, but no gateway in the config uses it: add ${set.gateway}() to gateway to call the model through ${name}.`;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* The provider's own package on direct, the Vercel AI Gateway on vercel, or OpenRouter's package
|
|
164
|
+
* on openrouter, or Vertex AI on vertex, whose token is minted when the model is called.
|
|
165
|
+
* Throws, naming the model, its prefix and the keys that would work, when none is
|
|
166
|
+
* set, when the key it would send has whitespace or a line break in it, or when Vercel's project
|
|
167
|
+
* token has expired.
|
|
168
|
+
*/
|
|
169
|
+
function judgeRoute(model, env = process.env, what = "judge model", gateway = "direct") {
|
|
170
|
+
if (gateway === "vertex") return { kind: "vertex" };
|
|
171
|
+
const provider = providerOf(model);
|
|
172
|
+
const direct = DIRECT_PROVIDERS[provider];
|
|
173
|
+
const keys = judgeCredentials(model, gateway);
|
|
174
|
+
if (!keys.length) throw new Error(`${what} "${model}" needs vercel() in the config's gateway list: direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models.`);
|
|
175
|
+
const key = keys.find((k) => env[k]);
|
|
176
|
+
if (key === void 0) throw new Error(`No credentials for ${what} "${model}" (provider "${provider}"). Set ${modelKeyOptions(model, gateway)}, in the environment or .env.local.${unusedGatewayKey(env, gateway)}`);
|
|
177
|
+
if (key === "VERCEL_OIDC_TOKEN") {
|
|
178
|
+
const expired = vercelCredentialProblem(env);
|
|
179
|
+
if (expired) throw new Error(`${what} "${model}": ${expired}`);
|
|
180
|
+
}
|
|
181
|
+
const at = badCharacterAt(env[key]);
|
|
182
|
+
if (at) throw new Error(`${key} for ${what} "${model}" has whitespace or a line break in it at character ${at}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
183
|
+
if (gateway === "openrouter") return { kind: "openrouter" };
|
|
184
|
+
return direct && key === direct.key ? {
|
|
185
|
+
kind: "direct",
|
|
186
|
+
provider,
|
|
187
|
+
pkg: direct.pkg
|
|
188
|
+
} : { kind: "gateway" };
|
|
189
|
+
}
|
|
190
|
+
/** Transient provider errors (429, 5xx, "high demand") are retried with backoff and recorded. */
|
|
191
|
+
async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
|
|
192
|
+
const { model, threshold } = options;
|
|
193
|
+
const gateway = typeof options.gateway === "object" ? options.gateway.kind : options.gateway;
|
|
194
|
+
const fallbacks = typeof options.gateway === "object" && options.gateway.kind === "openrouter" && options.gateway.fallbacks === true;
|
|
195
|
+
const vertex = gateway === "vertex" ? vertexSettings(typeof options.gateway === "object" && options.gateway.kind === "vertex" ? options.gateway : {}) : void 0;
|
|
196
|
+
let score;
|
|
197
|
+
let resolvedModel;
|
|
198
|
+
let billing = {};
|
|
199
|
+
let retries = [];
|
|
200
|
+
if (isFakeJudge(model)) score = typeof model.score === "function" ? model.score(state) : model.score;
|
|
201
|
+
else {
|
|
202
|
+
if (typeof model === "string") {
|
|
203
|
+
const problem = checkModelString(model);
|
|
204
|
+
if (problem) throw new Error(problem);
|
|
205
|
+
}
|
|
206
|
+
const { scoreWithModel } = await import("./model-CwwkrCb7.js");
|
|
207
|
+
const scored = await withRetries(() => scoreWithModel(model, redactSecrets(question), state, options.providerOptions, signal, gateway, fallbacks, vertex), retry, signal);
|
|
208
|
+
const { score: scoredAt, resolvedModel: served, ...billed } = scored.value;
|
|
209
|
+
score = scoredAt;
|
|
210
|
+
resolvedModel = served;
|
|
211
|
+
billing = billed;
|
|
212
|
+
retries = scored.retries;
|
|
213
|
+
}
|
|
214
|
+
return {
|
|
215
|
+
verdict: {
|
|
216
|
+
verdict: score >= threshold ? "pass" : "fail",
|
|
217
|
+
score,
|
|
218
|
+
threshold,
|
|
219
|
+
question,
|
|
220
|
+
model: judgeModelName(model),
|
|
221
|
+
...resolvedModel ? { resolvedModel } : {},
|
|
222
|
+
...gateway && typeof model === "string" ? { gateway } : {},
|
|
223
|
+
...billing
|
|
224
|
+
},
|
|
225
|
+
retries
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
export { RECOMMENDED_JUDGE_MODEL as a, judgeCredentials as c, modelKeyOptions as d, runJudge as f, isApiError as h, RECOMMENDED_ANALYSIS_MODEL as i, judgeModelName as l, unusedGatewayKey as m, DIRECT_PROVIDERS as n, checkModelString as o, setGatewayKey as p, OPENROUTER_JUDGE_MODEL as r, isFakeJudge as s, DEFAULT_THRESHOLD as t, judgeRoute as u };
|
|
229
|
+
|
|
230
|
+
//# sourceMappingURL=judge-COMXT1Hx.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-COMXT1Hx.js","names":["sleep"],"sources":["../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;;;;AAOA,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,MAAa,cAAc,UACzB,OAAQ,UAAU,KAAK,CAAC,EAAoB,eAAe;;AAG7D,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,SAAS,gBAAgB,OAAO;CACvE,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAMA,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;AC5DA,MAAa,0BAA0B;;AAEvC,MAAa,yBAAyB;;AAEtC,MAAa,6BAA6B;AAC1C,MAAa,oBAAoB;;AAKjC,MAAM,cAAuE;CAC3E,QAAQ,CAJY,YAAY,WAIb;CACnB,YAAY,CAAC,cAAc;AAC7B;;;;;AAMA,MAAa,mBAA+E;CAC1F,WAAW;EAAE,KAAK;EAAqB,KAAK;EAAqB,MAAM;CAAY;CACnF,QAAQ;EAAE,KAAK;EAAgC,KAAK;EAAkB,MAAM;CAAS;CACrF,QAAQ;EAAE,KAAK;EAAkB,KAAK;EAAkB,MAAM;CAAS;CACvE,eAAe;EAAE,KAAK;EAAuB,KAAK;EAAuB,MAAM;CAAc;AAC/F;AAcA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;;;;AAMA,SAAgB,iBACd,OACA,OAAO,mBACP,WAAW,CAAC,yBAAyB,UAAU,OAAO,GAClC;CACpB,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,GAAG,KAAK,iCAAiC,SAAS,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,MAAM;AAC5G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;;;;;;AAQvE,SAAgB,iBACd,OACA,UAAuB,UACb;CACV,IAAI,YAAY,KAAK,KAAK,YAAY,UAAU,OAAO,CAAC;CACxD,IAAI,YAAY,UAAU,OAAO,CAAC,GAAG,YAAY,QAAQ;CACzD,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;AAClC;;AAGA,SAAgB,gBAAgB,OAAe,KAAkB,UAAkB;CACjF,IAAI,OAAO,UAAU,OAAO;CAC5B,IAAI,OAAO,UAAU,OAAO,GAAG,YAAY,GAAG,CAAC,KAAK,MAAM,EAAE,QAAQ,GAAG;CACvE,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SACH,GAAG,OAAO,IAAI,mBACd,4CAA4C,OAAO,KAAK,gBAAgB,CAAC,CACtE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE;AACpB;;AAGA,SAAgB,cACd,KAC0D;CAC1D,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,WAAW,GAAG;EACzD,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;EACnC,IAAI,KAAK,OAAO;GAAE;GAAc;EAA8B;CAChE;AAEF;;;;;AAMA,SAAgB,iBAAiB,KAAwB,UAAuB,UAAkB;CAChG,IAAI,YAAY,UAAU,OAAO;CACjC,MAAM,MAAM,cAAc,GAAG;CAC7B,IAAI,CAAC,KAAK,OAAO;CACjB,MAAM,OAAO,IAAI,YAAY,WAAW,sBAAsB;CAC9D,OAAO,IAAI,IAAI,IAAI,qDAAqD,IAAI,QAAQ,0CAA0C,KAAK;AACrI;;;;;;;;AASA,SAAgB,WACd,OACA,MAAyB,QAAQ,KACjC,OAAO,eACP,UAAuB,UACX;CACZ,IAAI,YAAY,UAAU,OAAO,EAAE,MAAM,SAAS;CAClD,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,MAAM,OAAO,iBAAiB,OAAO,OAAO;CAC5C,IAAI,CAAC,KAAK,QACR,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,qEAAqE,OAAO,KAC5F,gBACF,CAAC,CACE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE,SAChB;CAEF,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;CACnC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,sBAAsB,KAAK,IAAI,MAAM,eAAe,SAAS,UAAU,gBAAgB,OAAO,OAAO,EAAE,qCAAqC,iBAAiB,KAAK,OAAO,GAC3K;CAEF,IAAI,QAAA,qBAAqB;EACvB,MAAM,UAAU,wBAAwB,GAAG;EAC3C,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,KAAK,IAAI,MAAM,KAAK,SAAS;CAC/D;CACA,MAAM,KAAK,eAAe,IAAI,IAAc;CAC5C,IAAI,IACF,MAAM,IAAI,MACR,GAAG,IAAI,OAAO,KAAK,IAAI,MAAM,sDAAsD,GAAG,sFACxF;CAEF,IAAI,YAAY,cAAc,OAAO,EAAE,MAAM,aAAa;CAC1D,OAAO,UAAU,QAAQ,OAAO,MAC5B;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI,IAC5C,EAAE,MAAM,UAAU;AACxB;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,MAAM,UAAU,OAAO,QAAQ,YAAY,WAAW,QAAQ,QAAQ,OAAO,QAAQ;CACrF,MAAM,YACJ,OAAO,QAAQ,YAAY,YAC3B,QAAQ,QAAQ,SAAS,gBACzB,QAAQ,QAAQ,cAAc;CAChC,MAAM,SACJ,YAAY,WACR,eACE,OAAO,QAAQ,YAAY,YAAY,QAAQ,QAAQ,SAAS,WAC5D,QAAQ,UACR,CAAC,CACP,IACA,KAAA;CACN,IAAI;CACJ,IAAI;CACJ,IAAI,UAAmD,CAAC;CACxD,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,OAAO,MAAM,UAAU,aAAa,MAAM,MAAM,KAAK,IAAI,MAAM;MAClE;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBAEjB,eACE,OACA,cAAc,QAAQ,GACtB,OACA,QAAQ,iBACR,QACA,SACA,WACA,MACF,GACF,OACA,MACF;EACA,MAAM,EAAE,OAAO,UAAU,eAAe,QAAQ,GAAG,WAAW,OAAO;EACrE,QAAQ;EACR,gBAAgB;EAChB,UAAU;EACV,UAAU,OAAO;CACnB;CAWA,OAAO;EAAE,SAAA;GATP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;GACzC,GAAI,WAAW,OAAO,UAAU,WAAW,EAAE,QAAQ,IAAI,CAAC;GAC1D,GAAG;EAEU;EAAG;CAAQ;AAC5B"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as JudgeState, Dt as JudgeInputs, Et as JudgeFailure, Ot as JudgeModel, R as ConfigJudgeOptions, Tt as JudgeCallOptions, at as FakeAnswer, c as BlindPair, fn as Verdict, it as FakeAnalysisModel, kt as JudgeOptions, n as AnalysisConfig, s as BlindAttempt, st as FakeJudgeModel, ut as FakeVote, wt as JudgeArtifactRef } from "./types-DBwFsGem.js";
|
|
2
2
|
import "./vertex-C7PI-WPc.js";
|
|
3
3
|
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
4
4
|
export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
|
package/dist/judge.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-
|
|
1
|
+
import { a as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-COMXT1Hx.js";
|
|
2
2
|
// src/judge.ts
|
|
3
3
|
/**
|
|
4
4
|
* A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { n as clip } from "./text-B8-W3Cvn.js";
|
|
2
|
+
import { i as sumUsage } from "./usage-DJiDnh2D.js";
|
|
3
|
+
import { a as liftInterval, m as round3, r as containsZero } from "./stats-C18W4xPX.js";
|
|
4
|
+
// src/view/judgments.ts
|
|
5
|
+
/** Show each key until this many; then the minimum score plus a count. */
|
|
6
|
+
const COLLAPSE_AFTER = 2;
|
|
7
|
+
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
8
|
+
/** Short names stay at this many characters; a collision takes the distinctive tail instead. */
|
|
9
|
+
const LABEL_MAX = 20;
|
|
10
|
+
const judgeText = (question) => question.replace(/\?+$/, "").trim();
|
|
11
|
+
/** Last `max` characters, cut at a word boundary where there is one, marked with a leading `…`. */
|
|
12
|
+
function clipTail(text, max) {
|
|
13
|
+
if (text.length <= max) return text;
|
|
14
|
+
const cut = text.slice(-(max - 1));
|
|
15
|
+
const space = cut.search(/\s/);
|
|
16
|
+
return `…${(space >= 0 && space < max / 2 ? cut.slice(space + 1) : cut).trimStart()}`;
|
|
17
|
+
}
|
|
18
|
+
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
19
|
+
function judgeLabel(question) {
|
|
20
|
+
return clip(judgeText(question), LABEL_MAX);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* One short label per key. Shared prefixes take the distinctive tail so
|
|
24
|
+
* `Does this change…` does not collapse several checks; a tail that still
|
|
25
|
+
* collides falls back to the key.
|
|
26
|
+
*/
|
|
27
|
+
function uniqueJudgeLabels(items) {
|
|
28
|
+
const first = /* @__PURE__ */ new Map();
|
|
29
|
+
for (const item of items) if (!first.has(item.key)) first.set(item.key, item);
|
|
30
|
+
const labels = /* @__PURE__ */ new Map();
|
|
31
|
+
const used = /* @__PURE__ */ new Map();
|
|
32
|
+
const setLabel = (key, label) => {
|
|
33
|
+
const prev = labels.get(key);
|
|
34
|
+
if (prev !== void 0) {
|
|
35
|
+
const group = used.get(prev);
|
|
36
|
+
if (group) {
|
|
37
|
+
const next = group.filter((k) => k !== key);
|
|
38
|
+
if (next.length) used.set(prev, next);
|
|
39
|
+
else used.delete(prev);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
labels.set(key, label);
|
|
43
|
+
const group = used.get(label) ?? [];
|
|
44
|
+
group.push(key);
|
|
45
|
+
used.set(label, group);
|
|
46
|
+
};
|
|
47
|
+
for (const row of first.values()) setLabel(row.key, judgeLabel(row.question));
|
|
48
|
+
for (const [label, keys] of [...used]) {
|
|
49
|
+
if (keys.length < 2) continue;
|
|
50
|
+
const tails = /* @__PURE__ */ new Map();
|
|
51
|
+
for (const key of keys) {
|
|
52
|
+
const row = first.get(key);
|
|
53
|
+
if (!row) continue;
|
|
54
|
+
const tail = clipTail(judgeText(row.question), LABEL_MAX);
|
|
55
|
+
const group = tails.get(tail) ?? [];
|
|
56
|
+
group.push(key);
|
|
57
|
+
tails.set(tail, group);
|
|
58
|
+
}
|
|
59
|
+
for (const [tail, tkeys] of tails) {
|
|
60
|
+
const [only] = tkeys;
|
|
61
|
+
const taken = (used.get(tail) ?? []).some((k) => !tkeys.includes(k));
|
|
62
|
+
if (only && tkeys.length === 1 && tail !== label && !taken) setLabel(only, tail);
|
|
63
|
+
else for (const key of tkeys) setLabel(key, key);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
for (const [, keys] of [...used]) {
|
|
67
|
+
if (keys.length < 2) continue;
|
|
68
|
+
for (const key of keys) setLabel(key, key);
|
|
69
|
+
}
|
|
70
|
+
return labels;
|
|
71
|
+
}
|
|
72
|
+
function keyTallies(e, cellKey) {
|
|
73
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
74
|
+
const expectLabels = /* @__PURE__ */ new Map();
|
|
75
|
+
for (const a of e.attempts) {
|
|
76
|
+
if (a.cell !== cellKey || !counted(a)) continue;
|
|
77
|
+
for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
|
|
78
|
+
const row = byKey.get(j.key) ?? {
|
|
79
|
+
label: "",
|
|
80
|
+
labelled: false,
|
|
81
|
+
question: j.question,
|
|
82
|
+
pass: 0,
|
|
83
|
+
counted: 0
|
|
84
|
+
};
|
|
85
|
+
row.counted++;
|
|
86
|
+
if (j.verdict === "pass") row.pass++;
|
|
87
|
+
byKey.set(j.key, row);
|
|
88
|
+
if (labeled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
|
|
92
|
+
key,
|
|
93
|
+
question: row.question
|
|
94
|
+
})));
|
|
95
|
+
for (const [key, row] of byKey) {
|
|
96
|
+
const label = expectLabels.get(key);
|
|
97
|
+
row.labelled = label !== void 0;
|
|
98
|
+
row.label = label ?? labels.get(key) ?? judgeLabel(row.question);
|
|
99
|
+
}
|
|
100
|
+
return byKey;
|
|
101
|
+
}
|
|
102
|
+
function pairedControl(e, cellKey) {
|
|
103
|
+
const cell = e.cells.find((c) => c.key === cellKey);
|
|
104
|
+
if (cell?.role !== "variant") return void 0;
|
|
105
|
+
return e.cells.find((x) => x.role === "control" && x.harness === cell.harness && x.sandbox === cell.sandbox);
|
|
106
|
+
}
|
|
107
|
+
function liftForKey(variant, control) {
|
|
108
|
+
if (!control || variant.counted === 0 || control.counted === 0) return null;
|
|
109
|
+
const { points, low, high } = liftInterval({
|
|
110
|
+
pass: variant.pass,
|
|
111
|
+
n: variant.counted
|
|
112
|
+
}, {
|
|
113
|
+
pass: control.pass,
|
|
114
|
+
n: control.counted
|
|
115
|
+
});
|
|
116
|
+
return {
|
|
117
|
+
points: round3(points),
|
|
118
|
+
control: {
|
|
119
|
+
pass: control.pass,
|
|
120
|
+
counted: control.counted
|
|
121
|
+
},
|
|
122
|
+
variant: {
|
|
123
|
+
pass: variant.pass,
|
|
124
|
+
counted: variant.counted
|
|
125
|
+
},
|
|
126
|
+
withinNoise: containsZero(low, high)
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides counted an attempt. */
|
|
130
|
+
function cellJudgeRates(e, cellKey) {
|
|
131
|
+
const tallies = keyTallies(e, cellKey);
|
|
132
|
+
const control = pairedControl(e, cellKey);
|
|
133
|
+
const controlTallies = control ? keyTallies(e, control.key) : void 0;
|
|
134
|
+
return [...tallies].map(([key, row]) => ({
|
|
135
|
+
key,
|
|
136
|
+
...row,
|
|
137
|
+
lift: liftForKey(row, controlTallies?.get(key))
|
|
138
|
+
}));
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
142
|
+
* of the cell's checks against its control is already there.
|
|
143
|
+
*/
|
|
144
|
+
function summaryJudgeRates(e, cell) {
|
|
145
|
+
const checked = new Set((cell.checks ?? []).filter((k) => k.control && k.verdict).map((k) => k.label));
|
|
146
|
+
return cellJudgeRates(e, cell.key).filter((r) => !(r.labelled && checked.has(r.label)));
|
|
147
|
+
}
|
|
148
|
+
const signedPoints = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
|
|
149
|
+
/** `login 4/5 · checkout 0/5`, with `(+80 pts)` on a key that has lift. */
|
|
150
|
+
function judgeRatesLabel(rates) {
|
|
151
|
+
return rates.map((r) => {
|
|
152
|
+
const n = `${r.label} ${r.pass}/${r.counted}`;
|
|
153
|
+
return r.lift ? `${n} (${signedPoints(r.lift.points)} pts)` : n;
|
|
154
|
+
}).join(" · ");
|
|
155
|
+
}
|
|
156
|
+
/** Each key, or the lowest score plus how many judges there were. */
|
|
157
|
+
function collapsedJudges(judgments) {
|
|
158
|
+
const labels = uniqueJudgeLabels(judgments);
|
|
159
|
+
const items = judgments.map((j) => ({
|
|
160
|
+
verdict: j.verdict,
|
|
161
|
+
score: j.score,
|
|
162
|
+
threshold: j.threshold,
|
|
163
|
+
label: labels.get(j.key) ?? judgeLabel(j.question),
|
|
164
|
+
key: j.key
|
|
165
|
+
}));
|
|
166
|
+
if (items.length <= COLLAPSE_AFTER) return {
|
|
167
|
+
shown: items,
|
|
168
|
+
extra: 0
|
|
169
|
+
};
|
|
170
|
+
return {
|
|
171
|
+
shown: [items.reduce((a, b) => a.score <= b.score ? a : b)],
|
|
172
|
+
extra: items.length
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
176
|
+
function collapsedJudgeLabel(judgments) {
|
|
177
|
+
if (!judgments.length) return null;
|
|
178
|
+
const { shown, extra } = collapsedJudges(judgments);
|
|
179
|
+
const parts = shown.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold.toFixed(2)}`);
|
|
180
|
+
if (extra) parts.push(`${extra} judges`);
|
|
181
|
+
return parts.join(" · ");
|
|
182
|
+
}
|
|
183
|
+
const JUDGE_ERROR = /^judge failed for /;
|
|
184
|
+
/**
|
|
185
|
+
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
186
|
+
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
187
|
+
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
188
|
+
* reaches is named by its question.
|
|
189
|
+
*/
|
|
190
|
+
function labeledJudgments(a) {
|
|
191
|
+
const passes = (a.assertions ?? []).filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
|
|
192
|
+
const zip = passes.length === a.judgements.length;
|
|
193
|
+
return a.judgements.map((judgment, i) => {
|
|
194
|
+
const x = zip ? passes[i] : void 0;
|
|
195
|
+
return {
|
|
196
|
+
judgment,
|
|
197
|
+
label: x?.labelled ? x.label : judgeText(judgment.question),
|
|
198
|
+
labelled: x?.labelled === true,
|
|
199
|
+
...x ? { check: x.label } : {}
|
|
200
|
+
};
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
/** Sum of counted attempts' usage; null when none of them reported any. */
|
|
204
|
+
function cellUsage(e, cellKey) {
|
|
205
|
+
return sumUsage(e.attempts.filter((a) => a.cell === cellKey && counted(a)).map((a) => a.usage));
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Each `c.exec` check's `expect` label, by position: the attempt's `toSucceed` assertions in call
|
|
209
|
+
* order, when they are exactly as many as its checks; undefined where no label reaches.
|
|
210
|
+
*/
|
|
211
|
+
function checkLabels(a) {
|
|
212
|
+
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
213
|
+
const zip = succeeds.length === a.checks.length;
|
|
214
|
+
return a.checks.map((_, i) => {
|
|
215
|
+
const x = zip ? succeeds[i] : void 0;
|
|
216
|
+
return x?.labelled ? x.label : void 0;
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
|
|
221
|
+
* are exactly as many as its checks; empty otherwise.
|
|
222
|
+
*/
|
|
223
|
+
function checksByLabel(a) {
|
|
224
|
+
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
225
|
+
if (succeeds.length !== a.checks.length) return /* @__PURE__ */ new Map();
|
|
226
|
+
return new Map(succeeds.map((x, i) => [x.label, a.checks[i]]));
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
|
|
230
|
+
* when it had none or passed on its first try.
|
|
231
|
+
*/
|
|
232
|
+
function triesNote(exitCode, tried, of) {
|
|
233
|
+
if (of <= 1 || exitCode === 0 && tried <= 1) return null;
|
|
234
|
+
if (exitCode === 0) return `passed on try ${tried} of ${of}`;
|
|
235
|
+
return tried >= of ? `failed all ${of} tries` : `failed on try ${tried} of ${of}`;
|
|
236
|
+
}
|
|
237
|
+
/** {@link triesNote} for a recorded check. */
|
|
238
|
+
const checkTries = (c) => c.retries && c.tries ? triesNote(c.exitCode, c.tries.length, c.retries + 1) : null;
|
|
239
|
+
/** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
|
|
240
|
+
function commandLine(command, max = 80) {
|
|
241
|
+
const lines = command.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
242
|
+
return clip(lines.length > 1 ? `${lines[0]} …` : lines[0] ?? "", max);
|
|
243
|
+
}
|
|
244
|
+
export { checksByLabel as a, commandLine as c, labeledJudgments as d, summaryJudgeRates as f, checkTries as i, judgeLabel as l, uniqueJudgeLabels as m, cellUsage as n, collapsedJudgeLabel as o, triesNote as p, checkLabels as r, collapsedJudges as s, cellJudgeRates as t, judgeRatesLabel as u };
|
|
245
|
+
|
|
246
|
+
//# sourceMappingURL=judgments-CzgpsTqv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judgments-CzgpsTqv.js","names":[],"sources":["../src/view/judgments.ts"],"mappings":";;;;;AAeA,MAAM,iBAAiB;AAEvB,MAAM,WAAW,MAAsC,EAAE,YAAY,UAAU,EAAE,YAAY;;AAG7F,MAAM,YAAY;AAElB,MAAM,aAAa,aAAqB,SAAS,QAAQ,QAAQ,EAAE,CAAC,CAAC,KAAK;;AAG1E,SAAS,SAAS,MAAc,KAAqB;CACnD,IAAI,KAAK,UAAU,KAAK,OAAO;CAC/B,MAAM,MAAM,KAAK,MAAM,EAAE,MAAM,EAAE;CACjC,MAAM,QAAQ,IAAI,OAAO,IAAI;CAE7B,OAAO,KADM,SAAS,KAAK,QAAQ,MAAM,IAAI,IAAI,MAAM,QAAQ,CAAC,IAAI,IAAA,CACpD,UAAU;AAC5B;;AAGA,SAAgB,WAAW,UAA0B;CACnD,OAAO,KAAK,UAAU,QAAQ,GAAG,SAAS;AAC5C;;;;;;AAOA,SAAgB,kBACd,OACqB;CACrB,MAAM,wBAAQ,IAAI,IAA+C;CACjE,KAAK,MAAM,QAAQ,OACjB,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,GAAG,MAAM,IAAI,KAAK,KAAK,IAAI;CAEpD,MAAM,yBAAS,IAAI,IAAoB;CACvC,MAAM,uBAAO,IAAI,IAAsB;CAEvC,MAAM,YAAY,KAAa,UAAkB;EAC/C,MAAM,OAAO,OAAO,IAAI,GAAG;EAC3B,IAAI,SAAS,KAAA,GAAW;GACtB,MAAM,QAAQ,KAAK,IAAI,IAAI;GAC3B,IAAI,OAAO;IACT,MAAM,OAAO,MAAM,QAAQ,MAAM,MAAM,GAAG;IAC1C,IAAI,KAAK,QAAQ,KAAK,IAAI,MAAM,IAAI;SAC/B,KAAK,OAAO,IAAI;GACvB;EACF;EACA,OAAO,IAAI,KAAK,KAAK;EACrB,MAAM,QAAQ,KAAK,IAAI,KAAK,KAAK,CAAC;EAClC,MAAM,KAAK,GAAG;EACd,KAAK,IAAI,OAAO,KAAK;CACvB;CAEA,KAAK,MAAM,OAAO,MAAM,OAAO,GAAG,SAAS,IAAI,KAAK,WAAW,IAAI,QAAQ,CAAC;CAE5E,KAAK,MAAM,CAAC,OAAO,SAAS,CAAC,GAAG,IAAI,GAAG;EACrC,IAAI,KAAK,SAAS,GAAG;EACrB,MAAM,wBAAQ,IAAI,IAAsB;EACxC,KAAK,MAAM,OAAO,MAAM;GACtB,MAAM,MAAM,MAAM,IAAI,GAAG;GACzB,IAAI,CAAC,KAAK;GACV,MAAM,OAAO,SAAS,UAAU,IAAI,QAAQ,GAAG,SAAS;GACxD,MAAM,QAAQ,MAAM,IAAI,IAAI,KAAK,CAAC;GAClC,MAAM,KAAK,GAAG;GACd,MAAM,IAAI,MAAM,KAAK;EACvB;EACA,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO;GACjC,MAAM,CAAC,QAAQ;GACf,MAAM,SAAS,KAAK,IAAI,IAAI,KAAK,CAAC,EAAA,CAAG,MAAM,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;GACnE,IAAI,QAAQ,MAAM,WAAW,KAAK,SAAS,SAAS,CAAC,OAAO,SAAS,MAAM,IAAI;QAC1E,KAAK,MAAM,OAAO,OAAO,SAAS,KAAK,GAAG;EACjD;CACF;CAEA,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG;EAChC,IAAI,KAAK,SAAS,GAAG;EACrB,KAAK,MAAM,OAAO,MAAM,SAAS,KAAK,GAAG;CAC3C;CAEA,OAAO;AACT;AAuBA,SAAS,WAAW,GAAgC,SAAwC;CAC1F,MAAM,wBAAQ,IAAI,IAAsB;CACxC,MAAM,+BAAe,IAAI,IAAoB;CAC7C,KAAK,MAAM,KAAK,EAAE,UAAU;EAC1B,IAAI,EAAE,SAAS,WAAW,CAAC,QAAQ,CAAC,GAAG;EACvC,KAAK,MAAM,EAAE,UAAU,GAAG,OAAO,UAAU,aAAa,iBAAiB,CAAC,GAAG;GAC3E,MAAM,MAAM,MAAM,IAAI,EAAE,GAAG,KAAK;IAC9B,OAAO;IACP,UAAU;IACV,UAAU,EAAE;IACZ,MAAM;IACN,SAAS;GACX;GACA,IAAI;GACJ,IAAI,EAAE,YAAY,QAAQ,IAAI;GAC9B,MAAM,IAAI,EAAE,KAAK,GAAG;GACpB,IAAI,WAAW,CAAC,aAAa,IAAI,EAAE,GAAG,GAAG,aAAa,IAAI,EAAE,KAAK,KAAK;EACxE;CACF;CACA,MAAM,SAAS,kBACb,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,UAAU;EAAE;EAAK,UAAU,IAAI;CAAS,EAAE,CAClE;CACA,KAAK,MAAM,CAAC,KAAK,QAAQ,OAAO;EAC9B,MAAM,QAAQ,aAAa,IAAI,GAAG;EAClC,IAAI,WAAW,UAAU,KAAA;EACzB,IAAI,QAAQ,SAAS,OAAO,IAAI,GAAG,KAAK,WAAW,IAAI,QAAQ;CACjE;CACA,OAAO;AACT;AAEA,SAAS,cAAc,GAA6B,SAAyC;CAC3F,MAAM,OAAO,EAAE,MAAM,MAAM,MAAM,EAAE,QAAQ,OAAO;CAClD,IAAI,MAAM,SAAS,WAAW,OAAO,KAAA;CACrC,OAAO,EAAE,MAAM,MACZ,MAAM,EAAE,SAAS,aAAa,EAAE,YAAY,KAAK,WAAW,EAAE,YAAY,KAAK,OAClF;AACF;AAEA,SAAS,WACP,SACA,SACsB;CACtB,IAAI,CAAC,WAAW,QAAQ,YAAY,KAAK,QAAQ,YAAY,GAAG,OAAO;CACvE,MAAM,EAAE,QAAQ,KAAK,SAAS,aAC5B;EAAE,MAAM,QAAQ;EAAM,GAAG,QAAQ;CAAQ,GACzC;EAAE,MAAM,QAAQ;EAAM,GAAG,QAAQ;CAAQ,CAC3C;CACA,OAAO;EACL,QAAQ,OAAO,MAAM;EACrB,SAAS;GAAE,MAAM,QAAQ;GAAM,SAAS,QAAQ;EAAQ;EACxD,SAAS;GAAE,MAAM,QAAQ;GAAM,SAAS,QAAQ;EAAQ;EACxD,aAAa,aAAa,KAAK,IAAI;CACrC;AACF;;AAGA,SAAgB,eACd,GACA,SACgB;CAChB,MAAM,UAAU,WAAW,GAAG,OAAO;CACrC,MAAM,UAAU,cAAc,GAAG,OAAO;CACxC,MAAM,iBAAiB,UAAU,WAAW,GAAG,QAAQ,GAAG,IAAI,KAAA;CAC9D,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,UAAU;EACvC;EACA,GAAG;EACH,MAAM,WAAW,KAAK,gBAAgB,IAAI,GAAG,CAAC;CAChD,EAAE;AACJ;;;;;AAMA,SAAgB,kBACd,GACA,MACgB;CAChB,MAAM,UAAU,IAAI,KACjB,KAAK,UAAU,CAAC,EAAA,CAAG,QAAQ,MAAM,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,KAAK,MAAM,EAAE,KAAK,CAC9E;CACA,OAAO,eAAe,GAAG,KAAK,GAAG,CAAC,CAAC,QAAQ,MAAM,EAAE,EAAE,YAAY,QAAQ,IAAI,EAAE,KAAK,EAAE;AACxF;AAEA,MAAM,gBAAgB,MAAc,GAAG,KAAK,IAAI,MAAM,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG;;AAGxF,SAAgB,gBAAgB,OAAwC;CACtE,OAAO,MACJ,KAAK,MAAM;EACV,MAAM,IAAI,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,GAAG,EAAE;EACpC,OAAO,EAAE,OAAO,GAAG,EAAE,IAAI,aAAa,EAAE,KAAK,MAAM,EAAE,SAAS;CAChE,CAAC,CAAC,CACD,KAAK,KAAK;AACf;;AAWA,SAAgB,gBAAgB,WAG9B;CACA,MAAM,SAAS,kBAAkB,SAAS;CAC1C,MAAM,QAAQ,UAAU,KAAK,OAAO;EAClC,SAAS,EAAE;EACX,OAAO,EAAE;EACT,WAAW,EAAE;EACb,OAAO,OAAO,IAAI,EAAE,GAAG,KAAK,WAAW,EAAE,QAAQ;EACjD,KAAK,EAAE;CACT,EAAE;CACF,IAAI,MAAM,UAAU,gBAAgB,OAAO;EAAE,OAAO;EAAO,OAAO;CAAE;CAEpE,OAAO;EAAE,OAAO,CADJ,MAAM,QAAQ,GAAG,MAAO,EAAE,SAAS,EAAE,QAAQ,IAAI,CAC1C,CAAC;EAAG,OAAO,MAAM;CAAO;AAC7C;;AAGA,SAAgB,oBAAoB,WAAqD;CACvF,IAAI,CAAC,UAAU,QAAQ,OAAO;CAC9B,MAAM,EAAE,OAAO,UAAU,gBAAgB,SAAS;CAClD,MAAM,QAAQ,MAAM,KAAK,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,UAAU,QAAQ,CAAC,GAAG;CAC7F,IAAI,OAAO,MAAM,KAAK,GAAG,MAAM,QAAQ;CACvC,OAAO,MAAM,KAAK,KAAK;AACzB;AAEA,MAAM,cAAc;;;;;;;AAQpB,SAAgB,iBACd,GACuF;CACvF,MAAM,UAAU,EAAE,cAAc,CAAC,EAAA,CAAG,QACjC,MAAM,gBAAgB,KAAK,EAAE,OAAO,KAAK,CAAC,YAAY,KAAK,EAAE,WAAW,EAAE,CAC7E;CACA,MAAM,MAAM,OAAO,WAAW,EAAE,WAAW;CAC3C,OAAO,EAAE,WAAW,KAAK,UAAU,MAAM;EACvC,MAAM,IAAI,MAAM,OAAO,KAAK,KAAA;EAC5B,OAAO;GACL;GACA,OAAO,GAAG,WAAW,EAAE,QAAQ,UAAU,SAAS,QAAQ;GAC1D,UAAU,GAAG,aAAa;GAC1B,GAAI,IAAI,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;EAChC;CACF,CAAC;AACH;;AAGA,SAAgB,UAAU,GAAgC,SAAoC;CAC5F,OAAO,SAAS,EAAE,SAAS,QAAQ,MAAM,EAAE,SAAS,WAAW,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,KAAK,CAAC;AAChG;;;;;AAMA,SAAgB,YACd,GAC2B;CAC3B,MAAM,YAAY,EAAE,cAAc,CAAC,EAAA,CAAG,QAAQ,MAAM,mBAAmB,KAAK,EAAE,OAAO,CAAC;CACtF,MAAM,MAAM,SAAS,WAAW,EAAE,OAAO;CACzC,OAAO,EAAE,OAAO,KAAK,GAAG,MAAM;EAC5B,MAAM,IAAI,MAAM,SAAS,KAAK,KAAA;EAC9B,OAAO,GAAG,WAAW,EAAE,QAAQ,KAAA;CACjC,CAAC;AACH;;;;;AAMA,SAAgB,cACd,GACgB;CAChB,MAAM,YAAY,EAAE,cAAc,CAAC,EAAA,CAAG,QAAQ,MAAM,mBAAmB,KAAK,EAAE,OAAO,CAAC;CACtF,IAAI,SAAS,WAAW,EAAE,OAAO,QAAQ,uBAAO,IAAI,IAAI;CACxD,OAAO,IAAI,IAAI,SAAS,KAAK,GAAG,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAO,CAAC,CAAC;AACpE;;;;;AAMA,SAAgB,UAAU,UAAkB,OAAe,IAA2B;CACpF,IAAI,MAAM,KAAM,aAAa,KAAK,SAAS,GAAI,OAAO;CACtD,IAAI,aAAa,GAAG,OAAO,iBAAiB,MAAM,MAAM;CACxD,OAAO,SAAS,KAAK,cAAc,GAAG,UAAU,iBAAiB,MAAM,MAAM;AAC/E;;AAGA,MAAa,cAAc,MAIpB,EAAE,WAAW,EAAE,QAAQ,UAAU,EAAE,UAAU,EAAE,MAAM,QAAQ,EAAE,UAAU,CAAC,IAAI;;AAGrF,SAAgB,YAAY,SAAiB,MAAM,IAAY;CAC7D,MAAM,QAAQ,QACX,MAAM,IAAI,CAAC,CACX,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,CACpB,OAAO,OAAO;CACjB,OAAO,KAAK,MAAM,SAAS,IAAI,GAAG,MAAM,GAAG,MAAO,MAAM,MAAM,IAAK,GAAG;AACxE"}
|