reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,8 +1,56 @@
|
|
|
1
1
|
import { c as shardPath, i as manifestPath, s as shardOf } from "./paths-BmR-7tx3.js";
|
|
2
|
-
import { n as clip } from "./text-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { a as plural, n as clip } from "./text-DTigcA64.js";
|
|
3
|
+
import { l as sampleReasons } from "./pairs-puYi-zj2.js";
|
|
4
|
+
import { c as minimumP, d as round3, i as mannWhitney, n as containsZero, r as liftInterval, s as median, u as quantile } from "./stats-hfpuYsAM.js";
|
|
5
|
+
import { a as totalTokens, i as sumUsage, n as formatTokens, r as promptTokens } from "./usage-BNY1ABrU.js";
|
|
6
|
+
import { l as readableTranscriptLine } from "./record-DZBCjuwm.js";
|
|
7
|
+
//#region src/view/gate.ts
|
|
8
|
+
/** `all`, `none` or `passRate 0.5`; `{ passRate: 1 }` is `all`, which scores the same. */
|
|
9
|
+
const gateLabel = (gate) => typeof gate === "string" ? gate : gate.passRate === 1 ? "all" : `passRate ${gate.passRate}`;
|
|
10
|
+
/** The gate's part of a fingerprint: none on `'all'`, so such a cell keeps its fingerprint. */
|
|
11
|
+
const gatePart = (gate) => {
|
|
12
|
+
const label = gateLabel(gate);
|
|
13
|
+
return label === "all" ? {} : { gate: label };
|
|
14
|
+
};
|
|
15
|
+
/** Any failure fails; then infra errors; then all-skipped; else pass. */
|
|
16
|
+
function worstOutcome(outcomes) {
|
|
17
|
+
if (outcomes.includes("fail")) return "fail";
|
|
18
|
+
if (outcomes.includes("infra_error")) return "infra_error";
|
|
19
|
+
if (outcomes.length > 0 && outcomes.every((o) => o === "skipped")) return "skipped";
|
|
20
|
+
return outcomes.length ? "pass" : "skipped";
|
|
21
|
+
}
|
|
22
|
+
const passRateOf = (outcomes) => {
|
|
23
|
+
const pass = outcomes.filter((o) => o === "pass").length;
|
|
24
|
+
const counted = pass + outcomes.filter((o) => o === "fail").length;
|
|
25
|
+
return counted ? pass / counted : null;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* A cell's outcome by its gate. `'all'` matches `{ passRate: 1 }`: every counted attempt must
|
|
29
|
+
* pass, and it is an infra error only when nothing counted. `'none'` takes the worst attempt.
|
|
30
|
+
*/
|
|
31
|
+
function cellOutcome(outcomes, gate) {
|
|
32
|
+
if (gate === "none") return worstOutcome(outcomes);
|
|
33
|
+
const required = gate === "all" ? 1 : gate.passRate;
|
|
34
|
+
const rate = passRateOf(outcomes);
|
|
35
|
+
if (rate === null) return worstOutcome(outcomes);
|
|
36
|
+
return rate >= required ? "pass" : "fail";
|
|
37
|
+
}
|
|
38
|
+
/** The cell's attempts, less the fails the run's interrupt canceled, which the agent didn't fail. */
|
|
39
|
+
const outcomesOf = (c) => [
|
|
40
|
+
...Array(c.pass).fill("pass"),
|
|
41
|
+
...Array(c.fail - (c.canceled ?? 0)).fill("fail"),
|
|
42
|
+
...Array(c.infraError).fill("infra_error"),
|
|
43
|
+
...Array(c.skipped).fill("skipped")
|
|
44
|
+
];
|
|
45
|
+
/**
|
|
46
|
+
* A cell's recorded attempts scored under another gate. A cell its own gate passes but that
|
|
47
|
+
* failed anyway fell short of its `minLift`, which no gate changes, so it still fails.
|
|
48
|
+
*/
|
|
49
|
+
function rescore(c, gate) {
|
|
50
|
+
const outcomes = outcomesOf(c);
|
|
51
|
+
return c.outcome === "fail" && cellOutcome(outcomes, c.gate) === "pass" ? "fail" : cellOutcome(outcomes, gate);
|
|
52
|
+
}
|
|
53
|
+
//#endregion
|
|
6
54
|
//#region src/view/compare.ts
|
|
7
55
|
const STACK_FIELDS = [
|
|
8
56
|
{
|
|
@@ -125,11 +173,32 @@ const PART_LABELS = {
|
|
|
125
173
|
image: "image",
|
|
126
174
|
agentCli: "agent CLI",
|
|
127
175
|
judge: "judge",
|
|
128
|
-
context: "agent context"
|
|
176
|
+
context: "agent context",
|
|
177
|
+
gateway: "gateway"
|
|
129
178
|
};
|
|
130
|
-
|
|
179
|
+
const showRevision = (r) => r === void 0 ? "unset" : JSON.stringify(r);
|
|
180
|
+
const SOURCE_LABELS = {
|
|
181
|
+
body: "body",
|
|
182
|
+
options: "options",
|
|
183
|
+
case: "case data"
|
|
184
|
+
};
|
|
185
|
+
/** What in the eval's own source differs: its body, options, case data and revision. */
|
|
186
|
+
function sourceChanged(base, head) {
|
|
187
|
+
const out = [];
|
|
188
|
+
if (base.source && head.source) {
|
|
189
|
+
for (const key of Object.keys(SOURCE_LABELS)) if (base.source[key] !== head.source[key]) out.push(`${SOURCE_LABELS[key]} changed`);
|
|
190
|
+
} else if (base.source || head.source) out.push("fingerprint scheme changed");
|
|
191
|
+
else out.push(...sourceChanges(base.sources, head.sources));
|
|
192
|
+
if (base.revision !== head.revision) out.push(`revision ${showRevision(base.revision)} → ${showRevision(head.revision)}`);
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Why two runs of an eval aren't comparable: what in its source differs (`body`, `options`,
|
|
197
|
+
* `case data`, `revision 2 → 3`; a record from before `source`, its files), then the fingerprint
|
|
198
|
+
* parts that differ. Each reason but the revision ends in ` changed`.
|
|
199
|
+
*/
|
|
131
200
|
function whyChanged(base, head) {
|
|
132
|
-
const out =
|
|
201
|
+
const out = sourceChanged(base, head);
|
|
133
202
|
const keys = [...new Set([...base.cells, ...head.cells].map((c) => c.key))];
|
|
134
203
|
const parts = /* @__PURE__ */ new Set();
|
|
135
204
|
for (const key of keys) {
|
|
@@ -140,9 +209,48 @@ function whyChanged(base, head) {
|
|
|
140
209
|
continue;
|
|
141
210
|
}
|
|
142
211
|
for (const part of Object.keys(PART_LABELS)) if (b.fingerprintParts[part] !== h.fingerprintParts[part]) parts.add(PART_LABELS[part]);
|
|
212
|
+
if (gateLabel(b.gate) !== gateLabel(h.gate)) parts.add("gate");
|
|
143
213
|
}
|
|
144
214
|
return [...out, ...[...parts].map((p) => `${p} changed`)];
|
|
145
215
|
}
|
|
216
|
+
const withoutGate = ({ gate: _, ...parts }) => JSON.stringify(parts);
|
|
217
|
+
/**
|
|
218
|
+
* Whether two runs of a cell are comparable: the same fingerprint, the gate aside. A record from
|
|
219
|
+
* before the gate was a part has none, so its parts are compared without it.
|
|
220
|
+
*/
|
|
221
|
+
const sameCell = (b, h) => b.fingerprint === h.fingerprint || (b.fingerprintParts.gate !== void 0 || h.fingerprintParts.gate !== void 0) && withoutGate(b.fingerprintParts) === withoutGate(h.fingerprintParts);
|
|
222
|
+
/** Whether two runs of an eval ran the same cells, each comparable. */
|
|
223
|
+
function sameEval(b, h) {
|
|
224
|
+
if (b.fingerprint === h.fingerprint) return true;
|
|
225
|
+
if (b.cells.length !== h.cells.length) return false;
|
|
226
|
+
const before = new Map(b.cells.map((c) => [c.key, c]));
|
|
227
|
+
return h.cells.every((c) => {
|
|
228
|
+
const was = before.get(c.key);
|
|
229
|
+
return was !== void 0 && sameCell(was, c);
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* When the gate of a cell both sides ran changed: each side's gates, and base's outcome over
|
|
234
|
+
* those cells with each one's attempts scored under head's gate.
|
|
235
|
+
*/
|
|
236
|
+
function gateChange(base, head) {
|
|
237
|
+
const before = new Map(base.map((c) => [c.key, c]));
|
|
238
|
+
const pairs = head.flatMap((h) => {
|
|
239
|
+
const b = before.get(h.key);
|
|
240
|
+
return b ? [[b, h]] : [];
|
|
241
|
+
});
|
|
242
|
+
const changed = pairs.filter(([b, h]) => gateLabel(b.gate) !== gateLabel(h.gate));
|
|
243
|
+
if (!changed.length) return void 0;
|
|
244
|
+
const labels = (side) => [...new Set(changed.map((p) => gateLabel(p[side].gate)))].join(", ");
|
|
245
|
+
return {
|
|
246
|
+
base: labels(0),
|
|
247
|
+
head: labels(1),
|
|
248
|
+
rescored: outcomeOf(pairs.map(([b, h]) => ({
|
|
249
|
+
gate: h.gate,
|
|
250
|
+
outcome: rescore(b, h.gate)
|
|
251
|
+
})))
|
|
252
|
+
};
|
|
253
|
+
}
|
|
146
254
|
/** An eval's outcome from some of its cells, as the run rolls it up: ungated failures pass. */
|
|
147
255
|
function outcomeOf(cells) {
|
|
148
256
|
const outcomes = cells.map((c) => c.gate === "none" && c.outcome === "fail" ? "pass" : c.outcome);
|
|
@@ -151,16 +259,16 @@ function outcomeOf(cells) {
|
|
|
151
259
|
return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
|
|
152
260
|
}
|
|
153
261
|
/**
|
|
154
|
-
* Whether a failing eval failed only on attempts the run's interrupt
|
|
155
|
-
* attempt in a gated cell was cut short, so the agent didn't fail it (`reeval show`'s `
|
|
262
|
+
* Whether a failing eval failed only on attempts the run's interrupt canceled: every failed
|
|
263
|
+
* attempt in a gated cell was cut short, so the agent didn't fail it (`reeval show`'s `canceled`).
|
|
156
264
|
*/
|
|
157
|
-
const
|
|
265
|
+
const canceledOn = (outcome, cells) => {
|
|
158
266
|
const gated = cells.filter((c) => c.gate !== "none");
|
|
159
|
-
return outcome === "fail" && gated.some((c) => c.fail > 0) && gated.every((c) => c.fail === (c.
|
|
267
|
+
return outcome === "fail" && gated.some((c) => c.fail > 0) && gated.every((c) => c.fail === (c.canceled ?? 0));
|
|
160
268
|
};
|
|
161
|
-
function
|
|
162
|
-
const sides = ["base", "head"].filter((s) =>
|
|
163
|
-
return sides.length ? {
|
|
269
|
+
function canceledSides(outcomes, cells) {
|
|
270
|
+
const sides = ["base", "head"].filter((s) => canceledOn(outcomes[s], cells[s] ?? []));
|
|
271
|
+
return sides.length ? { canceled: sides } : {};
|
|
164
272
|
}
|
|
165
273
|
/** An eval only head has. */
|
|
166
274
|
const addedRow = (h) => ({
|
|
@@ -168,7 +276,7 @@ const addedRow = (h) => ({
|
|
|
168
276
|
base: null,
|
|
169
277
|
head: h.outcome,
|
|
170
278
|
status: "added",
|
|
171
|
-
...
|
|
279
|
+
...canceledSides({
|
|
172
280
|
base: null,
|
|
173
281
|
head: h.outcome
|
|
174
282
|
}, { head: h.cells })
|
|
@@ -181,7 +289,7 @@ const addedRow = (h) => ({
|
|
|
181
289
|
function sharedScope(b, h) {
|
|
182
290
|
const before = new Map(b.cells.map((c) => [c.key, c]));
|
|
183
291
|
const shared = h.cells.filter((c) => before.has(c.key));
|
|
184
|
-
if (!shared.length || shared.some((c) => before.get(c.key)
|
|
292
|
+
if (!shared.length || shared.some((c) => !sameCell(before.get(c.key) ?? c, c))) return null;
|
|
185
293
|
const keys = new Set(h.cells.map((c) => c.key));
|
|
186
294
|
const baseCells = shared.map((c) => before.get(c.key) ?? c);
|
|
187
295
|
return {
|
|
@@ -215,8 +323,134 @@ function compareCells(base, head) {
|
|
|
215
323
|
}];
|
|
216
324
|
});
|
|
217
325
|
}
|
|
326
|
+
/** A family with a gate: some pooled lift has a `minLift`. One without can't fail, so has no row. */
|
|
327
|
+
const gated = (f) => !!f?.lifts.some((l) => l.minLift !== void 0);
|
|
328
|
+
const liftSide = ({ variant: _, vs: __, ...side }) => side;
|
|
329
|
+
/** Each variant's pooled lift on both sides, head's order first. */
|
|
330
|
+
function familyLifts(b, h) {
|
|
331
|
+
const out = /* @__PURE__ */ new Map();
|
|
332
|
+
for (const [side, f] of [["head", h], ["base", b]]) for (const l of f?.lifts ?? []) {
|
|
333
|
+
const row = out.get(l.variant) ?? {
|
|
334
|
+
key: l.variant,
|
|
335
|
+
vs: l.vs,
|
|
336
|
+
base: null,
|
|
337
|
+
head: null
|
|
338
|
+
};
|
|
339
|
+
row[side] = liftSide(l);
|
|
340
|
+
out.set(l.variant, row);
|
|
341
|
+
}
|
|
342
|
+
return [...out.values()];
|
|
343
|
+
}
|
|
344
|
+
const liftsOutcome = (lifts) => lifts.some((l) => l.outcome === "fail") ? "fail" : "pass";
|
|
345
|
+
/** A gated family only head has. */
|
|
346
|
+
const addedFamilyRow = (h) => ({
|
|
347
|
+
id: h.id,
|
|
348
|
+
kind: "family",
|
|
349
|
+
base: null,
|
|
350
|
+
head: h.outcome,
|
|
351
|
+
status: "added",
|
|
352
|
+
lifts: familyLifts(null, h)
|
|
353
|
+
});
|
|
354
|
+
/** The rows of a run with nothing to compare it with: every eval and gated family is new. */
|
|
355
|
+
const newRows = (head) => [...head.evals.map(addedRow), ...(head.families ?? []).filter(gated).map(addedFamilyRow)];
|
|
356
|
+
/**
|
|
357
|
+
* A gated family's row. Its pooled outcome is comparable only when both runs ran the same cases
|
|
358
|
+
* and every case is: a case that changed or hit an infrastructure error makes the family `drift`
|
|
359
|
+
* or `infra` as it does the case, and a failing side with a canceled attempt in any case reads
|
|
360
|
+
* `canceled`, since the pooled lift counts it. With different variants, it's over the lifts both
|
|
361
|
+
* have, and with none in common it can't compare them.
|
|
362
|
+
*/
|
|
363
|
+
function familyRow(id, b, h, ctx) {
|
|
364
|
+
if (!b) return addedFamilyRow(h);
|
|
365
|
+
const lifts = familyLifts(b, h ?? null);
|
|
366
|
+
if (!h) return {
|
|
367
|
+
id,
|
|
368
|
+
kind: "family",
|
|
369
|
+
base: b.outcome,
|
|
370
|
+
head: null,
|
|
371
|
+
status: "removed",
|
|
372
|
+
lifts
|
|
373
|
+
};
|
|
374
|
+
const shared = lifts.flatMap((l) => l.base && l.head ? [{
|
|
375
|
+
base: l.base,
|
|
376
|
+
head: l.head
|
|
377
|
+
}] : []);
|
|
378
|
+
const scoped = shared.length > 0 && shared.length < lifts.length;
|
|
379
|
+
const disjoint = !shared.length && b.lifts.length > 0 && h.lifts.length > 0;
|
|
380
|
+
const row = {
|
|
381
|
+
id,
|
|
382
|
+
kind: "family",
|
|
383
|
+
base: scoped ? liftsOutcome(shared.map((l) => l.base)) : b.outcome,
|
|
384
|
+
head: scoped ? liftsOutcome(shared.map((l) => l.head)) : h.outcome,
|
|
385
|
+
lifts,
|
|
386
|
+
...scoped || disjoint ? { cells: {
|
|
387
|
+
added: lifts.filter((l) => !l.base).map((l) => l.key),
|
|
388
|
+
removed: lifts.filter((l) => !l.head).map((l) => l.key)
|
|
389
|
+
} } : {}
|
|
390
|
+
};
|
|
391
|
+
const prefix = id.slice(0, id.lastIndexOf(" › ") + 3);
|
|
392
|
+
const name = (caseId) => caseId.startsWith(prefix) ? caseId.slice(prefix.length) : caseId;
|
|
393
|
+
const before = new Set(b.caseIds);
|
|
394
|
+
const after = new Set(h.caseIds);
|
|
395
|
+
const caseRows = [.../* @__PURE__ */ new Set([...h.caseIds, ...b.caseIds])].flatMap((c) => ctx.rows.get(c) ?? []);
|
|
396
|
+
const changes = [.../* @__PURE__ */ new Set([
|
|
397
|
+
...h.caseIds.filter((c) => !before.has(c)).map((c) => `case ${name(c)} added`),
|
|
398
|
+
...b.caseIds.filter((c) => !after.has(c)).map((c) => `case ${name(c)} removed`),
|
|
399
|
+
...caseRows.flatMap((r) => r.status === "drift" ? r.changes ?? [] : [])
|
|
400
|
+
])];
|
|
401
|
+
const canceled = ["base", "head"].filter((s) => row[s] === "fail" && ctx.canceledIn(s, s === "base" ? b.caseIds : h.caseIds));
|
|
402
|
+
if (canceled.length) row.canceled = canceled;
|
|
403
|
+
if (caseRows.some((r) => r.status === "infra")) return {
|
|
404
|
+
...row,
|
|
405
|
+
status: "infra"
|
|
406
|
+
};
|
|
407
|
+
if (changes.length) return {
|
|
408
|
+
...row,
|
|
409
|
+
status: "drift",
|
|
410
|
+
changes
|
|
411
|
+
};
|
|
412
|
+
if (disjoint || row.canceled || caseRows.some((r) => r.status === "drift")) return {
|
|
413
|
+
...row,
|
|
414
|
+
status: "drift"
|
|
415
|
+
};
|
|
416
|
+
if (ctx.graderChanged && row.base !== row.head) return {
|
|
417
|
+
...row,
|
|
418
|
+
status: "drift"
|
|
419
|
+
};
|
|
420
|
+
if (row.base === "pass" && row.head === "fail") return {
|
|
421
|
+
...row,
|
|
422
|
+
status: "regressed"
|
|
423
|
+
};
|
|
424
|
+
if (row.base === "fail" && row.head === "pass") return {
|
|
425
|
+
...row,
|
|
426
|
+
status: "fixed"
|
|
427
|
+
};
|
|
428
|
+
return {
|
|
429
|
+
...row,
|
|
430
|
+
status: "same"
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
/** Each eval's row, then each family's that either run gated. */
|
|
218
434
|
function compare(base, head) {
|
|
219
435
|
const graderChanged = stackChanges(base, head).some((c) => c.outsideFingerprint);
|
|
436
|
+
const rows = evalRows(base, head, graderChanged);
|
|
437
|
+
const before = new Map((base.families ?? []).map((f) => [f.id, f]));
|
|
438
|
+
const after = new Map((head.families ?? []).map((f) => [f.id, f]));
|
|
439
|
+
const families = [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((id) => gated(before.get(id)) || gated(after.get(id)));
|
|
440
|
+
const runs = {
|
|
441
|
+
base,
|
|
442
|
+
head
|
|
443
|
+
};
|
|
444
|
+
const ctx = {
|
|
445
|
+
rows: new Map(rows.map((r) => [r.id, r])),
|
|
446
|
+
graderChanged,
|
|
447
|
+
canceledIn: (side, caseIds) => runs[side].evals.some((e) => caseIds.includes(e.id) && e.cells.some((c) => (c.canceled ?? 0) > 0))
|
|
448
|
+
};
|
|
449
|
+
return [...rows, ...families.map((id) => familyRow(id, before.get(id), after.get(id), ctx))];
|
|
450
|
+
}
|
|
451
|
+
/** The key that tells a family's row from an eval's, whose ids can coincide. */
|
|
452
|
+
const rowKey = (row) => row.kind === "family" ? `family\0${row.id}` : row.id;
|
|
453
|
+
function evalRows(base, head, graderChanged) {
|
|
220
454
|
return [.../* @__PURE__ */ new Set([...head.evals.map((e) => e.id), ...base.evals.map((e) => e.id)])].map((id) => {
|
|
221
455
|
const b = base.evals.find((e) => e.id === id);
|
|
222
456
|
const h = head.evals.find((e) => e.id === id);
|
|
@@ -226,12 +460,13 @@ function compare(base, head) {
|
|
|
226
460
|
base: b.outcome,
|
|
227
461
|
head: null,
|
|
228
462
|
status: "removed",
|
|
229
|
-
...
|
|
463
|
+
...canceledSides({
|
|
230
464
|
base: b.outcome,
|
|
231
465
|
head: null
|
|
232
466
|
}, { base: b.cells })
|
|
233
467
|
};
|
|
234
|
-
const
|
|
468
|
+
const comparable = sameEval(b, h);
|
|
469
|
+
const scoped = comparable ? null : sharedScope(b, h);
|
|
235
470
|
const outcomes = scoped ? {
|
|
236
471
|
base: scoped.base,
|
|
237
472
|
head: scoped.head
|
|
@@ -243,7 +478,7 @@ function compare(base, head) {
|
|
|
243
478
|
id,
|
|
244
479
|
...outcomes,
|
|
245
480
|
...scoped ? { cells: scoped.cells } : {},
|
|
246
|
-
...
|
|
481
|
+
...canceledSides(outcomes, scoped?.over ?? {
|
|
247
482
|
base: b.cells,
|
|
248
483
|
head: h.cells
|
|
249
484
|
})
|
|
@@ -254,346 +489,44 @@ function compare(base, head) {
|
|
|
254
489
|
...row,
|
|
255
490
|
status: "infra"
|
|
256
491
|
};
|
|
257
|
-
if (
|
|
492
|
+
if (!comparable && !scoped) return {
|
|
258
493
|
...row,
|
|
259
494
|
status: "drift",
|
|
260
495
|
changes: whyChanged(b, h)
|
|
261
496
|
};
|
|
262
|
-
|
|
497
|
+
const over = scoped?.over ?? {
|
|
498
|
+
base: b.cells ?? [],
|
|
499
|
+
head: h.cells ?? []
|
|
500
|
+
};
|
|
501
|
+
const gate = gateChange(over.base, over.head);
|
|
502
|
+
if (gate) Object.assign(row, {
|
|
503
|
+
changes: ["gate changed"],
|
|
504
|
+
gate
|
|
505
|
+
});
|
|
506
|
+
const from = gate?.rescored ?? row.base;
|
|
507
|
+
if (row.canceled) return {
|
|
263
508
|
...row,
|
|
264
509
|
status: "drift"
|
|
265
510
|
};
|
|
266
|
-
if (graderChanged &&
|
|
511
|
+
if (graderChanged && from !== row.head) return {
|
|
267
512
|
...row,
|
|
268
513
|
status: "drift"
|
|
269
514
|
};
|
|
270
|
-
if (
|
|
515
|
+
if (from === "pass" && row.head === "fail") return {
|
|
271
516
|
...row,
|
|
272
517
|
status: "regressed"
|
|
273
518
|
};
|
|
274
|
-
if (
|
|
519
|
+
if (from === "fail" && row.head === "pass") return {
|
|
275
520
|
...row,
|
|
276
521
|
status: "fixed"
|
|
277
522
|
};
|
|
278
523
|
return {
|
|
279
524
|
...row,
|
|
280
|
-
status: "same"
|
|
525
|
+
status: row.base === row.head ? "same" : "gate"
|
|
281
526
|
};
|
|
282
527
|
});
|
|
283
528
|
}
|
|
284
529
|
//#endregion
|
|
285
|
-
//#region src/stats.ts
|
|
286
|
-
/** Two-sided 95%. */
|
|
287
|
-
const Z = 1.959964;
|
|
288
|
-
/**
|
|
289
|
-
* The Wilson score interval for `pass` successes out of `n`. Unlike the normal approximation it
|
|
290
|
-
* stays inside [0, 1] and is sensible at 0/n and n/n. With no attempts it's [0, 1].
|
|
291
|
-
*/
|
|
292
|
-
function wilson(pass, n) {
|
|
293
|
-
if (n === 0) return {
|
|
294
|
-
low: 0,
|
|
295
|
-
high: 1
|
|
296
|
-
};
|
|
297
|
-
const p = pass / n;
|
|
298
|
-
const z2 = Z * Z;
|
|
299
|
-
const denominator = 1 + z2 / n;
|
|
300
|
-
const center = (p + z2 / (2 * n)) / denominator;
|
|
301
|
-
const half = Z * Math.sqrt(p * (1 - p) / n + z2 / (4 * n * n)) / denominator;
|
|
302
|
-
return {
|
|
303
|
-
low: Math.max(0, center - half),
|
|
304
|
-
high: Math.min(1, center + half)
|
|
305
|
-
};
|
|
306
|
-
}
|
|
307
|
-
/**
|
|
308
|
-
* The difference of two pass rates (variant minus control), with Newcombe's hybrid score
|
|
309
|
-
* interval: each rate's Wilson interval combined, so it inherits Wilson's behaviour at 0 and 1.
|
|
310
|
-
*/
|
|
311
|
-
function liftInterval(variant, control) {
|
|
312
|
-
const p1 = variant.n ? variant.pass / variant.n : 0;
|
|
313
|
-
const p2 = control.n ? control.pass / control.n : 0;
|
|
314
|
-
const a = wilson(variant.pass, variant.n);
|
|
315
|
-
const b = wilson(control.pass, control.n);
|
|
316
|
-
const points = p1 - p2;
|
|
317
|
-
return {
|
|
318
|
-
points,
|
|
319
|
-
low: points - Math.sqrt((p1 - a.low) ** 2 + (b.high - p2) ** 2),
|
|
320
|
-
high: points + Math.sqrt((a.high - p1) ** 2 + (p2 - b.low) ** 2)
|
|
321
|
-
};
|
|
322
|
-
}
|
|
323
|
-
/** A 95% interval is within noise when it contains 0; touching 0 does not exclude it. */
|
|
324
|
-
const containsZero = (low, high) => low <= 0 && high >= 0;
|
|
325
|
-
/** Rounded to three places, as the record stores rates and points. */
|
|
326
|
-
const round3 = (x) => Math.round(x * 1e3) / 1e3;
|
|
327
|
-
/** Linear-interpolated quantile (`q` from 0 to 1) of unsorted numbers; null for none. */
|
|
328
|
-
function quantile(values, q) {
|
|
329
|
-
if (values.length === 0) return null;
|
|
330
|
-
const sorted = [...values].sort((a, b) => a - b);
|
|
331
|
-
const at = (sorted.length - 1) * q;
|
|
332
|
-
const lo = Math.floor(at);
|
|
333
|
-
const hi = Math.ceil(at);
|
|
334
|
-
const a = sorted[lo];
|
|
335
|
-
return a + (sorted[hi] - a) * (at - lo);
|
|
336
|
-
}
|
|
337
|
-
const median = (values) => quantile(values, .5);
|
|
338
|
-
/** Above this many ways to split the pooled values, the normal approximation stands in. */
|
|
339
|
-
const EXACT_LIMIT = 2e5;
|
|
340
|
-
const choose = (n, k) => {
|
|
341
|
-
let r = 1;
|
|
342
|
-
for (let i = 1; i <= k; i++) r = r * (n - k + i) / i;
|
|
343
|
-
return Math.round(r);
|
|
344
|
-
};
|
|
345
|
-
/**
|
|
346
|
-
* Two-sided Mann–Whitney (Wilcoxon rank-sum) p-value for `xs` against `ys`: exact, by every split
|
|
347
|
-
* of the pooled values, with mid-ranks for ties; the normal approximation with a tie correction
|
|
348
|
-
* above `EXACT_LIMIT` splits. `sign` is the sign of `xs`'s rank sum minus its expectation: below 0,
|
|
349
|
-
* `xs` ranks lower. `floor` is the smallest p any split of these pooled values could give, so ties
|
|
350
|
-
* count: `[2, 2, 2, 3]` against `[2, 2, 3, 3]` can never go below 10/70. Null when either side is
|
|
351
|
-
* empty.
|
|
352
|
-
*/
|
|
353
|
-
function mannWhitney(xs, ys) {
|
|
354
|
-
const n1 = xs.length;
|
|
355
|
-
const n2 = ys.length;
|
|
356
|
-
if (n1 === 0 || n2 === 0) return null;
|
|
357
|
-
const pooled = [...xs, ...ys].map((v, i) => ({
|
|
358
|
-
v,
|
|
359
|
-
i
|
|
360
|
-
})).sort((a, b) => a.v - b.v);
|
|
361
|
-
const ranks = new Array(pooled.length);
|
|
362
|
-
for (let i = 0; i < pooled.length;) {
|
|
363
|
-
let j = i;
|
|
364
|
-
while (j + 1 < pooled.length && pooled[j + 1]?.v === pooled[i]?.v) j++;
|
|
365
|
-
for (let k = i; k <= j; k++) ranks[pooled[k]?.i] = (i + j) / 2 + 1;
|
|
366
|
-
i = j + 1;
|
|
367
|
-
}
|
|
368
|
-
const n = n1 + n2;
|
|
369
|
-
const expected = n1 * (n + 1) / 2;
|
|
370
|
-
const deviation = ranks.slice(0, n1).reduce((a, b) => a + b, 0) - expected;
|
|
371
|
-
const sign = Math.sign(deviation);
|
|
372
|
-
const observed = Math.abs(deviation);
|
|
373
|
-
if (choose(n, n1) > EXACT_LIMIT) {
|
|
374
|
-
const counts = /* @__PURE__ */ new Map();
|
|
375
|
-
for (const { v } of pooled) counts.set(v, (counts.get(v) ?? 0) + 1);
|
|
376
|
-
const ties = [...counts.values()].reduce((a, t) => a + t ** 3 - t, 0);
|
|
377
|
-
const variance = n1 * n2 / 12 * (n + 1 - ties / (n * (n - 1)));
|
|
378
|
-
if (variance <= 0) return {
|
|
379
|
-
p: 1,
|
|
380
|
-
sign,
|
|
381
|
-
floor: 1
|
|
382
|
-
};
|
|
383
|
-
const pOf = (d) => Math.min(1, 2 * (1 - normalCdf(Math.max(0, d - .5) / Math.sqrt(variance))));
|
|
384
|
-
const sorted = pooled.map(({ i }) => ranks[i]);
|
|
385
|
-
const sum = (rs) => rs.reduce((a, b) => a + b, 0);
|
|
386
|
-
const widest = Math.max(Math.abs(sum(sorted.slice(0, n1)) - expected), Math.abs(sum(sorted.slice(n - n1)) - expected));
|
|
387
|
-
return {
|
|
388
|
-
p: pOf(observed),
|
|
389
|
-
sign,
|
|
390
|
-
floor: pOf(widest)
|
|
391
|
-
};
|
|
392
|
-
}
|
|
393
|
-
let extreme = 0;
|
|
394
|
-
let widest = 0;
|
|
395
|
-
let atWidest = 0;
|
|
396
|
-
let total = 0;
|
|
397
|
-
const pick = (start, left, sum) => {
|
|
398
|
-
if (left === 0) {
|
|
399
|
-
total++;
|
|
400
|
-
const d = Math.abs(sum - expected);
|
|
401
|
-
if (d >= observed - 1e-9) extreme++;
|
|
402
|
-
if (d > widest + 1e-9) {
|
|
403
|
-
widest = d;
|
|
404
|
-
atWidest = 1;
|
|
405
|
-
} else if (d >= widest - 1e-9) atWidest++;
|
|
406
|
-
return;
|
|
407
|
-
}
|
|
408
|
-
for (let i = start; i <= n - left; i++) pick(i + 1, left - 1, sum + ranks[i]);
|
|
409
|
-
};
|
|
410
|
-
pick(0, n1, 0);
|
|
411
|
-
return {
|
|
412
|
-
p: extreme / total,
|
|
413
|
-
sign,
|
|
414
|
-
floor: atWidest / total
|
|
415
|
-
};
|
|
416
|
-
}
|
|
417
|
-
/**
|
|
418
|
-
* The smallest two-sided p an exact rank test can reach with these group sizes and no ties: for
|
|
419
|
-
* planning `runs` before there's data. On data, `mannWhitney`'s `floor` counts the ties.
|
|
420
|
-
*/
|
|
421
|
-
const minimumP = (n1, n2) => n1 === 0 || n2 === 0 ? 1 : Math.min(1, 2 / choose(n1 + n2, n1));
|
|
422
|
-
/**
|
|
423
|
-
* Counted attempts a side before a pass-rate lift can be called: below it, every view says
|
|
424
|
-
* `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
|
|
425
|
-
* `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
|
|
426
|
-
* makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
|
|
427
|
-
* the time when nothing changed.
|
|
428
|
-
*/
|
|
429
|
-
const EFFECT_MIN_COUNTED = 4;
|
|
430
|
-
/**
|
|
431
|
-
* The difference of two paired pass rates (variant minus control) with Newcombe's score interval
|
|
432
|
-
* for paired proportions (his method 10): each marginal rate's Wilson interval, combined with the
|
|
433
|
-
* correlation between the sides, so concordant pairs narrow the interval.
|
|
434
|
-
*/
|
|
435
|
-
function pairedLiftInterval(c) {
|
|
436
|
-
const { both: a, variantOnly: b, controlOnly: k, neither: d } = c;
|
|
437
|
-
const n = a + b + k + d;
|
|
438
|
-
if (n === 0) return {
|
|
439
|
-
points: 0,
|
|
440
|
-
low: -1,
|
|
441
|
-
high: 1
|
|
442
|
-
};
|
|
443
|
-
const p1 = (a + b) / n;
|
|
444
|
-
const p2 = (a + k) / n;
|
|
445
|
-
const v = wilson(a + b, n);
|
|
446
|
-
const w = wilson(a + k, n);
|
|
447
|
-
const denominator = (a + b) * (k + d) * (a + k) * (b + d);
|
|
448
|
-
const numerator = a * d - b * k > 0 ? Math.max(a * d - b * k - n / 2, 0) : a * d - b * k;
|
|
449
|
-
const phi = denominator === 0 ? 0 : numerator / Math.sqrt(denominator);
|
|
450
|
-
const points = (b - k) / n;
|
|
451
|
-
const dl = Math.sqrt(Math.max(0, (p1 - v.low) ** 2 - 2 * phi * (p1 - v.low) * (w.high - p2) + (w.high - p2) ** 2));
|
|
452
|
-
const du = Math.sqrt(Math.max(0, (v.high - p1) ** 2 - 2 * phi * (v.high - p1) * (p2 - w.low) + (p2 - w.low) ** 2));
|
|
453
|
-
return {
|
|
454
|
-
points,
|
|
455
|
-
low: Math.max(-1, points - dl),
|
|
456
|
-
high: Math.min(1, points + du)
|
|
457
|
-
};
|
|
458
|
-
}
|
|
459
|
-
/** McNemar's exact test: the two-sided binomial p of the discordant pairs' split at ½. */
|
|
460
|
-
function mcnemar(variantOnly, controlOnly) {
|
|
461
|
-
const n = variantOnly + controlOnly;
|
|
462
|
-
if (n === 0) return 1;
|
|
463
|
-
const low = Math.min(variantOnly, controlOnly);
|
|
464
|
-
let logTerm = -n * Math.LN2;
|
|
465
|
-
let logTail = logTerm;
|
|
466
|
-
for (let i = 0; i < low; i++) {
|
|
467
|
-
logTerm += Math.log((n - i) / (i + 1));
|
|
468
|
-
const top = Math.max(logTail, logTerm);
|
|
469
|
-
logTail = top + Math.log1p(Math.exp(Math.min(logTail, logTerm) - top));
|
|
470
|
-
}
|
|
471
|
-
return Math.min(1, 2 * Math.exp(logTail));
|
|
472
|
-
}
|
|
473
|
-
/** Two-sided 95% Student t quantiles for 1 to 30 degrees of freedom. */
|
|
474
|
-
const T975 = [
|
|
475
|
-
12.706,
|
|
476
|
-
4.303,
|
|
477
|
-
3.182,
|
|
478
|
-
2.776,
|
|
479
|
-
2.571,
|
|
480
|
-
2.447,
|
|
481
|
-
2.365,
|
|
482
|
-
2.306,
|
|
483
|
-
2.262,
|
|
484
|
-
2.228,
|
|
485
|
-
2.201,
|
|
486
|
-
2.179,
|
|
487
|
-
2.16,
|
|
488
|
-
2.145,
|
|
489
|
-
2.131,
|
|
490
|
-
2.12,
|
|
491
|
-
2.11,
|
|
492
|
-
2.101,
|
|
493
|
-
2.093,
|
|
494
|
-
2.086,
|
|
495
|
-
2.08,
|
|
496
|
-
2.074,
|
|
497
|
-
2.069,
|
|
498
|
-
2.064,
|
|
499
|
-
2.06,
|
|
500
|
-
2.056,
|
|
501
|
-
2.052,
|
|
502
|
-
2.048,
|
|
503
|
-
2.045,
|
|
504
|
-
2.042
|
|
505
|
-
];
|
|
506
|
-
/** The two-sided 95% t quantile; above 30 degrees of freedom, the Cornish–Fisher expansion. */
|
|
507
|
-
function t975(df) {
|
|
508
|
-
if (df < 1) return Number.POSITIVE_INFINITY;
|
|
509
|
-
const tabled = T975[df - 1];
|
|
510
|
-
if (tabled !== void 0) return tabled;
|
|
511
|
-
return Z + (Z ** 3 + Z) / (4 * df) + (5 * Z ** 5 + 16 * Z ** 3 + 3 * Z) / (96 * df * df);
|
|
512
|
-
}
|
|
513
|
-
/**
|
|
514
|
-
* The mean of `values` with a 95% t interval over them: with each value one case, attempts
|
|
515
|
-
* within a case are never counted as independent. A single value gives an unbounded interval.
|
|
516
|
-
*/
|
|
517
|
-
function meanInterval(values) {
|
|
518
|
-
const n = values.length;
|
|
519
|
-
if (n === 0) return {
|
|
520
|
-
mean: 0,
|
|
521
|
-
low: Number.NEGATIVE_INFINITY,
|
|
522
|
-
high: Number.POSITIVE_INFINITY
|
|
523
|
-
};
|
|
524
|
-
const mean = values.reduce((s, v) => s + v, 0) / n;
|
|
525
|
-
if (n === 1) return {
|
|
526
|
-
mean,
|
|
527
|
-
low: Number.NEGATIVE_INFINITY,
|
|
528
|
-
high: Number.POSITIVE_INFINITY
|
|
529
|
-
};
|
|
530
|
-
const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1);
|
|
531
|
-
const half = t975(n - 1) * Math.sqrt(variance / n);
|
|
532
|
-
return {
|
|
533
|
-
mean,
|
|
534
|
-
low: mean - half,
|
|
535
|
-
high: mean + half
|
|
536
|
-
};
|
|
537
|
-
}
|
|
538
|
-
/**
|
|
539
|
-
* Two-sided Wilcoxon signed-rank p for paired differences: zeros dropped, mid-ranks for ties,
|
|
540
|
-
* exact over every sign assignment up to 20 non-zero differences, else the normal approximation.
|
|
541
|
-
* `sign` is the sign of the positive differences' rank sum minus its expectation: below 0, the
|
|
542
|
-
* negative differences outrank the positive ones. Null when every difference is zero. Ties are
|
|
543
|
-
* exact equality, so form each difference of two rates with one division (`(a·d − c·b) / (b·d)`),
|
|
544
|
-
* never as `a/b − c/d`.
|
|
545
|
-
*/
|
|
546
|
-
function signedRank(differences) {
|
|
547
|
-
const d = differences.filter((x) => x !== 0);
|
|
548
|
-
const n = d.length;
|
|
549
|
-
if (n === 0) return null;
|
|
550
|
-
const sorted = d.map((v, i) => ({
|
|
551
|
-
a: Math.abs(v),
|
|
552
|
-
i
|
|
553
|
-
})).sort((x, y) => x.a - y.a);
|
|
554
|
-
const ranks = new Array(n);
|
|
555
|
-
for (let i = 0; i < n;) {
|
|
556
|
-
let j = i;
|
|
557
|
-
while (j + 1 < n && sorted[j + 1]?.a === sorted[i]?.a) j++;
|
|
558
|
-
for (let k = i; k <= j; k++) ranks[sorted[k]?.i] = (i + j) / 2 + 1;
|
|
559
|
-
i = j + 1;
|
|
560
|
-
}
|
|
561
|
-
const total = n * (n + 1) / 2;
|
|
562
|
-
const plus = d.reduce((s, v, i) => v > 0 ? s + ranks[i] : s, 0);
|
|
563
|
-
const sign = Math.sign(plus - total / 2);
|
|
564
|
-
const observed = Math.abs(plus - total / 2);
|
|
565
|
-
if (n > 20) {
|
|
566
|
-
const counts = /* @__PURE__ */ new Map();
|
|
567
|
-
for (const { a } of sorted) counts.set(a, (counts.get(a) ?? 0) + 1);
|
|
568
|
-
const ties = [...counts.values()].reduce((s, t) => s + t ** 3 - t, 0);
|
|
569
|
-
const variance = n * (n + 1) * (2 * n + 1) / 24 - ties / 48;
|
|
570
|
-
if (variance <= 0) return {
|
|
571
|
-
p: 1,
|
|
572
|
-
sign
|
|
573
|
-
};
|
|
574
|
-
const z = Math.max(0, observed - .5) / Math.sqrt(variance);
|
|
575
|
-
return {
|
|
576
|
-
p: Math.min(1, 2 * (1 - normalCdf(z))),
|
|
577
|
-
sign
|
|
578
|
-
};
|
|
579
|
-
}
|
|
580
|
-
let extreme = 0;
|
|
581
|
-
for (let mask = 0; mask < 2 ** n; mask++) {
|
|
582
|
-
let sum = 0;
|
|
583
|
-
for (let i = 0; i < n; i++) if (mask & 1 << i) sum += ranks[i];
|
|
584
|
-
if (Math.abs(sum - total / 2) >= observed - 1e-9) extreme++;
|
|
585
|
-
}
|
|
586
|
-
return {
|
|
587
|
-
p: extreme / 2 ** n,
|
|
588
|
-
sign
|
|
589
|
-
};
|
|
590
|
-
}
|
|
591
|
-
function normalCdf(z) {
|
|
592
|
-
const t = 1 / (1 + .3275911 * Math.abs(z) / Math.SQRT2);
|
|
593
|
-
const erf = 1 - t * (.254829592 + t * (-.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * Math.exp(-(z * z) / 2);
|
|
594
|
-
return z >= 0 ? (1 + erf) / 2 : (1 - erf) / 2;
|
|
595
|
-
}
|
|
596
|
-
//#endregion
|
|
597
530
|
//#region src/view/effort.ts
|
|
598
531
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
599
532
|
const EFFORT_ALPHA = .05;
|
|
@@ -771,7 +704,7 @@ function effortChangeLabel(base, head) {
|
|
|
771
704
|
return `${core} · tokens ${formatTokens(base.tokens.median)} → ${formatTokens(head.tokens.median)} (${formatRatio(tok)})`;
|
|
772
705
|
}
|
|
773
706
|
//#endregion
|
|
774
|
-
//#region src/view/
|
|
707
|
+
//#region src/view/judgments.ts
|
|
775
708
|
/** Show each key until this many; then the minimum score plus a count. */
|
|
776
709
|
const COLLAPSE_AFTER = 2;
|
|
777
710
|
const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
@@ -844,7 +777,7 @@ function keyTallies(e, cellKey) {
|
|
|
844
777
|
const expectLabels = /* @__PURE__ */ new Map();
|
|
845
778
|
for (const a of e.attempts) {
|
|
846
779
|
if (a.cell !== cellKey || !counted(a)) continue;
|
|
847
|
-
for (const {
|
|
780
|
+
for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
|
|
848
781
|
const row = byKey.get(j.key) ?? {
|
|
849
782
|
label: "",
|
|
850
783
|
labelled: false,
|
|
@@ -855,7 +788,7 @@ function keyTallies(e, cellKey) {
|
|
|
855
788
|
row.counted++;
|
|
856
789
|
if (j.verdict === "pass") row.pass++;
|
|
857
790
|
byKey.set(j.key, row);
|
|
858
|
-
if (
|
|
791
|
+
if (labeled && !expectLabels.has(j.key)) expectLabels.set(j.key, label);
|
|
859
792
|
}
|
|
860
793
|
}
|
|
861
794
|
const labels = uniqueJudgeLabels([...byKey].map(([key, row]) => ({
|
|
@@ -908,7 +841,7 @@ function cellJudgeRates(e, cellKey) {
|
|
|
908
841
|
}));
|
|
909
842
|
}
|
|
910
843
|
/**
|
|
911
|
-
* The judge rates a cell's summary line adds to its checks: a
|
|
844
|
+
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
912
845
|
* of the cell's checks against its control is already there.
|
|
913
846
|
*/
|
|
914
847
|
function summaryJudgeRates(e, cell) {
|
|
@@ -924,9 +857,9 @@ function judgeRatesLabel(rates) {
|
|
|
924
857
|
}).join(" · ");
|
|
925
858
|
}
|
|
926
859
|
/** Each key, or the lowest score plus how many judges there were. */
|
|
927
|
-
function collapsedJudges(
|
|
928
|
-
const labels = uniqueJudgeLabels(
|
|
929
|
-
const items =
|
|
860
|
+
function collapsedJudges(judgments) {
|
|
861
|
+
const labels = uniqueJudgeLabels(judgments);
|
|
862
|
+
const items = judgments.map((j) => ({
|
|
930
863
|
verdict: j.verdict,
|
|
931
864
|
score: j.score,
|
|
932
865
|
threshold: j.threshold,
|
|
@@ -943,28 +876,28 @@ function collapsedJudges(judgements) {
|
|
|
943
876
|
};
|
|
944
877
|
}
|
|
945
878
|
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
946
|
-
function collapsedJudgeLabel(
|
|
947
|
-
if (!
|
|
948
|
-
const { shown, extra } = collapsedJudges(
|
|
879
|
+
function collapsedJudgeLabel(judgments) {
|
|
880
|
+
if (!judgments.length) return null;
|
|
881
|
+
const { shown, extra } = collapsedJudges(judgments);
|
|
949
882
|
const parts = shown.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold.toFixed(2)}`);
|
|
950
883
|
if (extra) parts.push(`${extra} judges`);
|
|
951
884
|
return parts.join(" · ");
|
|
952
885
|
}
|
|
953
886
|
const JUDGE_ERROR = /^judge failed for /;
|
|
954
887
|
/**
|
|
955
|
-
* Each
|
|
956
|
-
* failed judge call, which recorded no
|
|
957
|
-
* its
|
|
888
|
+
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
889
|
+
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
890
|
+
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
958
891
|
* reaches is named by its question.
|
|
959
892
|
*/
|
|
960
|
-
function
|
|
893
|
+
function labeledJudgments(a) {
|
|
961
894
|
const passes = (a.assertions ?? []).filter((x) => /(^|\.)toPass$/.test(x.matcher) && !JUDGE_ERROR.test(x.message ?? ""));
|
|
962
895
|
const zip = passes.length === a.judgements.length;
|
|
963
|
-
return a.judgements.map((
|
|
896
|
+
return a.judgements.map((judgment, i) => {
|
|
964
897
|
const x = zip ? passes[i] : void 0;
|
|
965
898
|
return {
|
|
966
|
-
|
|
967
|
-
label: x?.labelled ? x.label : judgeText(
|
|
899
|
+
judgment,
|
|
900
|
+
label: x?.labelled ? x.label : judgeText(judgment.question),
|
|
968
901
|
labelled: x?.labelled === true,
|
|
969
902
|
...x ? { check: x.label } : {}
|
|
970
903
|
};
|
|
@@ -986,6 +919,26 @@ function checkLabels(a) {
|
|
|
986
919
|
return x?.labelled ? x.label : void 0;
|
|
987
920
|
});
|
|
988
921
|
}
|
|
922
|
+
/**
|
|
923
|
+
* Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
|
|
924
|
+
* are exactly as many as its checks; empty otherwise.
|
|
925
|
+
*/
|
|
926
|
+
function checksByLabel(a) {
|
|
927
|
+
const succeeds = (a.assertions ?? []).filter((x) => /(^|\.)toSucceed$/.test(x.matcher));
|
|
928
|
+
if (succeeds.length !== a.checks.length) return /* @__PURE__ */ new Map();
|
|
929
|
+
return new Map(succeeds.map((x, i) => [x.label, a.checks[i]]));
|
|
930
|
+
}
|
|
931
|
+
/**
|
|
932
|
+
* How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
|
|
933
|
+
* when it had none or passed on its first try.
|
|
934
|
+
*/
|
|
935
|
+
function triesNote(exitCode, tried, of) {
|
|
936
|
+
if (of <= 1 || exitCode === 0 && tried <= 1) return null;
|
|
937
|
+
if (exitCode === 0) return `passed on try ${tried} of ${of}`;
|
|
938
|
+
return tried >= of ? `failed all ${of} tries` : `failed on try ${tried} of ${of}`;
|
|
939
|
+
}
|
|
940
|
+
/** {@link triesNote} for a recorded check. */
|
|
941
|
+
const checkTries = (c) => c.retries && c.tries ? triesNote(c.exitCode, c.tries.length, c.retries + 1) : null;
|
|
989
942
|
/** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
|
|
990
943
|
function commandLine(command, max = 80) {
|
|
991
944
|
const lines = command.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
@@ -1101,7 +1054,6 @@ const shortSha = (sha) => sha ? sha.slice(0, 7) : "no git";
|
|
|
1101
1054
|
* Edit calls become one file edit per file they change, so they're counted apart, never summed.
|
|
1102
1055
|
*/
|
|
1103
1056
|
function activityLabel(a) {
|
|
1104
|
-
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
1105
1057
|
return `${plural(a.tools, "tool call")} · ${plural(a.fileEdits, "file edit")}`;
|
|
1106
1058
|
}
|
|
1107
1059
|
/**
|
|
@@ -1298,7 +1250,7 @@ function cellRows(e) {
|
|
|
1298
1250
|
attempts: e.attempts.filter((a) => a.cell === cell.key)
|
|
1299
1251
|
}));
|
|
1300
1252
|
}
|
|
1301
|
-
/** One row per
|
|
1253
|
+
/** One row per pairwise question, with each cut's win rates in the record's order. */
|
|
1302
1254
|
function answerRows(e) {
|
|
1303
1255
|
return (e.answers ?? []).map((a) => {
|
|
1304
1256
|
const rates = (cut) => a.rates.filter((r) => r.cut === cut).map((r) => ({
|
|
@@ -1321,6 +1273,33 @@ function answerRows(e) {
|
|
|
1321
1273
|
};
|
|
1322
1274
|
});
|
|
1323
1275
|
}
|
|
1276
|
+
/** One row per question asked of each attempt on its own, in first-asked order. */
|
|
1277
|
+
function attemptAnswerRows(e) {
|
|
1278
|
+
const rows = /* @__PURE__ */ new Map();
|
|
1279
|
+
for (const a of e.attempts) for (const q of a.answers ?? []) {
|
|
1280
|
+
const row = rows.get(q.question) ?? {
|
|
1281
|
+
question: q.question,
|
|
1282
|
+
model: q.resolvedModel ?? q.model,
|
|
1283
|
+
choices: q.choices ? q.choices.map((choice) => ({
|
|
1284
|
+
choice,
|
|
1285
|
+
n: 0
|
|
1286
|
+
})) : null,
|
|
1287
|
+
answers: []
|
|
1288
|
+
};
|
|
1289
|
+
rows.set(q.question, row);
|
|
1290
|
+
const picked = q.error === void 0 && row.choices?.find((c) => c.choice === q.choice);
|
|
1291
|
+
if (picked) picked.n++;
|
|
1292
|
+
row.answers.push({
|
|
1293
|
+
attempt: a.id,
|
|
1294
|
+
outcome: a.outcome,
|
|
1295
|
+
choice: q.choice ?? null,
|
|
1296
|
+
text: q.text,
|
|
1297
|
+
error: q.error ?? null,
|
|
1298
|
+
input: q.input?.path ?? null
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
return [...rows.values()];
|
|
1302
|
+
}
|
|
1324
1303
|
/** Whether an eval needs the cells table: several cells, or several runs of one. */
|
|
1325
1304
|
const isMatrix = (e) => e.cells.length > 1 || e.cells.some((c) => c.runs > 1);
|
|
1326
1305
|
/** One eval across runs, oldest first. */
|
|
@@ -1349,6 +1328,7 @@ const COMPARE_ORDER = [
|
|
|
1349
1328
|
"regressed",
|
|
1350
1329
|
"fixed",
|
|
1351
1330
|
"drift",
|
|
1331
|
+
"gate",
|
|
1352
1332
|
"infra",
|
|
1353
1333
|
"added",
|
|
1354
1334
|
"removed",
|
|
@@ -1360,6 +1340,8 @@ const COMPARE_ORDER = [
|
|
|
1360
1340
|
* the eval line labels it `gated`: `2/4 · pass@1 100% gated`.
|
|
1361
1341
|
*/
|
|
1362
1342
|
const hasUngatedCells = (cells) => cells.some((c) => c.gate === "none");
|
|
1343
|
+
/** A study's pass: every cell `gate: 'none'`, so nothing it saw passed or failed it. */
|
|
1344
|
+
const isNotGatedPass = (e) => e.outcome === "pass" && e.cells.length > 0 && e.cells.every((c) => c.gate === "none");
|
|
1363
1345
|
/**
|
|
1364
1346
|
* An expected miss: a failed attempt in an ungated cell (a control, by default). It's reported
|
|
1365
1347
|
* apart from failures, since it fails neither its eval nor the run.
|
|
@@ -1384,8 +1366,9 @@ function gatedPassAt1(cells) {
|
|
|
1384
1366
|
/** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
|
|
1385
1367
|
function compareView(base, head, records = {}) {
|
|
1386
1368
|
const rows = compare(base, head).map((row) => {
|
|
1387
|
-
const
|
|
1388
|
-
const
|
|
1369
|
+
const evalOf = (run) => row.kind === "family" ? void 0 : run.evals.find((e) => e.id === row.id);
|
|
1370
|
+
const b = evalOf(base);
|
|
1371
|
+
const h = evalOf(head);
|
|
1389
1372
|
const baseKeys = new Set(b?.cells.map((c) => c.key));
|
|
1390
1373
|
const shared = row.cells && new Set(h?.cells.filter((c) => baseKeys.has(c.key)).map((c) => c.key));
|
|
1391
1374
|
const passAt1 = (e) => {
|
|
@@ -1397,8 +1380,8 @@ function compareView(base, head, records = {}) {
|
|
|
1397
1380
|
slug: h?.slug ?? b?.slug ?? null,
|
|
1398
1381
|
basePassAt1: passAt1(b),
|
|
1399
1382
|
headPassAt1: passAt1(h),
|
|
1400
|
-
baseScore: meanScore(records.base?.evals.find((e) => e.id === row.id)),
|
|
1401
|
-
headScore: meanScore(records.head?.evals.find((e) => e.id === row.id)),
|
|
1383
|
+
baseScore: b ? meanScore(records.base?.evals.find((e) => e.id === row.id)) : null,
|
|
1384
|
+
headScore: h ? meanScore(records.head?.evals.find((e) => e.id === row.id)) : null,
|
|
1402
1385
|
perCell: row.perCell ?? []
|
|
1403
1386
|
};
|
|
1404
1387
|
});
|
|
@@ -1442,31 +1425,11 @@ function transcriptItems(lines) {
|
|
|
1442
1425
|
};
|
|
1443
1426
|
});
|
|
1444
1427
|
}
|
|
1445
|
-
const HEADER = /^diff --git a\/(.+?) b\/(.+)$/m;
|
|
1446
|
-
/** Splits a git patch into one entry per file, with line counts. */
|
|
1447
|
-
function splitPatch(patch) {
|
|
1448
|
-
if (!patch.trim()) return [];
|
|
1449
|
-
return patch.split(/(?=^diff --git )/m).filter((part) => part.startsWith("diff --git ")).map((text) => {
|
|
1450
|
-
const header = HEADER.exec(text);
|
|
1451
|
-
const path = header?.[2] ?? header?.[1] ?? "unknown";
|
|
1452
|
-
let added = 0;
|
|
1453
|
-
let removed = 0;
|
|
1454
|
-
for (const line of text.split("\n")) if (line.startsWith("+") && !line.startsWith("+++ ")) added++;
|
|
1455
|
-
else if (line.startsWith("-") && !line.startsWith("--- ")) removed++;
|
|
1456
|
-
return {
|
|
1457
|
-
path,
|
|
1458
|
-
status: /^new file mode/m.test(text) ? "added" : /^deleted file mode/m.test(text) ? "deleted" : /^rename from/m.test(text) ? "renamed" : /^(Binary files|GIT binary patch)/m.test(text) ? "binary" : "modified",
|
|
1459
|
-
added,
|
|
1460
|
-
removed,
|
|
1461
|
-
text
|
|
1462
|
-
};
|
|
1463
|
-
});
|
|
1464
|
-
}
|
|
1465
1428
|
const findEval = (record, slug) => record.evals.find((e) => e.slug === slug);
|
|
1466
1429
|
const findAttempt = (e, attemptId) => e.attempts.find((a) => a.id === attemptId);
|
|
1467
1430
|
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
1468
1431
|
const defaultAttempt = (e) => e.attempts.find((a) => a.outcome === "fail" || a.outcome === "infra_error") ?? e.attempts[0];
|
|
1469
1432
|
//#endregion
|
|
1470
|
-
export {
|
|
1433
|
+
export { cellContrast as $, shortSha as A, commandLine as B, isUngatedMiss as C, gateLabel as Ct, runOutcome as D, runLead as E, worstOutcome as Et, checkLabels as F, triesNote as G, judgeRatesLabel as H, checkTries as I, EFFORT_ALPHA as J, uniqueJudgeLabels as K, checksByLabel as L, transcriptItems as M, cellJudgeRates as N, runRows as O, cellUsage as P, bothUnderATenth as Q, collapsedJudgeLabel as R, isNotGatedPass as S, cellOutcome as St, parseTranscript as T, passRateOf as Tt, labeledJudgments as U, judgeLabel as V, summaryJudgeRates as W, TOKENS_UNREPORTED as X, EFFORT_MIN_RUNS as Y, VERDICT_TEXT as Z, formatUsage as _, newRows as _t, buildIndex as a, effortLabel as at, indexFiles as b, stackChanges as bt, defaultAttempt as c, effortSummary as ct, evalTree as d, formatRatio as dt, cellEffort as et, failedJudgeLabels as f, formatSeconds as ft, formatPercent as g, compare as gt, formatDuration as h, cellsChanged as ht, attemptAnswerRows as i, effortChangeLabel as it, splitEvalId as j, runtimeLabel as k, evalHistory as l, effortVerdict as lt, findEval as m, addedRow as mt, activityLabel as n, contrastEffort as nt, cellRows as o, effortOf as ot, findAttempt as p, tokenValues as pt, DIRECTIONAL_MIN as q, answerRows as r, directional as rt, compareView as s, effortSample as st, COMPARE_ORDER as t, cellSamples as tt, evalMarkOutcome as u, formatCount as ut, harnessLabel as v, rowKey as vt, languageLabel as w, gatePart as wt, isMatrix as x, whyChanged as xt, hasUngatedCells as y, sourceChanges as yt, collapsedJudges as z };
|
|
1471
1434
|
|
|
1472
|
-
//# sourceMappingURL=model-
|
|
1435
|
+
//# sourceMappingURL=model-CTnyvh7H.js.map
|