reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,15 +1,23 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { n as
|
|
5
|
-
import "./
|
|
6
|
-
import
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
1
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
+
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
3
|
+
import { at as questionShows, rt as DEFAULT_QUESTION_STATE } from "./config-k6m7XbAz.js";
|
|
4
|
+
import { n as clip, r as fenced } from "./text-DTigcA64.js";
|
|
5
|
+
import { a as pairTallies, f as splitsLabel, l as sampleReasons, r as headToHeadLines } from "./pairs-puYi-zj2.js";
|
|
6
|
+
import "./stats-hfpuYsAM.js";
|
|
7
|
+
import { l as resolveAnalysisModel, n as callAnalysisModel, t as analysisModelName, u as SYSTEM_PROMPT } from "./model-Bq3G0OG1.js";
|
|
8
|
+
import { n as formatTokens } from "./usage-BNY1ABrU.js";
|
|
9
|
+
import { t as ANALYSIS_PROMPT_VERSION } from "./analysis-C2-aEbr7.js";
|
|
10
|
+
import { Q as bothUnderATenth, U as labeledJudgments, gt as compare } from "./model-CTnyvh7H.js";
|
|
11
|
+
import { g as familyRateLabel, h as familyLiftLabel, p as familyEffortLabel } from "./suite-Co8cwelb.js";
|
|
12
|
+
import { r as runHeader } from "./remarks-D_3qhBWU.js";
|
|
13
|
+
import { i as checkCommand, o as commandSummary } from "./cli-spec-D0qyrPZi.js";
|
|
14
|
+
import { o as parseRecord, p as scratchDir } from "./files-Bi461Be2.js";
|
|
15
|
+
import { a as pickBaseline, o as resolveRun, t as RefError } from "./views-BFO1BLKu.js";
|
|
16
|
+
import { c as lines, n as allRuns, t as CommandError, u as reachability } from "./commands-BU6YJCz9.js";
|
|
17
|
+
import { r as contrastsOf } from "./contrasts-DLZF4dGF.js";
|
|
18
|
+
import { l as TOOL_CLASSES, s as openHistoryDb } from "./db-CpsifVln.js";
|
|
19
|
+
import { t as author } from "./provenance-BizwUn3h.js";
|
|
20
|
+
import { a as readCached, c as writeCached, o as saveAnalysis } from "./files-H24pz0hL.js";
|
|
13
21
|
import { basename, join } from "node:path";
|
|
14
22
|
import { readFile } from "node:fs/promises";
|
|
15
23
|
//#region src/analysis/digest.ts
|
|
@@ -19,8 +27,9 @@ const RULES = [
|
|
|
19
27
|
`Effort (tool calls, agent time, and tokens when both sides have usage samples) is compared with an exact Mann-Whitney test: "fewer"/"more" at p < 0.05; "insufficient" when no split of the attempts' values can reach it (2 vs 2 cannot, and ties raise the floor) — never because a harness dropped usage. Attempts without usage are left out of the token sample only. If either side then has no token sample, the token metric is omitted (no line, not insufficient). A Goose cell does not blank tokens on Cursor cells in the same study. Pooled across evals, tokens are an effect only when at least 2/3 of the strata that reported usage agree. Effort's separated (given from 3 vs 3) says, per metric, that the two sides' ranges do not overlap: every attempt on one side beat every attempt on the other. insufficient with separated true is a directional signal: report the ratio, the n and "no overlap, not significant" as directional, never as an effect. separated false is no direction at all.`,
|
|
20
28
|
"A family fact is one pooled evaluate.each: its cases are paired samples of one suite, so its lift (Newcombe paired interval and McNemar at one attempt a case, a t interval over cases otherwise) is the result for those cases; their own pass-rate facts carry no lift. insufficient (fewer than 6 discordant pairs or 6 cases) is not measured. A family's minLift gates the pooled lower bound.",
|
|
21
29
|
"Each factor claim has a tier: controlled (two cells in one run that differ in one factor), pooled (the same contrast over comparable runs, and across evals), observed (a change between runs, which is weaker; \"confounded\" when several parts changed), not_measured (one level only).",
|
|
22
|
-
`A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "
|
|
30
|
+
`A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "mixed": it passed and failed, too few times to call it flaky.`,
|
|
23
31
|
"Drift marked noise changed only an agent CLI's own runtime state (e.g. ~/.cursor/cli-config.json): not a real change, and those runs stay comparable.",
|
|
32
|
+
"Drift on the gate is a policy change: the gate scores the same attempts, so the runs stay comparable. A compare row with status gate flipped only because its gate did (gate.rescored, base's attempts under head's gate, matches head): never call it a regression or a fix.",
|
|
24
33
|
"An eval where every cell passes at least 95% is saturated: pass@1 cannot tell cells apart there, effort can.",
|
|
25
34
|
"Judge verdicts carry no recorded reason; an explanation of a failed verdict is your reading of the cited output, and must say so.",
|
|
26
35
|
"A criteria fact lists each check (expect label, judges included) per cell: rows only for labels whose rate differs between cells, or is mixed within one; neverMet labels failed on every counted attempt of every cell, alwaysMet passed on all of them. A neverMet or alwaysMet label is the same in every arm: it can't explain a difference between arms.",
|
|
@@ -29,7 +38,7 @@ const RULES = [
|
|
|
29
38
|
"A context-use fact counts each attempt's tool calls by class (context, dependency, web, verify, workspace) and lists which placed files each arm opened: how much an arm leaned on what its environment declared versus digging in installed packages. The classes are a heuristic over tool names and paths; cite them as counts, not as proof of intent.",
|
|
30
39
|
"A contrast fact (kind contrast) is one pair of arms on one harness, a against b, when an eval has two variants or more: pass rate and lift of a over b, the checks that tell them apart (discriminates or mixed; sameOnBoth counts the rest) and effort. It answers how two variants differ from each other, not only from the control; gating stays each variant against its control.",
|
|
31
40
|
"A check (kind checks) whose verdict is both-pass or both-fail passes, or fails, on both the variant and its control: it is no evidence for or against the variant. Only discriminates rows say which requirement separated them.",
|
|
32
|
-
"Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can
|
|
41
|
+
"Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favor the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
|
|
33
42
|
];
|
|
34
43
|
const pts = (x) => x === null || x === void 0 ? null : `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
|
|
35
44
|
const liftText = (points, lo, hi) => `${pts(points)} pts (95% ${pts(lo)}…${pts(hi)})`;
|
|
@@ -45,10 +54,12 @@ function digestLiftVerdict(variantCounted, controlCounted, withinNoise) {
|
|
|
45
54
|
function evalChangedLift() {
|
|
46
55
|
return "not comparable: the eval changed";
|
|
47
56
|
}
|
|
57
|
+
/** What the eval's own source is keyed on: `source` and `revision`, else (older records) its files. */
|
|
48
58
|
function sourcesKey(e) {
|
|
59
|
+
if (e.source) return JSON.stringify([e.source, e.revision ?? null]);
|
|
49
60
|
return JSON.stringify(e.sources.map((s) => [s.path, s.sha256]));
|
|
50
61
|
}
|
|
51
|
-
/** The eval's
|
|
62
|
+
/** The eval's own source, or the sources part of its fingerprint, differs across the two runs. */
|
|
52
63
|
function evalSourcesChanged(base, head) {
|
|
53
64
|
if (sourcesKey(base) !== sourcesKey(head)) return true;
|
|
54
65
|
const prev = new Map(base.cells.map((c) => [c.key, c.fingerprintParts.sources]));
|
|
@@ -128,10 +139,10 @@ function criteriaOf(e, tag) {
|
|
|
128
139
|
const scores = /* @__PURE__ */ new Map();
|
|
129
140
|
for (const a of e.attempts) {
|
|
130
141
|
if (!counted(a)) continue;
|
|
131
|
-
for (const j of
|
|
142
|
+
for (const j of labeledJudgments(a)) {
|
|
132
143
|
if (j.check === void 0) continue;
|
|
133
144
|
const id = `${a.cell}\0${j.check}`;
|
|
134
|
-
scores.set(id, [...scores.get(id) ?? [], j.
|
|
145
|
+
scores.set(id, [...scores.get(id) ?? [], j.judgment.score]);
|
|
135
146
|
}
|
|
136
147
|
}
|
|
137
148
|
const rows = [];
|
|
@@ -190,7 +201,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
190
201
|
const any = a.files.transcript ?? a.files.judgeInput ?? a.checks.find((c) => c.log)?.log;
|
|
191
202
|
const assertions = a.assertions ?? [];
|
|
192
203
|
const failed = failedLabels(a);
|
|
193
|
-
const judges =
|
|
204
|
+
const judges = labeledJudgments(a);
|
|
194
205
|
attempts[id] = {
|
|
195
206
|
run: short.get(r.record.runId),
|
|
196
207
|
runId: r.record.runId,
|
|
@@ -204,7 +215,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
204
215
|
agentS: Math.round(a.timings.agentMs / 1e3),
|
|
205
216
|
...assertions.length ? { checks: `${assertions.filter((x) => x.status === "pass").length}/${assertions.length}` } : {},
|
|
206
217
|
...failed.length ? { failed } : {},
|
|
207
|
-
...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.
|
|
218
|
+
...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgment.score)])) } : {},
|
|
208
219
|
execs: a.checks.length,
|
|
209
220
|
artifacts: any ? `${r.record.runId}/${any.path.split("/").slice(0, 3).join("/")}` : null
|
|
210
221
|
};
|
|
@@ -279,36 +290,39 @@ async function buildDigestKeepingDropped(input) {
|
|
|
279
290
|
fact("effort", {
|
|
280
291
|
run: short.get(runId),
|
|
281
292
|
eval: tagOf(first.eval_key),
|
|
282
|
-
cells: rows.map((row) =>
|
|
283
|
-
|
|
284
|
-
n: row.n,
|
|
285
|
-
medianToolCalls: row.tools_median,
|
|
286
|
-
medianAgentS: row.agent_s_median,
|
|
287
|
-
...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
|
|
288
|
-
...row.workspace_calls_median != null ? { medianCallsByClass: Object.fromEntries(TOOL_CLASSES.flatMap((k) => {
|
|
293
|
+
cells: rows.map((row) => {
|
|
294
|
+
const byClass = TOOL_CLASSES.flatMap((k) => {
|
|
289
295
|
const m = row[`${k}_calls_median`];
|
|
290
296
|
return m ? [[k, m]] : [];
|
|
291
|
-
})
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
...row.
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
297
|
+
});
|
|
298
|
+
return {
|
|
299
|
+
cell: cellTag(runId, row.eval_key, row.cell_key),
|
|
300
|
+
n: row.n,
|
|
301
|
+
medianToolCalls: row.tools_median,
|
|
302
|
+
medianAgentS: row.agent_s_median,
|
|
303
|
+
...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
|
|
304
|
+
...byClass.length ? { medianCallsByClass: Object.fromEntries(byClass) } : {},
|
|
305
|
+
...row.control_key ? { vsControl: {
|
|
306
|
+
n: `${row.n} vs ${row.control_n}`,
|
|
307
|
+
toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
|
|
308
|
+
agentS: bothUnderATenth(Number(row.agent_s_median) * 1e3, Number(row.control_agent_s_median) * 1e3) ? "<0.1 vs <0.1" : `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
|
|
309
|
+
p: row.tools_p,
|
|
310
|
+
verdict: row.verdict,
|
|
311
|
+
timeVerdict: row.time_verdict,
|
|
312
|
+
...row.tokens_verdict ? {
|
|
313
|
+
tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
|
|
314
|
+
tokensP: row.tokens_p,
|
|
315
|
+
tokensVerdict: row.tokens_verdict
|
|
316
|
+
} : {},
|
|
317
|
+
...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
|
|
318
|
+
tools: row.separated === 1,
|
|
319
|
+
agentS: row.time_separated === 1,
|
|
320
|
+
...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
|
|
321
|
+
} } : {},
|
|
322
|
+
...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
|
|
323
|
+
} } : {}
|
|
324
|
+
};
|
|
325
|
+
})
|
|
312
326
|
});
|
|
313
327
|
}
|
|
314
328
|
const changedEvalKeys = /* @__PURE__ */ new Set();
|
|
@@ -507,7 +521,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
507
521
|
}) : failures.slice(0, 8);
|
|
508
522
|
for (const { r, e, a } of failureFacts) {
|
|
509
523
|
const failed = failedLabels(a);
|
|
510
|
-
const judges =
|
|
524
|
+
const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
|
|
511
525
|
fact("failure", {
|
|
512
526
|
attempt: idOf.get(`${r.record.runId}\0${a.key}`),
|
|
513
527
|
outcome: a.outcome,
|
|
@@ -517,9 +531,9 @@ async function buildDigestKeepingDropped(input) {
|
|
|
517
531
|
...failed.length ? { failed } : {},
|
|
518
532
|
...judges.length ? { judges: judges.map((j) => ({
|
|
519
533
|
label: clip(j.label, LABEL_CHARS),
|
|
520
|
-
question: clip(j.
|
|
521
|
-
score: j.
|
|
522
|
-
threshold: j.
|
|
534
|
+
question: clip(j.judgment.question, 300),
|
|
535
|
+
score: j.judgment.score,
|
|
536
|
+
threshold: j.judgment.threshold
|
|
523
537
|
})) } : {},
|
|
524
538
|
failingExecs: a.checks.map((c, i) => ({
|
|
525
539
|
exec: i + 1,
|
|
@@ -560,16 +574,20 @@ async function buildDigestKeepingDropped(input) {
|
|
|
560
574
|
cells: d.cells,
|
|
561
575
|
changed: d.changed,
|
|
562
576
|
...d.changed_paths ? { paths: String(d.changed_paths).split(", ") } : {},
|
|
563
|
-
verdict: d.noise ? "noise: agent runtime state only" : `real change: ${d.real_changed}`
|
|
577
|
+
verdict: d.noise ? "noise: agent runtime state only" : d.real_changed === "gate" ? "policy change: the gate scores the same attempts differently" : `real change: ${d.real_changed}`
|
|
564
578
|
});
|
|
579
|
+
const [baseHeader, headHeader] = [runHeader(base.record, true), runHeader(head.record, true)];
|
|
580
|
+
const inScope = (id) => scopeEvals.some((e) => e.id === id);
|
|
581
|
+
const familyCases = new Map([...baseHeader.families ?? [], ...headHeader.families ?? []].map((f) => [f.id, f.caseIds]));
|
|
565
582
|
fact("compare", {
|
|
566
583
|
base: "r1",
|
|
567
584
|
head: "r2",
|
|
568
|
-
rows: compare(
|
|
569
|
-
eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id),
|
|
585
|
+
rows: compare(baseHeader, headHeader).filter((row) => row.kind === "family" ? (familyCases.get(row.id) ?? []).some(inScope) : inScope(row.id)).map((row) => ({
|
|
586
|
+
...row.kind === "family" ? { family: row.id } : { eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id) },
|
|
570
587
|
status: row.status,
|
|
571
588
|
...row.changes?.length ? { changes: row.changes } : {},
|
|
572
|
-
...row.
|
|
589
|
+
...row.gate ? { gate: row.gate } : {},
|
|
590
|
+
...row.canceled ? { canceled: row.canceled } : {}
|
|
573
591
|
}))
|
|
574
592
|
});
|
|
575
593
|
}
|
|
@@ -602,6 +620,37 @@ async function buildDigestKeepingDropped(input) {
|
|
|
602
620
|
});
|
|
603
621
|
}
|
|
604
622
|
}
|
|
623
|
+
for (const r of runs) for (const e of r.record.evals) {
|
|
624
|
+
if (!evalKeys.has(e.key)) continue;
|
|
625
|
+
const byQuestion = /* @__PURE__ */ new Map();
|
|
626
|
+
for (const a of e.attempts) for (const q of a.answers ?? []) byQuestion.set(q.question, [...byQuestion.get(q.question) ?? [], {
|
|
627
|
+
attempt: idOf.get(`${r.record.runId}\0${a.key}`) ?? a.id,
|
|
628
|
+
answer: q
|
|
629
|
+
}]);
|
|
630
|
+
for (const [question, asked] of byQuestion) {
|
|
631
|
+
const first = asked[0]?.answer;
|
|
632
|
+
const answered = asked.filter((x) => x.answer.error === void 0);
|
|
633
|
+
const failed = asked.find((x) => x.answer.error !== void 0)?.answer.error;
|
|
634
|
+
fact("attempt-question", {
|
|
635
|
+
run: short.get(r.record.runId),
|
|
636
|
+
eval: tagOf(e.key),
|
|
637
|
+
question: clip(question, 120),
|
|
638
|
+
model: first.resolvedModel ?? first.model,
|
|
639
|
+
shows: questionShows(first.stateOptions, first.artifacts),
|
|
640
|
+
asked: asked.length,
|
|
641
|
+
...first.choices ? { choices: first.choices.map((choice) => ({
|
|
642
|
+
choice,
|
|
643
|
+
n: answered.filter((x) => x.answer.choice === choice).length
|
|
644
|
+
})) } : {},
|
|
645
|
+
answers: answered.slice(0, 5).map((x) => ({
|
|
646
|
+
attempt: x.attempt,
|
|
647
|
+
...x.answer.choice !== void 0 ? { choice: x.answer.choice } : {},
|
|
648
|
+
text: clip(x.answer.text, 300)
|
|
649
|
+
})),
|
|
650
|
+
...failed ? { error: clip(failed, 200) } : {}
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
}
|
|
605
654
|
for (const r of runs) for (const e of r.record.evals) {
|
|
606
655
|
if (!evalKeys.has(e.key)) continue;
|
|
607
656
|
const tag = (key) => cellTag(r.record.runId, e.key, key);
|
|
@@ -700,8 +749,8 @@ async function buildDigestKeepingDropped(input) {
|
|
|
700
749
|
for (const [i, { r, e, a }] of [...perCell.values()].slice(0, 6).entries()) {
|
|
701
750
|
const id = idOf.get(`${r.record.runId}\0${a.key}`);
|
|
702
751
|
const lines = await lastText(r.record.runId, a);
|
|
703
|
-
const judges =
|
|
704
|
-
for (const { label,
|
|
752
|
+
const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
|
|
753
|
+
for (const { label, judgment: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
|
|
705
754
|
for (const x of a.assertions ?? []) {
|
|
706
755
|
if (x.status === "pass" || /(^|\.)toPass$/.test(x.matcher)) continue;
|
|
707
756
|
lines.push(`check ${x.status}: ${clip(x.label, LABEL_CHARS)}: ${clip(x.message ?? "", 300)}`);
|
|
@@ -732,7 +781,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
732
781
|
}
|
|
733
782
|
}
|
|
734
783
|
const digest = {
|
|
735
|
-
digestVersion:
|
|
784
|
+
digestVersion: 12,
|
|
736
785
|
scope: {
|
|
737
786
|
kind: "run",
|
|
738
787
|
head: short.get(head.record.runId),
|
|
@@ -796,6 +845,77 @@ function citable(d) {
|
|
|
796
845
|
};
|
|
797
846
|
}
|
|
798
847
|
//#endregion
|
|
848
|
+
//#region src/analysis/render.ts
|
|
849
|
+
const cite = (cs) => cs.length ? ` ${cs.map((c) => `[${c}]`).join(" ")}` : "";
|
|
850
|
+
const unverified = (numbers) => numbers?.length ? `unverified: ${numbers.join(", ")}` : "";
|
|
851
|
+
const note = (c) => [c.confidence === "high" ? "" : `${c.confidence} confidence`, unverified(c.unverifiedNumbers)].filter(Boolean).join("; ");
|
|
852
|
+
function footer(doc) {
|
|
853
|
+
const v = doc.validation;
|
|
854
|
+
const tokens = doc.usage.inputTokens !== null ? `${doc.usage.inputTokens} in / ${doc.usage.outputTokens ?? "?"} out tokens · ` : "";
|
|
855
|
+
return `${doc.resolvedModel ?? doc.model} · ${doc.promptVersion} · digest v${doc.inputs.digestVersion} · ${tokens}${Math.round(doc.latencyMs / 1e3)}s · ${v.citations} citations checked, ${v.invalid.length} invalid, ${v.dropped} dropped · ${v.commands} commands checked, ${v.invalidCommands.length} rejected`;
|
|
856
|
+
}
|
|
857
|
+
/** The analysis as Markdown (`--md`, and the `.md` twin of a saved analysis). */
|
|
858
|
+
function renderMarkdown(doc) {
|
|
859
|
+
const a = doc.analysis;
|
|
860
|
+
const md = [
|
|
861
|
+
`## reeval analysis · ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
|
|
862
|
+
"",
|
|
863
|
+
`**TL;DR** (${a.confidence.overall}). ${a.tldr}`,
|
|
864
|
+
""
|
|
865
|
+
];
|
|
866
|
+
for (const [title, items] of [
|
|
867
|
+
["What happened", a.whatHappened],
|
|
868
|
+
["Results", a.results],
|
|
869
|
+
["Why", a.why]
|
|
870
|
+
]) {
|
|
871
|
+
if (!items.length) continue;
|
|
872
|
+
md.push(`### ${title}`, "");
|
|
873
|
+
for (const c of items) md.push(`- ${c.text}${cite(c.cites)}${note(c) ? ` *(${note(c)})*` : ""}`);
|
|
874
|
+
md.push("");
|
|
875
|
+
}
|
|
876
|
+
if (a.anomalies.length) {
|
|
877
|
+
md.push("### Anomalies", "");
|
|
878
|
+
for (const x of a.anomalies) md.push(`- **${x.kind}**: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` *(${unverified(x.unverifiedNumbers)})*` : ""}`);
|
|
879
|
+
md.push("");
|
|
880
|
+
}
|
|
881
|
+
md.push("### Suggested actions", "");
|
|
882
|
+
if (!a.actions.length) md.push("None: nothing here is worth acting on.");
|
|
883
|
+
a.actions.forEach((x, i) => {
|
|
884
|
+
md.push(`${i + 1}. **${x.title}** (${x.kind} · impact ${x.impact} · effort ${x.effort}) `, ` Target: \`${x.target}\`. ${x.change} `, ` Expect: ${x.expectedEffect} `, x.verify ? ` Verify: \`${x.verify}\`${cite(x.cites)}` : ` Verify: none (the suggested command isn't reeval's CLI: ${x.verifyProblem ?? "unknown"})${cite(x.cites)}`);
|
|
885
|
+
});
|
|
886
|
+
md.push("", `**Confidence: ${a.confidence.overall}.** ${a.confidence.why}`, "", `<sub>${footer(doc)}</sub>`, "");
|
|
887
|
+
return md.join("\n");
|
|
888
|
+
}
|
|
889
|
+
/** The analysis for a terminal: the same sections, plain. */
|
|
890
|
+
function renderText(doc) {
|
|
891
|
+
const a = doc.analysis;
|
|
892
|
+
const lines = [
|
|
893
|
+
`analysis ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
|
|
894
|
+
"",
|
|
895
|
+
`TL;DR (${a.confidence.overall}) ${a.tldr}`
|
|
896
|
+
];
|
|
897
|
+
for (const [title, items] of [
|
|
898
|
+
["What happened", a.whatHappened],
|
|
899
|
+
["Results", a.results],
|
|
900
|
+
["Why", a.why]
|
|
901
|
+
]) {
|
|
902
|
+
if (!items.length) continue;
|
|
903
|
+
lines.push("", title);
|
|
904
|
+
for (const c of items) lines.push(` - ${c.text}${cite(c.cites)}${note(c) ? ` (${note(c)})` : ""}`);
|
|
905
|
+
}
|
|
906
|
+
if (a.anomalies.length) {
|
|
907
|
+
lines.push("", "Anomalies");
|
|
908
|
+
for (const x of a.anomalies) lines.push(` - ${x.kind}: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` (${unverified(x.unverifiedNumbers)})` : ""}`);
|
|
909
|
+
}
|
|
910
|
+
lines.push("", "Suggested actions");
|
|
911
|
+
if (!a.actions.length) lines.push(" none: nothing here is worth acting on");
|
|
912
|
+
a.actions.forEach((x, i) => {
|
|
913
|
+
lines.push(` ${i + 1}. ${x.title} (${x.kind} · impact ${x.impact} · effort ${x.effort})${cite(x.cites)}`, ` target: ${x.target}. ${x.change}`, ` expect: ${x.expectedEffect}`, x.verify ? ` verify: ${x.verify}` : ` verify: none (${x.verifyProblem ?? "rejected"})`);
|
|
914
|
+
});
|
|
915
|
+
lines.push("", `Confidence: ${a.confidence.overall}. ${a.confidence.why}`, "", footer(doc));
|
|
916
|
+
return lines.join("\n");
|
|
917
|
+
}
|
|
918
|
+
//#endregion
|
|
799
919
|
//#region src/analysis/validate.ts
|
|
800
920
|
const EXEC = /^(.+):exec(\d+)$/;
|
|
801
921
|
const LINE = /^(E\d+):L(\d+)$/;
|
|
@@ -1029,6 +1149,246 @@ function validateAnalysis(output, digest, tables, evals) {
|
|
|
1029
1149
|
};
|
|
1030
1150
|
}
|
|
1031
1151
|
//#endregion
|
|
1032
|
-
|
|
1152
|
+
//#region src/analysis/command.ts
|
|
1153
|
+
async function load(config, entry) {
|
|
1154
|
+
const dir = entry.recorded ? config.history.dir : scratchDir(config.root);
|
|
1155
|
+
const bytes = await readFile(join(dir, `${entry.runId}.json`));
|
|
1156
|
+
return {
|
|
1157
|
+
record: parseRecord(bytes, entry.runId),
|
|
1158
|
+
dir,
|
|
1159
|
+
bytes
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
/** The model's input: the project's own instructions, which set the question, then the digest as JSON. */
|
|
1163
|
+
const promptOf = (d) => `${d.instructions ? `PROJECT INSTRUCTIONS (the question this analysis answers): ${d.instructions}\n\n` : ""}DIGEST:\n${JSON.stringify(d)}`;
|
|
1164
|
+
/** `--context`: the same digest and rules, for the user's own coding agent. */
|
|
1165
|
+
function contextMarkdown(d, head, base) {
|
|
1166
|
+
return [
|
|
1167
|
+
`# reeval analysis context: ${head}${base ? ` vs ${base}` : ""}`,
|
|
1168
|
+
"",
|
|
1169
|
+
"Analyze this reeval run for the engineer who wrote the evals. The digest below holds facts reeval computed (F…), evidence excerpts (E…, transcript lines as L<n>) and the attempts table. Cite those ids for every claim, keep to the verdicts in the facts, and suggest at most 5 concrete actions, each verified with a reeval command from the CLI list.",
|
|
1170
|
+
"",
|
|
1171
|
+
...d.frame === "study" ? ["This run is a study: every cell is ungated (gate: 'none'), so it measures how the arms differ. Failing attempts are data, not a failed gate.", ""] : [],
|
|
1172
|
+
...d.instructions ? [
|
|
1173
|
+
"## Project instructions (the question this analysis answers)",
|
|
1174
|
+
"",
|
|
1175
|
+
d.instructions,
|
|
1176
|
+
""
|
|
1177
|
+
] : [],
|
|
1178
|
+
"## How reeval decided each verdict",
|
|
1179
|
+
"",
|
|
1180
|
+
...RULES.map((r) => `- ${r}`),
|
|
1181
|
+
"",
|
|
1182
|
+
"## Instructions reeval gives its own analysis model",
|
|
1183
|
+
"",
|
|
1184
|
+
SYSTEM_PROMPT,
|
|
1185
|
+
"",
|
|
1186
|
+
"## reeval CLI (verify commands must be one of these)",
|
|
1187
|
+
"",
|
|
1188
|
+
...commandSummary().map((c) => `- \`${c}\``),
|
|
1189
|
+
"",
|
|
1190
|
+
"## To dig further",
|
|
1191
|
+
"",
|
|
1192
|
+
"- `reeval query effort`, `reeval query factor_effects`, `reeval query skill_reads`, `reeval query flaky_cells`, `reeval query drift`",
|
|
1193
|
+
`- \`reeval query "SELECT line, tool, target FROM tool_calls WHERE attempt_key = '<key>'"\`: an attempt's tool calls`,
|
|
1194
|
+
"- `reeval show <run> --json`: the whole record; transcripts are under the run's artifact folder",
|
|
1195
|
+
"",
|
|
1196
|
+
"## Digest",
|
|
1197
|
+
"",
|
|
1198
|
+
...fenced([JSON.stringify(d, null, 1)], "json"),
|
|
1199
|
+
""
|
|
1200
|
+
].join("\n");
|
|
1201
|
+
}
|
|
1202
|
+
function citedSummary(d, cites) {
|
|
1203
|
+
const out = {};
|
|
1204
|
+
for (const c of cites) {
|
|
1205
|
+
const fact = d.facts.find((f) => f.id === c);
|
|
1206
|
+
const evidence = d.evidence.find((e) => e.id === c || c.startsWith(`${e.id}:`));
|
|
1207
|
+
const attempt = d.attempts[c.replace(/:exec\d+$/, "")];
|
|
1208
|
+
if (fact) {
|
|
1209
|
+
const { id: _id, ...rest } = fact;
|
|
1210
|
+
out[c] = clip(JSON.stringify(rest), 400);
|
|
1211
|
+
} else if (evidence) {
|
|
1212
|
+
const line = /:L(\d+)$/.exec(c);
|
|
1213
|
+
out[c] = line ? clip(evidence.lines.find((l) => l.startsWith(`L${line[1]} `)) ?? evidence.ref, 400) : `${evidence.ref}: ${evidence.why}`;
|
|
1214
|
+
} else if (attempt) out[c] = `${attempt.runId} · ${attempt.attempt} · ${attempt.outcome} · ${attempt.tools} tool calls${attempt.checks ? ` · checks ${attempt.checks}` : ""}`;
|
|
1215
|
+
}
|
|
1216
|
+
return out;
|
|
1217
|
+
}
|
|
1218
|
+
/** The run `reeval analyze` compares `head` with when `--vs` is omitted: its nearest recorded ancestor. */
|
|
1219
|
+
async function defaultBase(config, entries, head) {
|
|
1220
|
+
const reach = await reachability(config.root, [head.git?.sha]);
|
|
1221
|
+
const from = head.git && (!reach || reach.has(head.git.sha)) ? head.git.sha : "HEAD";
|
|
1222
|
+
const ancestry = await lines(config.root, [
|
|
1223
|
+
"rev-list",
|
|
1224
|
+
"--max-count=5000",
|
|
1225
|
+
from
|
|
1226
|
+
]);
|
|
1227
|
+
return pickBaseline(entries.filter((e) => e.recorded && e.runId !== head.runId), head, ancestry);
|
|
1228
|
+
}
|
|
1229
|
+
/** Reads the runs and builds the digest and its cache key; no model call. */
|
|
1230
|
+
async function prepareAnalysis(config, head, base, evalFilter) {
|
|
1231
|
+
const headRun = await load(config, head);
|
|
1232
|
+
const baseRun = base ? await load(config, base) : null;
|
|
1233
|
+
const db = await openHistoryDb(config.root, config.history.dir);
|
|
1234
|
+
let built;
|
|
1235
|
+
let tables;
|
|
1236
|
+
try {
|
|
1237
|
+
built = await buildDigestKeepingDropped({
|
|
1238
|
+
head: headRun,
|
|
1239
|
+
base: baseRun,
|
|
1240
|
+
evalFilter,
|
|
1241
|
+
db,
|
|
1242
|
+
maxChars: config.analysis.maxInputTokens * 4,
|
|
1243
|
+
...config.analysis.instructions ? { instructions: config.analysis.instructions } : {}
|
|
1244
|
+
});
|
|
1245
|
+
tables = db.schema().map((o) => o.name);
|
|
1246
|
+
} finally {
|
|
1247
|
+
db.close();
|
|
1248
|
+
}
|
|
1249
|
+
const modelName = (() => {
|
|
1250
|
+
try {
|
|
1251
|
+
const m = analysisModelName(config);
|
|
1252
|
+
return typeof m === "string" ? m : "fake";
|
|
1253
|
+
} catch {
|
|
1254
|
+
return "none";
|
|
1255
|
+
}
|
|
1256
|
+
})();
|
|
1257
|
+
const scope = {
|
|
1258
|
+
kind: "run",
|
|
1259
|
+
head: head.runId,
|
|
1260
|
+
base: base?.runId ?? null,
|
|
1261
|
+
evals: evalFilter
|
|
1262
|
+
};
|
|
1263
|
+
const key = sha256(JSON.stringify({
|
|
1264
|
+
scope,
|
|
1265
|
+
hash: digestHash(built.digest),
|
|
1266
|
+
prompt: ANALYSIS_PROMPT_VERSION,
|
|
1267
|
+
model: modelName
|
|
1268
|
+
})).replace(/^sha256:/, "");
|
|
1269
|
+
return {
|
|
1270
|
+
head,
|
|
1271
|
+
base,
|
|
1272
|
+
headRun,
|
|
1273
|
+
baseRun,
|
|
1274
|
+
...built,
|
|
1275
|
+
tables,
|
|
1276
|
+
scope,
|
|
1277
|
+
modelName,
|
|
1278
|
+
key
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
const docBase = async (config, p) => ({
|
|
1282
|
+
kind: "reeval-analysis",
|
|
1283
|
+
schemaVersion: 1,
|
|
1284
|
+
scope: p.scope,
|
|
1285
|
+
anchor: p.head.runId,
|
|
1286
|
+
inputs: {
|
|
1287
|
+
records: [p.baseRun, p.headRun].flatMap((r) => r ? [{
|
|
1288
|
+
runId: r.record.runId,
|
|
1289
|
+
sha256: sha256(r.bytes)
|
|
1290
|
+
}] : []),
|
|
1291
|
+
digestVersion: p.digest.digestVersion,
|
|
1292
|
+
digestHash: digestHash(p.digest)
|
|
1293
|
+
},
|
|
1294
|
+
promptVersion: ANALYSIS_PROMPT_VERSION,
|
|
1295
|
+
author: await author(config.root)
|
|
1296
|
+
});
|
|
1297
|
+
/** One structured call on the prepared digest (and what tools read), validated: the document the cache holds. */
|
|
1298
|
+
async function analyzeDigest(config, p, model, reads) {
|
|
1299
|
+
const seen = reads?.evidence.length ? {
|
|
1300
|
+
...p.digest,
|
|
1301
|
+
evidence: [...p.digest.evidence, ...reads.evidence]
|
|
1302
|
+
} : p.digest;
|
|
1303
|
+
const answer = await callAnalysisModel(model, seen, `${promptOf(p.digest)}${reads?.text ?? ""}`, config.analysis.maxOutputTokens);
|
|
1304
|
+
const evals = p.headRun.record.evals.map((e) => ({
|
|
1305
|
+
file: e.file,
|
|
1306
|
+
name: [...e.group, e.name].join(" "),
|
|
1307
|
+
id: e.id,
|
|
1308
|
+
tags: e.tags,
|
|
1309
|
+
cells: e.cells.map((c) => c.key)
|
|
1310
|
+
}));
|
|
1311
|
+
const { body, validation } = validateAnalysis(answer.output, seen, p.tables, evals);
|
|
1312
|
+
const cites = [
|
|
1313
|
+
...body.whatHappened,
|
|
1314
|
+
...body.results,
|
|
1315
|
+
...body.why,
|
|
1316
|
+
...body.anomalies,
|
|
1317
|
+
...body.actions
|
|
1318
|
+
].flatMap((x) => x.cites);
|
|
1319
|
+
return {
|
|
1320
|
+
...await docBase(config, p),
|
|
1321
|
+
model: model.name,
|
|
1322
|
+
resolvedModel: answer.resolvedModel,
|
|
1323
|
+
route: model.route,
|
|
1324
|
+
usage: answer.usage,
|
|
1325
|
+
latencyMs: answer.latencyMs,
|
|
1326
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1327
|
+
validation,
|
|
1328
|
+
analysis: body,
|
|
1329
|
+
attempts: Object.fromEntries(Object.entries(p.digest.attempts).map(([id, a]) => [id, {
|
|
1330
|
+
runId: a.runId,
|
|
1331
|
+
evalSlug: a.evalSlug,
|
|
1332
|
+
attempt: a.attempt
|
|
1333
|
+
}])),
|
|
1334
|
+
cited: citedSummary(seen, [...new Set(cites)])
|
|
1335
|
+
};
|
|
1336
|
+
}
|
|
1337
|
+
async function analyzeCommand(ctx, args) {
|
|
1338
|
+
const { config, flags } = ctx;
|
|
1339
|
+
if (flags.json && flags.md) throw new UsageError("reeval analyze takes --json or --md, not both");
|
|
1340
|
+
if (flags.context && flags.save) throw new UsageError("reeval analyze --context makes no analysis to save; drop --save");
|
|
1341
|
+
const entries = await allRuns(config);
|
|
1342
|
+
let evalFilter = flags.eval ?? null;
|
|
1343
|
+
let head;
|
|
1344
|
+
try {
|
|
1345
|
+
head = resolveRun(entries, args[0] ?? "latest");
|
|
1346
|
+
} catch (error) {
|
|
1347
|
+
const needle = args[0]?.toLowerCase();
|
|
1348
|
+
const latest = entries.at(-1);
|
|
1349
|
+
if (!(error instanceof RefError) || !needle || !latest?.evals.some((e) => e.id.toLowerCase().includes(needle))) throw error;
|
|
1350
|
+
evalFilter ??= args[0];
|
|
1351
|
+
head = latest;
|
|
1352
|
+
}
|
|
1353
|
+
const base = flags.vs === "none" ? null : flags.vs ? resolveRun(entries, flags.vs) : await defaultBase(config, entries, head);
|
|
1354
|
+
if (base?.runId === head.runId) throw new UsageError("reeval analyze --vs names the run itself");
|
|
1355
|
+
if (flags.save && !head.recorded) throw new CommandError(`${head.runId} isn't recorded, so there's no history folder to save into. Run reeval record ${head.runId} first, then reeval analyze ${head.runId} --save.`);
|
|
1356
|
+
const model = flags.context || flags["dry-run"] ? null : resolveAnalysisModel(config);
|
|
1357
|
+
const prepared = await prepareAnalysis(config, head, base, evalFilter);
|
|
1358
|
+
const { digest, key, modelName } = prepared;
|
|
1359
|
+
if (evalFilter && digest.setup.evals.length === 0) throw new UsageError(`No eval in ${head.runId} matches "${evalFilter}"`);
|
|
1360
|
+
if (flags.context) {
|
|
1361
|
+
ctx.out(contextMarkdown(digest, head.runId, base?.runId ?? null));
|
|
1362
|
+
return 0;
|
|
1363
|
+
}
|
|
1364
|
+
const cached = flags["no-cache"] ? null : await readCached(config.root, key);
|
|
1365
|
+
if (flags["dry-run"]) {
|
|
1366
|
+
const chars = digestChars(digest);
|
|
1367
|
+
ctx.out([
|
|
1368
|
+
`digest ${head.runId}${base ? ` vs ${base.runId}` : ""}: ${digest.facts.length} facts, ${digest.evidence.length} evidence items${digest.dropped.length ? ` (${digest.dropped.length} dropped to fit)` : ""}, ${Object.keys(digest.attempts).length} attempts`,
|
|
1369
|
+
`size ${chars} characters ≈ ${Math.ceil(chars / 4)} tokens (cap ${config.analysis.maxInputTokens} in, ${config.analysis.maxOutputTokens} out)`,
|
|
1370
|
+
`model ${modelName} · ${ANALYSIS_PROMPT_VERSION} · cache ${cached ? "hit" : "miss"} (.reeval/analysis/${key}.json)`
|
|
1371
|
+
].join("\n"));
|
|
1372
|
+
return 0;
|
|
1373
|
+
}
|
|
1374
|
+
let doc;
|
|
1375
|
+
if (cached) {
|
|
1376
|
+
doc = cached;
|
|
1377
|
+
ctx.err(`cached analysis from ${cached.createdAt} (.reeval/analysis/${key}.json); --no-cache calls the model again`);
|
|
1378
|
+
} else {
|
|
1379
|
+
const resolved = model ?? resolveAnalysisModel(config);
|
|
1380
|
+
ctx.err(`analyzing ${head.runId}${base ? ` vs ${base.runId}` : ""} with ${resolved.name}…`);
|
|
1381
|
+
doc = await analyzeDigest(config, prepared, resolved);
|
|
1382
|
+
await writeCached(config.root, key, doc);
|
|
1383
|
+
}
|
|
1384
|
+
if (flags.save) {
|
|
1385
|
+
const saved = await saveAnalysis(config.history.dir, doc, key, renderMarkdown(doc));
|
|
1386
|
+
ctx.err(`saved ${saved.map((p) => fromCwd(p)).join(" and ")}. Commit them with git.`);
|
|
1387
|
+
}
|
|
1388
|
+
ctx.out(flags.json ? JSON.stringify(doc, null, 2) : flags.md ? renderMarkdown(doc) : renderText(doc));
|
|
1389
|
+
return 0;
|
|
1390
|
+
}
|
|
1391
|
+
//#endregion
|
|
1392
|
+
export { analyzeCommand };
|
|
1033
1393
|
|
|
1034
|
-
//# sourceMappingURL=
|
|
1394
|
+
//# sourceMappingURL=command-9xm5ImqI.js.map
|