reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,16 +1,23 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
1
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
+
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
3
|
+
import { lt as UNRECORDED_QUESTION_STATE, ut as questionShows } from "./config-CneBZB84.js";
|
|
4
|
+
import { n as clip, r as fenced } from "./text-DTigcA64.js";
|
|
5
|
+
import { d as pairTallies, h as sampleReasons, l as headToHeadLines, v as splitsLabel } from "./gate-BC6SdQZs.js";
|
|
6
|
+
import "./stats-hfpuYsAM.js";
|
|
7
|
+
import { l as resolveAnalysisModel, n as callAnalysisModel, t as analysisModelName, u as SYSTEM_PROMPT } from "./model-2rBZrdZA.js";
|
|
5
8
|
import { n as formatTokens } from "./usage-BNY1ABrU.js";
|
|
6
|
-
import "./analysis-C2-aEbr7.js";
|
|
7
|
-
import {
|
|
8
|
-
import "./
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
9
|
+
import { t as ANALYSIS_PROMPT_VERSION } from "./analysis-C2-aEbr7.js";
|
|
10
|
+
import { Q as bothUnderATenth, U as labeledJudgments, gt as compare } from "./model-Bz7-j71_.js";
|
|
11
|
+
import { g as familyRateLabel, h as familyLiftLabel, p as familyEffortLabel } from "./suite-DieLXvCT.js";
|
|
12
|
+
import { r as runHeader } from "./remarks-IFFe_oty.js";
|
|
13
|
+
import { i as checkCommand, o as commandSummary } from "./cli-spec-CPWT1mAG.js";
|
|
14
|
+
import { o as parseRecord, p as scratchDir } from "./files-BRViE7m-.js";
|
|
15
|
+
import { a as pickBaseline, o as resolveRun, t as RefError } from "./views-B9STsLB4.js";
|
|
16
|
+
import { c as lines, n as allRuns, t as CommandError, u as reachability } from "./commands-BvTNZ2-b.js";
|
|
17
|
+
import { r as contrastsOf } from "./contrasts-lhFsekco.js";
|
|
18
|
+
import { l as TOOL_CLASSES, s as openHistoryDb } from "./db-Cnh3jIv_.js";
|
|
19
|
+
import { t as author } from "./provenance-7Hzg88f4.js";
|
|
20
|
+
import { a as readCached, c as writeCached, o as saveAnalysis } from "./files-BSqA2sZE.js";
|
|
14
21
|
import { basename, join } from "node:path";
|
|
15
22
|
import { readFile } from "node:fs/promises";
|
|
16
23
|
//#region src/analysis/digest.ts
|
|
@@ -20,8 +27,9 @@ const RULES = [
|
|
|
20
27
|
`Effort (tool calls, agent time, and tokens when both sides have usage samples) is compared with an exact Mann-Whitney test: "fewer"/"more" at p < 0.05; "insufficient" when no split of the attempts' values can reach it (2 vs 2 cannot, and ties raise the floor) — never because a harness dropped usage. Attempts without usage are left out of the token sample only. If either side then has no token sample, the token metric is omitted (no line, not insufficient). A Goose cell does not blank tokens on Cursor cells in the same study. Pooled across evals, tokens are an effect only when at least 2/3 of the strata that reported usage agree. Effort's separated (given from 3 vs 3) says, per metric, that the two sides' ranges do not overlap: every attempt on one side beat every attempt on the other. insufficient with separated true is a directional signal: report the ratio, the n and "no overlap, not significant" as directional, never as an effect. separated false is no direction at all.`,
|
|
21
28
|
"A family fact is one pooled evaluate.each: its cases are paired samples of one suite, so its lift (Newcombe paired interval and McNemar at one attempt a case, a t interval over cases otherwise) is the result for those cases; their own pass-rate facts carry no lift. insufficient (fewer than 6 discordant pairs or 6 cases) is not measured. A family's minLift gates the pooled lower bound.",
|
|
22
29
|
"Each factor claim has a tier: controlled (two cells in one run that differ in one factor), pooled (the same contrast over comparable runs, and across evals), observed (a change between runs, which is weaker; \"confounded\" when several parts changed), not_measured (one level only).",
|
|
23
|
-
`A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "
|
|
30
|
+
`A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "mixed": it passed and failed, too few times to call it flaky.`,
|
|
24
31
|
"Drift marked noise changed only an agent CLI's own runtime state (e.g. ~/.cursor/cli-config.json): not a real change, and those runs stay comparable.",
|
|
32
|
+
"Drift on the gate is a policy change: the gate scores the same attempts, so the runs stay comparable. A compare row with status gate flipped only because its gate did (gate.rescored, base's attempts under head's gate, matches head): never call it a regression or a fix.",
|
|
25
33
|
"An eval where every cell passes at least 95% is saturated: pass@1 cannot tell cells apart there, effort can.",
|
|
26
34
|
"Judge verdicts carry no recorded reason; an explanation of a failed verdict is your reading of the cited output, and must say so.",
|
|
27
35
|
"A criteria fact lists each check (expect label, judges included) per cell: rows only for labels whose rate differs between cells, or is mixed within one; neverMet labels failed on every counted attempt of every cell, alwaysMet passed on all of them. A neverMet or alwaysMet label is the same in every arm: it can't explain a difference between arms.",
|
|
@@ -30,7 +38,7 @@ const RULES = [
|
|
|
30
38
|
"A context-use fact counts each attempt's tool calls by class (context, dependency, web, verify, workspace) and lists which placed files each arm opened: how much an arm leaned on what its environment declared versus digging in installed packages. The classes are a heuristic over tool names and paths; cite them as counts, not as proof of intent.",
|
|
31
39
|
"A contrast fact (kind contrast) is one pair of arms on one harness, a against b, when an eval has two variants or more: pass rate and lift of a over b, the checks that tell them apart (discriminates or mixed; sameOnBoth counts the rest) and effort. It answers how two variants differ from each other, not only from the control; gating stays each variant against its control.",
|
|
32
40
|
"A check (kind checks) whose verdict is both-pass or both-fail passes, or fails, on both the variant and its control: it is no evidence for or against the variant. Only discriminates rows say which requirement separated them.",
|
|
33
|
-
"Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can
|
|
41
|
+
"Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favor the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
|
|
34
42
|
];
|
|
35
43
|
const pts = (x) => x === null || x === void 0 ? null : `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
|
|
36
44
|
const liftText = (points, lo, hi) => `${pts(points)} pts (95% ${pts(lo)}…${pts(hi)})`;
|
|
@@ -46,10 +54,12 @@ function digestLiftVerdict(variantCounted, controlCounted, withinNoise) {
|
|
|
46
54
|
function evalChangedLift() {
|
|
47
55
|
return "not comparable: the eval changed";
|
|
48
56
|
}
|
|
57
|
+
/** What the eval's own source is keyed on: `source` and `revision`, else (older records) its files. */
|
|
49
58
|
function sourcesKey(e) {
|
|
59
|
+
if (e.source) return JSON.stringify([e.source, e.revision ?? null]);
|
|
50
60
|
return JSON.stringify(e.sources.map((s) => [s.path, s.sha256]));
|
|
51
61
|
}
|
|
52
|
-
/** The eval's
|
|
62
|
+
/** The eval's own source, or the sources part of its fingerprint, differs across the two runs. */
|
|
53
63
|
function evalSourcesChanged(base, head) {
|
|
54
64
|
if (sourcesKey(base) !== sourcesKey(head)) return true;
|
|
55
65
|
const prev = new Map(base.cells.map((c) => [c.key, c.fingerprintParts.sources]));
|
|
@@ -129,10 +139,10 @@ function criteriaOf(e, tag) {
|
|
|
129
139
|
const scores = /* @__PURE__ */ new Map();
|
|
130
140
|
for (const a of e.attempts) {
|
|
131
141
|
if (!counted(a)) continue;
|
|
132
|
-
for (const j of
|
|
142
|
+
for (const j of labeledJudgments(a)) {
|
|
133
143
|
if (j.check === void 0) continue;
|
|
134
144
|
const id = `${a.cell}\0${j.check}`;
|
|
135
|
-
scores.set(id, [...scores.get(id) ?? [], j.
|
|
145
|
+
scores.set(id, [...scores.get(id) ?? [], j.judgment.score]);
|
|
136
146
|
}
|
|
137
147
|
}
|
|
138
148
|
const rows = [];
|
|
@@ -191,7 +201,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
191
201
|
const any = a.files.transcript ?? a.files.judgeInput ?? a.checks.find((c) => c.log)?.log;
|
|
192
202
|
const assertions = a.assertions ?? [];
|
|
193
203
|
const failed = failedLabels(a);
|
|
194
|
-
const judges =
|
|
204
|
+
const judges = labeledJudgments(a);
|
|
195
205
|
attempts[id] = {
|
|
196
206
|
run: short.get(r.record.runId),
|
|
197
207
|
runId: r.record.runId,
|
|
@@ -205,7 +215,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
205
215
|
agentS: Math.round(a.timings.agentMs / 1e3),
|
|
206
216
|
...assertions.length ? { checks: `${assertions.filter((x) => x.status === "pass").length}/${assertions.length}` } : {},
|
|
207
217
|
...failed.length ? { failed } : {},
|
|
208
|
-
...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.
|
|
218
|
+
...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgment.score)])) } : {},
|
|
209
219
|
execs: a.checks.length,
|
|
210
220
|
artifacts: any ? `${r.record.runId}/${any.path.split("/").slice(0, 3).join("/")}` : null
|
|
211
221
|
};
|
|
@@ -280,36 +290,39 @@ async function buildDigestKeepingDropped(input) {
|
|
|
280
290
|
fact("effort", {
|
|
281
291
|
run: short.get(runId),
|
|
282
292
|
eval: tagOf(first.eval_key),
|
|
283
|
-
cells: rows.map((row) =>
|
|
284
|
-
|
|
285
|
-
n: row.n,
|
|
286
|
-
medianToolCalls: row.tools_median,
|
|
287
|
-
medianAgentS: row.agent_s_median,
|
|
288
|
-
...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
|
|
289
|
-
...row.workspace_calls_median != null ? { medianCallsByClass: Object.fromEntries(TOOL_CLASSES.flatMap((k) => {
|
|
293
|
+
cells: rows.map((row) => {
|
|
294
|
+
const byClass = TOOL_CLASSES.flatMap((k) => {
|
|
290
295
|
const m = row[`${k}_calls_median`];
|
|
291
296
|
return m ? [[k, m]] : [];
|
|
292
|
-
})
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
...row.
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
297
|
+
});
|
|
298
|
+
return {
|
|
299
|
+
cell: cellTag(runId, row.eval_key, row.cell_key),
|
|
300
|
+
n: row.n,
|
|
301
|
+
medianToolCalls: row.tools_median,
|
|
302
|
+
medianAgentS: row.agent_s_median,
|
|
303
|
+
...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
|
|
304
|
+
...byClass.length ? { medianCallsByClass: Object.fromEntries(byClass) } : {},
|
|
305
|
+
...row.control_key ? { vsControl: {
|
|
306
|
+
n: `${row.n} vs ${row.control_n}`,
|
|
307
|
+
toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
|
|
308
|
+
agentS: bothUnderATenth(Number(row.agent_s_median) * 1e3, Number(row.control_agent_s_median) * 1e3) ? "<0.1 vs <0.1" : `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
|
|
309
|
+
p: row.tools_p,
|
|
310
|
+
verdict: row.verdict,
|
|
311
|
+
timeVerdict: row.time_verdict,
|
|
312
|
+
...row.tokens_verdict ? {
|
|
313
|
+
tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
|
|
314
|
+
tokensP: row.tokens_p,
|
|
315
|
+
tokensVerdict: row.tokens_verdict
|
|
316
|
+
} : {},
|
|
317
|
+
...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
|
|
318
|
+
tools: row.separated === 1,
|
|
319
|
+
agentS: row.time_separated === 1,
|
|
320
|
+
...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
|
|
321
|
+
} } : {},
|
|
322
|
+
...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
|
|
323
|
+
} } : {}
|
|
324
|
+
};
|
|
325
|
+
})
|
|
313
326
|
});
|
|
314
327
|
}
|
|
315
328
|
const changedEvalKeys = /* @__PURE__ */ new Set();
|
|
@@ -508,7 +521,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
508
521
|
}) : failures.slice(0, 8);
|
|
509
522
|
for (const { r, e, a } of failureFacts) {
|
|
510
523
|
const failed = failedLabels(a);
|
|
511
|
-
const judges =
|
|
524
|
+
const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
|
|
512
525
|
fact("failure", {
|
|
513
526
|
attempt: idOf.get(`${r.record.runId}\0${a.key}`),
|
|
514
527
|
outcome: a.outcome,
|
|
@@ -518,9 +531,9 @@ async function buildDigestKeepingDropped(input) {
|
|
|
518
531
|
...failed.length ? { failed } : {},
|
|
519
532
|
...judges.length ? { judges: judges.map((j) => ({
|
|
520
533
|
label: clip(j.label, LABEL_CHARS),
|
|
521
|
-
question: clip(j.
|
|
522
|
-
score: j.
|
|
523
|
-
threshold: j.
|
|
534
|
+
question: clip(j.judgment.question, 300),
|
|
535
|
+
score: j.judgment.score,
|
|
536
|
+
threshold: j.judgment.threshold
|
|
524
537
|
})) } : {},
|
|
525
538
|
failingExecs: a.checks.map((c, i) => ({
|
|
526
539
|
exec: i + 1,
|
|
@@ -561,16 +574,20 @@ async function buildDigestKeepingDropped(input) {
|
|
|
561
574
|
cells: d.cells,
|
|
562
575
|
changed: d.changed,
|
|
563
576
|
...d.changed_paths ? { paths: String(d.changed_paths).split(", ") } : {},
|
|
564
|
-
verdict: d.noise ? "noise: agent runtime state only" : `real change: ${d.real_changed}`
|
|
577
|
+
verdict: d.noise ? "noise: agent runtime state only" : d.real_changed === "gate" ? "policy change: the gate scores the same attempts differently" : `real change: ${d.real_changed}`
|
|
565
578
|
});
|
|
579
|
+
const [baseHeader, headHeader] = [runHeader(base.record, true), runHeader(head.record, true)];
|
|
580
|
+
const inScope = (id) => scopeEvals.some((e) => e.id === id);
|
|
581
|
+
const familyCases = new Map([...baseHeader.families ?? [], ...headHeader.families ?? []].map((f) => [f.id, f.caseIds]));
|
|
566
582
|
fact("compare", {
|
|
567
583
|
base: "r1",
|
|
568
584
|
head: "r2",
|
|
569
|
-
rows: compare(
|
|
570
|
-
eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id),
|
|
585
|
+
rows: compare(baseHeader, headHeader).filter((row) => row.kind === "family" ? (familyCases.get(row.id) ?? []).some(inScope) : inScope(row.id)).map((row) => ({
|
|
586
|
+
...row.kind === "family" ? { family: row.id } : { eval: tagOf(scopeEvals.find((e) => e.id === row.id)?.key ?? row.id) },
|
|
571
587
|
status: row.status,
|
|
572
588
|
...row.changes?.length ? { changes: row.changes } : {},
|
|
573
|
-
...row.
|
|
589
|
+
...row.gate ? { gate: row.gate } : {},
|
|
590
|
+
...row.canceled ? { canceled: row.canceled } : {}
|
|
574
591
|
}))
|
|
575
592
|
});
|
|
576
593
|
}
|
|
@@ -586,7 +603,7 @@ async function buildDigestKeepingDropped(input) {
|
|
|
586
603
|
eval: tagOf(e.key),
|
|
587
604
|
question: clip(a.question, 120),
|
|
588
605
|
model: a.resolvedModel ?? a.model,
|
|
589
|
-
shows: questionShows(a.stateOptions ??
|
|
606
|
+
shows: questionShows(a.stateOptions ?? UNRECORDED_QUESTION_STATE, a.artifacts),
|
|
590
607
|
votes: a.votes.length,
|
|
591
608
|
ties,
|
|
592
609
|
...a.rates.length ? { rates: a.rates.map((x) => ({
|
|
@@ -732,8 +749,8 @@ async function buildDigestKeepingDropped(input) {
|
|
|
732
749
|
for (const [i, { r, e, a }] of [...perCell.values()].slice(0, 6).entries()) {
|
|
733
750
|
const id = idOf.get(`${r.record.runId}\0${a.key}`);
|
|
734
751
|
const lines = await lastText(r.record.runId, a);
|
|
735
|
-
const judges =
|
|
736
|
-
for (const { label,
|
|
752
|
+
const judges = labeledJudgments(a).filter((j) => j.judgment.verdict === "fail");
|
|
753
|
+
for (const { label, judgment: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
|
|
737
754
|
for (const x of a.assertions ?? []) {
|
|
738
755
|
if (x.status === "pass" || /(^|\.)toPass$/.test(x.matcher)) continue;
|
|
739
756
|
lines.push(`check ${x.status}: ${clip(x.label, LABEL_CHARS)}: ${clip(x.message ?? "", 300)}`);
|
|
@@ -828,6 +845,77 @@ function citable(d) {
|
|
|
828
845
|
};
|
|
829
846
|
}
|
|
830
847
|
//#endregion
|
|
848
|
+
//#region src/analysis/render.ts
|
|
849
|
+
const cite = (cs) => cs.length ? ` ${cs.map((c) => `[${c}]`).join(" ")}` : "";
|
|
850
|
+
const unverified = (numbers) => numbers?.length ? `unverified: ${numbers.join(", ")}` : "";
|
|
851
|
+
const note = (c) => [c.confidence === "high" ? "" : `${c.confidence} confidence`, unverified(c.unverifiedNumbers)].filter(Boolean).join("; ");
|
|
852
|
+
function footer(doc) {
|
|
853
|
+
const v = doc.validation;
|
|
854
|
+
const tokens = doc.usage.inputTokens !== null ? `${doc.usage.inputTokens} in / ${doc.usage.outputTokens ?? "?"} out tokens · ` : "";
|
|
855
|
+
return `${doc.resolvedModel ?? doc.model} · ${doc.promptVersion} · digest v${doc.inputs.digestVersion} · ${tokens}${Math.round(doc.latencyMs / 1e3)}s · ${v.citations} citations checked, ${v.invalid.length} invalid, ${v.dropped} dropped · ${v.commands} commands checked, ${v.invalidCommands.length} rejected`;
|
|
856
|
+
}
|
|
857
|
+
/** The analysis as Markdown (`--md`, and the `.md` twin of a saved analysis). */
|
|
858
|
+
function renderMarkdown(doc) {
|
|
859
|
+
const a = doc.analysis;
|
|
860
|
+
const md = [
|
|
861
|
+
`## reeval analysis · ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
|
|
862
|
+
"",
|
|
863
|
+
`**TL;DR** (${a.confidence.overall}). ${a.tldr}`,
|
|
864
|
+
""
|
|
865
|
+
];
|
|
866
|
+
for (const [title, items] of [
|
|
867
|
+
["What happened", a.whatHappened],
|
|
868
|
+
["Results", a.results],
|
|
869
|
+
["Why", a.why]
|
|
870
|
+
]) {
|
|
871
|
+
if (!items.length) continue;
|
|
872
|
+
md.push(`### ${title}`, "");
|
|
873
|
+
for (const c of items) md.push(`- ${c.text}${cite(c.cites)}${note(c) ? ` *(${note(c)})*` : ""}`);
|
|
874
|
+
md.push("");
|
|
875
|
+
}
|
|
876
|
+
if (a.anomalies.length) {
|
|
877
|
+
md.push("### Anomalies", "");
|
|
878
|
+
for (const x of a.anomalies) md.push(`- **${x.kind}**: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` *(${unverified(x.unverifiedNumbers)})*` : ""}`);
|
|
879
|
+
md.push("");
|
|
880
|
+
}
|
|
881
|
+
md.push("### Suggested actions", "");
|
|
882
|
+
if (!a.actions.length) md.push("None: nothing here is worth acting on.");
|
|
883
|
+
a.actions.forEach((x, i) => {
|
|
884
|
+
md.push(`${i + 1}. **${x.title}** (${x.kind} · impact ${x.impact} · effort ${x.effort}) `, ` Target: \`${x.target}\`. ${x.change} `, ` Expect: ${x.expectedEffect} `, x.verify ? ` Verify: \`${x.verify}\`${cite(x.cites)}` : ` Verify: none (the suggested command isn't reeval's CLI: ${x.verifyProblem ?? "unknown"})${cite(x.cites)}`);
|
|
885
|
+
});
|
|
886
|
+
md.push("", `**Confidence: ${a.confidence.overall}.** ${a.confidence.why}`, "", `<sub>${footer(doc)}</sub>`, "");
|
|
887
|
+
return md.join("\n");
|
|
888
|
+
}
|
|
889
|
+
/** The analysis for a terminal: the same sections, plain. */
|
|
890
|
+
function renderText(doc) {
|
|
891
|
+
const a = doc.analysis;
|
|
892
|
+
const lines = [
|
|
893
|
+
`analysis ${doc.scope.head}${doc.scope.base ? ` vs ${doc.scope.base}` : ""}`,
|
|
894
|
+
"",
|
|
895
|
+
`TL;DR (${a.confidence.overall}) ${a.tldr}`
|
|
896
|
+
];
|
|
897
|
+
for (const [title, items] of [
|
|
898
|
+
["What happened", a.whatHappened],
|
|
899
|
+
["Results", a.results],
|
|
900
|
+
["Why", a.why]
|
|
901
|
+
]) {
|
|
902
|
+
if (!items.length) continue;
|
|
903
|
+
lines.push("", title);
|
|
904
|
+
for (const c of items) lines.push(` - ${c.text}${cite(c.cites)}${note(c) ? ` (${note(c)})` : ""}`);
|
|
905
|
+
}
|
|
906
|
+
if (a.anomalies.length) {
|
|
907
|
+
lines.push("", "Anomalies");
|
|
908
|
+
for (const x of a.anomalies) lines.push(` - ${x.kind}: ${x.text}${cite(x.cites)}${unverified(x.unverifiedNumbers) ? ` (${unverified(x.unverifiedNumbers)})` : ""}`);
|
|
909
|
+
}
|
|
910
|
+
lines.push("", "Suggested actions");
|
|
911
|
+
if (!a.actions.length) lines.push(" none: nothing here is worth acting on");
|
|
912
|
+
a.actions.forEach((x, i) => {
|
|
913
|
+
lines.push(` ${i + 1}. ${x.title} (${x.kind} · impact ${x.impact} · effort ${x.effort})${cite(x.cites)}`, ` target: ${x.target}. ${x.change}`, ` expect: ${x.expectedEffect}`, x.verify ? ` verify: ${x.verify}` : ` verify: none (${x.verifyProblem ?? "rejected"})`);
|
|
914
|
+
});
|
|
915
|
+
lines.push("", `Confidence: ${a.confidence.overall}. ${a.confidence.why}`, "", footer(doc));
|
|
916
|
+
return lines.join("\n");
|
|
917
|
+
}
|
|
918
|
+
//#endregion
|
|
831
919
|
//#region src/analysis/validate.ts
|
|
832
920
|
const EXEC = /^(.+):exec(\d+)$/;
|
|
833
921
|
const LINE = /^(E\d+):L(\d+)$/;
|
|
@@ -1061,6 +1149,246 @@ function validateAnalysis(output, digest, tables, evals) {
|
|
|
1061
1149
|
};
|
|
1062
1150
|
}
|
|
1063
1151
|
//#endregion
|
|
1064
|
-
|
|
1152
|
+
//#region src/analysis/command.ts
|
|
1153
|
+
async function load(config, entry) {
|
|
1154
|
+
const dir = entry.recorded ? config.history.dir : scratchDir(config.root);
|
|
1155
|
+
const bytes = await readFile(join(dir, `${entry.runId}.json`));
|
|
1156
|
+
return {
|
|
1157
|
+
record: parseRecord(bytes, entry.runId),
|
|
1158
|
+
dir,
|
|
1159
|
+
bytes
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
/** The model's input: the project's own instructions, which set the question, then the digest as JSON. */
|
|
1163
|
+
const promptOf = (d) => `${d.instructions ? `PROJECT INSTRUCTIONS (the question this analysis answers): ${d.instructions}\n\n` : ""}DIGEST:\n${JSON.stringify(d)}`;
|
|
1164
|
+
/** `--context`: the same digest and rules, for the user's own coding agent. */
|
|
1165
|
+
function contextMarkdown(d, head, base) {
|
|
1166
|
+
return [
|
|
1167
|
+
`# reeval analysis context: ${head}${base ? ` vs ${base}` : ""}`,
|
|
1168
|
+
"",
|
|
1169
|
+
"Analyze this reeval run for the engineer who wrote the evals. The digest below holds facts reeval computed (F…), evidence excerpts (E…, transcript lines as L<n>) and the attempts table. Cite those ids for every claim, keep to the verdicts in the facts, and suggest at most 5 concrete actions, each verified with a reeval command from the CLI list.",
|
|
1170
|
+
"",
|
|
1171
|
+
...d.frame === "study" ? ["This run is a study: every cell is ungated (gate: 'none'), so it measures how the arms differ. Failing attempts are data, not a failed gate.", ""] : [],
|
|
1172
|
+
...d.instructions ? [
|
|
1173
|
+
"## Project instructions (the question this analysis answers)",
|
|
1174
|
+
"",
|
|
1175
|
+
d.instructions,
|
|
1176
|
+
""
|
|
1177
|
+
] : [],
|
|
1178
|
+
"## How reeval decided each verdict",
|
|
1179
|
+
"",
|
|
1180
|
+
...RULES.map((r) => `- ${r}`),
|
|
1181
|
+
"",
|
|
1182
|
+
"## Instructions reeval gives its own analysis model",
|
|
1183
|
+
"",
|
|
1184
|
+
SYSTEM_PROMPT,
|
|
1185
|
+
"",
|
|
1186
|
+
"## reeval CLI (verify commands must be one of these)",
|
|
1187
|
+
"",
|
|
1188
|
+
...commandSummary().map((c) => `- \`${c}\``),
|
|
1189
|
+
"",
|
|
1190
|
+
"## To dig further",
|
|
1191
|
+
"",
|
|
1192
|
+
"- `reeval query effort`, `reeval query factor_effects`, `reeval query skill_reads`, `reeval query flaky_cells`, `reeval query drift`",
|
|
1193
|
+
`- \`reeval query "SELECT line, tool, target FROM tool_calls WHERE attempt_key = '<key>'"\`: an attempt's tool calls`,
|
|
1194
|
+
"- `reeval show <run> --json`: the whole record; transcripts are under the run's artifact folder",
|
|
1195
|
+
"",
|
|
1196
|
+
"## Digest",
|
|
1197
|
+
"",
|
|
1198
|
+
...fenced([JSON.stringify(d, null, 1)], "json"),
|
|
1199
|
+
""
|
|
1200
|
+
].join("\n");
|
|
1201
|
+
}
|
|
1202
|
+
function citedSummary(d, cites) {
|
|
1203
|
+
const out = {};
|
|
1204
|
+
for (const c of cites) {
|
|
1205
|
+
const fact = d.facts.find((f) => f.id === c);
|
|
1206
|
+
const evidence = d.evidence.find((e) => e.id === c || c.startsWith(`${e.id}:`));
|
|
1207
|
+
const attempt = d.attempts[c.replace(/:exec\d+$/, "")];
|
|
1208
|
+
if (fact) {
|
|
1209
|
+
const { id: _id, ...rest } = fact;
|
|
1210
|
+
out[c] = clip(JSON.stringify(rest), 400);
|
|
1211
|
+
} else if (evidence) {
|
|
1212
|
+
const line = /:L(\d+)$/.exec(c);
|
|
1213
|
+
out[c] = line ? clip(evidence.lines.find((l) => l.startsWith(`L${line[1]} `)) ?? evidence.ref, 400) : `${evidence.ref}: ${evidence.why}`;
|
|
1214
|
+
} else if (attempt) out[c] = `${attempt.runId} · ${attempt.attempt} · ${attempt.outcome} · ${attempt.tools} tool calls${attempt.checks ? ` · checks ${attempt.checks}` : ""}`;
|
|
1215
|
+
}
|
|
1216
|
+
return out;
|
|
1217
|
+
}
|
|
1218
|
+
/** The run `reeval analyze` compares `head` with when `--vs` is omitted: its nearest recorded ancestor. */
|
|
1219
|
+
async function defaultBase(config, entries, head) {
|
|
1220
|
+
const reach = await reachability(config.root, [head.git?.sha]);
|
|
1221
|
+
const from = head.git && (!reach || reach.has(head.git.sha)) ? head.git.sha : "HEAD";
|
|
1222
|
+
const ancestry = await lines(config.root, [
|
|
1223
|
+
"rev-list",
|
|
1224
|
+
"--max-count=5000",
|
|
1225
|
+
from
|
|
1226
|
+
]);
|
|
1227
|
+
return pickBaseline(entries.filter((e) => e.recorded && e.runId !== head.runId), head, ancestry);
|
|
1228
|
+
}
|
|
1229
|
+
/** Reads the runs and builds the digest and its cache key; no model call. */
|
|
1230
|
+
async function prepareAnalysis(config, head, base, evalFilter) {
|
|
1231
|
+
const headRun = await load(config, head);
|
|
1232
|
+
const baseRun = base ? await load(config, base) : null;
|
|
1233
|
+
const db = await openHistoryDb(config.root, config.history.dir);
|
|
1234
|
+
let built;
|
|
1235
|
+
let tables;
|
|
1236
|
+
try {
|
|
1237
|
+
built = await buildDigestKeepingDropped({
|
|
1238
|
+
head: headRun,
|
|
1239
|
+
base: baseRun,
|
|
1240
|
+
evalFilter,
|
|
1241
|
+
db,
|
|
1242
|
+
maxChars: config.analysis.maxInputTokens * 4,
|
|
1243
|
+
...config.analysis.instructions ? { instructions: config.analysis.instructions } : {}
|
|
1244
|
+
});
|
|
1245
|
+
tables = db.schema().map((o) => o.name);
|
|
1246
|
+
} finally {
|
|
1247
|
+
db.close();
|
|
1248
|
+
}
|
|
1249
|
+
const modelName = (() => {
|
|
1250
|
+
try {
|
|
1251
|
+
const m = analysisModelName(config);
|
|
1252
|
+
return typeof m === "string" ? m : "fake";
|
|
1253
|
+
} catch {
|
|
1254
|
+
return "none";
|
|
1255
|
+
}
|
|
1256
|
+
})();
|
|
1257
|
+
const scope = {
|
|
1258
|
+
kind: "run",
|
|
1259
|
+
head: head.runId,
|
|
1260
|
+
base: base?.runId ?? null,
|
|
1261
|
+
evals: evalFilter
|
|
1262
|
+
};
|
|
1263
|
+
const key = sha256(JSON.stringify({
|
|
1264
|
+
scope,
|
|
1265
|
+
hash: digestHash(built.digest),
|
|
1266
|
+
prompt: ANALYSIS_PROMPT_VERSION,
|
|
1267
|
+
model: modelName
|
|
1268
|
+
})).replace(/^sha256:/, "");
|
|
1269
|
+
return {
|
|
1270
|
+
head,
|
|
1271
|
+
base,
|
|
1272
|
+
headRun,
|
|
1273
|
+
baseRun,
|
|
1274
|
+
...built,
|
|
1275
|
+
tables,
|
|
1276
|
+
scope,
|
|
1277
|
+
modelName,
|
|
1278
|
+
key
|
|
1279
|
+
};
|
|
1280
|
+
}
|
|
1281
|
+
const docBase = async (config, p) => ({
|
|
1282
|
+
kind: "reeval-analysis",
|
|
1283
|
+
schemaVersion: 1,
|
|
1284
|
+
scope: p.scope,
|
|
1285
|
+
anchor: p.head.runId,
|
|
1286
|
+
inputs: {
|
|
1287
|
+
records: [p.baseRun, p.headRun].flatMap((r) => r ? [{
|
|
1288
|
+
runId: r.record.runId,
|
|
1289
|
+
sha256: sha256(r.bytes)
|
|
1290
|
+
}] : []),
|
|
1291
|
+
digestVersion: p.digest.digestVersion,
|
|
1292
|
+
digestHash: digestHash(p.digest)
|
|
1293
|
+
},
|
|
1294
|
+
promptVersion: ANALYSIS_PROMPT_VERSION,
|
|
1295
|
+
author: await author(config.root)
|
|
1296
|
+
});
|
|
1297
|
+
/** One structured call on the prepared digest (and what tools read), validated: the document the cache holds. */
|
|
1298
|
+
async function analyzeDigest(config, p, model, reads) {
|
|
1299
|
+
const seen = reads?.evidence.length ? {
|
|
1300
|
+
...p.digest,
|
|
1301
|
+
evidence: [...p.digest.evidence, ...reads.evidence]
|
|
1302
|
+
} : p.digest;
|
|
1303
|
+
const answer = await callAnalysisModel(model, seen, `${promptOf(p.digest)}${reads?.text ?? ""}`, config.analysis.maxOutputTokens);
|
|
1304
|
+
const evals = p.headRun.record.evals.map((e) => ({
|
|
1305
|
+
file: e.file,
|
|
1306
|
+
name: [...e.group, e.name].join(" "),
|
|
1307
|
+
id: e.id,
|
|
1308
|
+
tags: e.tags,
|
|
1309
|
+
cells: e.cells.map((c) => c.key)
|
|
1310
|
+
}));
|
|
1311
|
+
const { body, validation } = validateAnalysis(answer.output, seen, p.tables, evals);
|
|
1312
|
+
const cites = [
|
|
1313
|
+
...body.whatHappened,
|
|
1314
|
+
...body.results,
|
|
1315
|
+
...body.why,
|
|
1316
|
+
...body.anomalies,
|
|
1317
|
+
...body.actions
|
|
1318
|
+
].flatMap((x) => x.cites);
|
|
1319
|
+
return {
|
|
1320
|
+
...await docBase(config, p),
|
|
1321
|
+
model: model.name,
|
|
1322
|
+
resolvedModel: answer.resolvedModel,
|
|
1323
|
+
route: model.route,
|
|
1324
|
+
usage: answer.usage,
|
|
1325
|
+
latencyMs: answer.latencyMs,
|
|
1326
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1327
|
+
validation,
|
|
1328
|
+
analysis: body,
|
|
1329
|
+
attempts: Object.fromEntries(Object.entries(p.digest.attempts).map(([id, a]) => [id, {
|
|
1330
|
+
runId: a.runId,
|
|
1331
|
+
evalSlug: a.evalSlug,
|
|
1332
|
+
attempt: a.attempt
|
|
1333
|
+
}])),
|
|
1334
|
+
cited: citedSummary(seen, [...new Set(cites)])
|
|
1335
|
+
};
|
|
1336
|
+
}
|
|
1337
|
+
async function analyzeCommand(ctx, args) {
|
|
1338
|
+
const { config, flags } = ctx;
|
|
1339
|
+
if (flags.json && flags.md) throw new UsageError("reeval analyze takes --json or --md, not both");
|
|
1340
|
+
if (flags.context && flags.save) throw new UsageError("reeval analyze --context makes no analysis to save; drop --save");
|
|
1341
|
+
const entries = await allRuns(config);
|
|
1342
|
+
let evalFilter = flags.eval ?? null;
|
|
1343
|
+
let head;
|
|
1344
|
+
try {
|
|
1345
|
+
head = resolveRun(entries, args[0] ?? "latest");
|
|
1346
|
+
} catch (error) {
|
|
1347
|
+
const needle = args[0]?.toLowerCase();
|
|
1348
|
+
const latest = entries.at(-1);
|
|
1349
|
+
if (!(error instanceof RefError) || !needle || !latest?.evals.some((e) => e.id.toLowerCase().includes(needle))) throw error;
|
|
1350
|
+
evalFilter ??= args[0];
|
|
1351
|
+
head = latest;
|
|
1352
|
+
}
|
|
1353
|
+
const base = flags.vs === "none" ? null : flags.vs ? resolveRun(entries, flags.vs) : await defaultBase(config, entries, head);
|
|
1354
|
+
if (base?.runId === head.runId) throw new UsageError("reeval analyze --vs names the run itself");
|
|
1355
|
+
if (flags.save && !head.recorded) throw new CommandError(`${head.runId} isn't recorded, so there's no history folder to save into. Run reeval record ${head.runId} first, then reeval analyze ${head.runId} --save.`);
|
|
1356
|
+
const model = flags.context || flags["dry-run"] ? null : resolveAnalysisModel(config);
|
|
1357
|
+
const prepared = await prepareAnalysis(config, head, base, evalFilter);
|
|
1358
|
+
const { digest, key, modelName } = prepared;
|
|
1359
|
+
if (evalFilter && digest.setup.evals.length === 0) throw new UsageError(`No eval in ${head.runId} matches "${evalFilter}"`);
|
|
1360
|
+
if (flags.context) {
|
|
1361
|
+
ctx.out(contextMarkdown(digest, head.runId, base?.runId ?? null));
|
|
1362
|
+
return 0;
|
|
1363
|
+
}
|
|
1364
|
+
const cached = flags["no-cache"] ? null : await readCached(config.root, key);
|
|
1365
|
+
if (flags["dry-run"]) {
|
|
1366
|
+
const chars = digestChars(digest);
|
|
1367
|
+
ctx.out([
|
|
1368
|
+
`digest ${head.runId}${base ? ` vs ${base.runId}` : ""}: ${digest.facts.length} facts, ${digest.evidence.length} evidence items${digest.dropped.length ? ` (${digest.dropped.length} dropped to fit)` : ""}, ${Object.keys(digest.attempts).length} attempts`,
|
|
1369
|
+
`size ${chars} characters ≈ ${Math.ceil(chars / 4)} tokens (cap ${config.analysis.maxInputTokens} in, ${config.analysis.maxOutputTokens} out)`,
|
|
1370
|
+
`model ${modelName} · ${ANALYSIS_PROMPT_VERSION} · cache ${cached ? "hit" : "miss"} (.reeval/analysis/${key}.json)`
|
|
1371
|
+
].join("\n"));
|
|
1372
|
+
return 0;
|
|
1373
|
+
}
|
|
1374
|
+
let doc;
|
|
1375
|
+
if (cached) {
|
|
1376
|
+
doc = cached;
|
|
1377
|
+
ctx.err(`cached analysis from ${cached.createdAt} (.reeval/analysis/${key}.json); --no-cache calls the model again`);
|
|
1378
|
+
} else {
|
|
1379
|
+
const resolved = model ?? resolveAnalysisModel(config);
|
|
1380
|
+
ctx.err(`analyzing ${head.runId}${base ? ` vs ${base.runId}` : ""} with ${resolved.name}…`);
|
|
1381
|
+
doc = await analyzeDigest(config, prepared, resolved);
|
|
1382
|
+
await writeCached(config.root, key, doc);
|
|
1383
|
+
}
|
|
1384
|
+
if (flags.save) {
|
|
1385
|
+
const saved = await saveAnalysis(config.history.dir, doc, key, renderMarkdown(doc));
|
|
1386
|
+
ctx.err(`saved ${saved.map((p) => fromCwd(p)).join(" and ")}. Commit them with git.`);
|
|
1387
|
+
}
|
|
1388
|
+
ctx.out(flags.json ? JSON.stringify(doc, null, 2) : flags.md ? renderMarkdown(doc) : renderText(doc));
|
|
1389
|
+
return 0;
|
|
1390
|
+
}
|
|
1391
|
+
//#endregion
|
|
1392
|
+
export { analyzeCommand };
|
|
1065
1393
|
|
|
1066
|
-
//# sourceMappingURL=
|
|
1394
|
+
//# sourceMappingURL=command-C-HeGlap.js.map
|