reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,67 +1,22 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { a as oneLine, i as formatDuration, n as clip, r as fenced } from "./text-CrVy5ioL.js";
|
|
1
|
+
import { a as plural, i as formatDuration, n as clip, r as fenced } from "./text-DTigcA64.js";
|
|
3
2
|
import { n as formatTokens } from "./usage-BNY1ABrU.js";
|
|
4
|
-
import { C as isUngatedMiss,
|
|
5
|
-
import { c as isStudy, r as countedOf, s as isNotGatedResult, t as
|
|
6
|
-
import {
|
|
7
|
-
import { o as warnings } from "./state-B1t143wG.js";
|
|
3
|
+
import { B as commandLine, C as isUngatedMiss, F as checkLabels, I as checkTries, S as isNotGatedPass, U as labeledJudgments, ft as formatSeconds, k as runtimeLabel, u as evalMarkOutcome, ut as formatCount, v as harnessLabel, w as languageLabel } from "./model-Bz7-j71_.js";
|
|
4
|
+
import { c as isStudy, r as countedOf, s as isNotGatedResult, t as canceledOf, u as suiteResult } from "./suite-DieLXvCT.js";
|
|
5
|
+
import { o as warnings } from "./state-C_kdhp5v.js";
|
|
8
6
|
import { t as colors } from "./colors-QV9dFVhN.js";
|
|
9
|
-
import { r as remarkLines } from "./remarks-
|
|
10
|
-
//#region src/insights/hash.ts
|
|
11
|
-
/** JSON with every object's keys sorted. */
|
|
12
|
-
function canonicalJson(value) {
|
|
13
|
-
return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
14
|
-
}
|
|
15
|
-
const factsHash = (suite) => `sha256:${sha256(canonicalJson(suite))}`;
|
|
16
|
-
//#endregion
|
|
7
|
+
import { r as remarkLines } from "./remarks-eBcrIZRu.js";
|
|
17
8
|
//#region src/ui/layout.ts
|
|
18
|
-
/** The block for an eval: a pure function of its
|
|
19
|
-
function layout(
|
|
9
|
+
/** The block for an eval: a pure function of its shape. */
|
|
10
|
+
function layout(across, isFamily, isCompare) {
|
|
20
11
|
if (isCompare) return {
|
|
21
12
|
kind: "regressions",
|
|
22
13
|
family: false
|
|
23
14
|
};
|
|
24
15
|
return {
|
|
25
|
-
kind:
|
|
16
|
+
kind: across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
|
|
26
17
|
family: isFamily
|
|
27
18
|
};
|
|
28
19
|
}
|
|
29
|
-
const moved = (c) => c.effect === "better" || c.effect === "worse";
|
|
30
|
-
/** A cell the agent failed, or that errored; not one whose only fails the interrupt cancelled. */
|
|
31
|
-
const failedCell = (c) => c.outcome === "fail" && c.failed > cancelledOf(c) || c.outcome === "infra_error";
|
|
32
|
-
/**
|
|
33
|
-
* The insight slots an eval's layout opens: which rows take a sentence (a failing case; an arm
|
|
34
|
-
* that failed or moved; a failing harness), whether it has findings (a choose or finish eval that
|
|
35
|
-
* measures a change), and whether it has named tools (a choose eval). An `other` eval takes only
|
|
36
|
-
* its answer, unless it compares harnesses, which takes a sentence on any job. What the run's
|
|
37
|
-
* interrupt cancelled opens nothing: the agent didn't fail it, and its effects aren't measured.
|
|
38
|
-
*/
|
|
39
|
-
function slotsOf(job, e) {
|
|
40
|
-
const known = job !== "other";
|
|
41
|
-
const measured = e.outcome !== "cancelled";
|
|
42
|
-
const harnesses = e.across === "harness" || e.across === "both";
|
|
43
|
-
const rows = [];
|
|
44
|
-
if (known || harnesses) {
|
|
45
|
-
if (e.cases) {
|
|
46
|
-
for (const k of e.cases) if (k.outcome !== "pass" && k.outcome !== "cancelled") rows.push(k.path);
|
|
47
|
-
for (const p of e.pooled ?? []) {
|
|
48
|
-
const pooledMoved = measured && (p.effect === "better" || p.effect === "worse");
|
|
49
|
-
if (p.outcome === "fail" || pooledMoved) rows.push(p.path);
|
|
50
|
-
}
|
|
51
|
-
} else if (e.across === "harness") {
|
|
52
|
-
for (const c of e.cells ?? []) if (failedCell(c)) rows.push(c.path);
|
|
53
|
-
} else if (e.across === "change" || e.across === "both") {
|
|
54
|
-
for (const c of e.cells ?? []) if (failedCell(c) || measured && moved(c)) rows.push(c.path);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
rows,
|
|
59
|
-
findings: known && e.across === "change" && measured,
|
|
60
|
-
namedTools: job === "choose"
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
/** The last line of a run's output while no analysis model writes its answers. */
|
|
64
|
-
const NO_MODEL_HINT = "Set analysis.model in reeval.config.ts for answers to what each eval asks.";
|
|
65
20
|
/** The narrowest a row's label column is, so short labels line up under the name. */
|
|
66
21
|
const LABEL = 17;
|
|
67
22
|
const ASSERTION_MAX = 160;
|
|
@@ -72,7 +27,7 @@ const VERDICT = {
|
|
|
72
27
|
pass: "pass",
|
|
73
28
|
fail: "fail",
|
|
74
29
|
infra_error: "infra error",
|
|
75
|
-
|
|
30
|
+
canceled: "canceled",
|
|
76
31
|
skipped: "skipped"
|
|
77
32
|
};
|
|
78
33
|
const EFFECT = {
|
|
@@ -95,9 +50,12 @@ function table(items, width = LABEL) {
|
|
|
95
50
|
const rows = items.filter((i) => "cells" in i);
|
|
96
51
|
const first = Math.max(width, ...rows.filter((r) => r.cells.length > 1).map((r) => r.indent + textOf(r.cells[0] ?? "").length));
|
|
97
52
|
const widths = [];
|
|
98
|
-
for (const r of rows)
|
|
99
|
-
if (
|
|
100
|
-
|
|
53
|
+
for (const r of rows) {
|
|
54
|
+
if (r.loose) continue;
|
|
55
|
+
r.cells.forEach((cell, i) => {
|
|
56
|
+
if (i > 0) widths[i] = Math.max(widths[i] ?? 0, textOf(cell).length);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
101
59
|
return items.map((item) => {
|
|
102
60
|
const pad = " ".repeat(item.indent);
|
|
103
61
|
if ("free" in item) return `${pad}${show(item.free)}`;
|
|
@@ -108,25 +66,28 @@ function table(items, width = LABEL) {
|
|
|
108
66
|
}).join(" ")}`.trimEnd();
|
|
109
67
|
});
|
|
110
68
|
}
|
|
111
|
-
const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "
|
|
69
|
+
const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "canceled" ? c.yellow : c.dim;
|
|
112
70
|
function verdict(c, e) {
|
|
113
71
|
if (e.todo) return t("todo", c.dim);
|
|
114
72
|
if (e.outcome === "pass" && isStudy(e.cells ?? [])) return t("not gated", c.dim);
|
|
115
73
|
return t(VERDICT[e.outcome], paintOutcome(c, e.outcome));
|
|
116
74
|
}
|
|
117
|
-
const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
118
75
|
/**
|
|
119
76
|
* `3 of 4`, with the attempts that didn't count named: `2 of 3 · 1 infra error · 1 skipped`. A
|
|
120
|
-
*
|
|
77
|
+
* canceled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart, and
|
|
78
|
+
* a best-of cell names the attempts its majority made unneeded: `0 of 3 · 2 not needed (best of 5)`.
|
|
121
79
|
*/
|
|
122
80
|
function cellCount(cell) {
|
|
123
|
-
const
|
|
81
|
+
const canceled = canceledOf(cell);
|
|
124
82
|
const counted = countedOf(cell);
|
|
83
|
+
const notNeeded = cell.notNeeded ?? 0;
|
|
84
|
+
const skipped = cell.skipped - notNeeded;
|
|
125
85
|
const parts = [
|
|
126
86
|
counted > 0 ? `${cell.passed} of ${counted}` : "",
|
|
127
87
|
cell.infraError ? plural(cell.infraError, "infra error") : "",
|
|
128
|
-
|
|
129
|
-
|
|
88
|
+
canceled ? `${canceled} canceled` : "",
|
|
89
|
+
skipped ? `${skipped} skipped` : "",
|
|
90
|
+
notNeeded ? `${notNeeded} not needed (best of ${cell.bestOf})` : ""
|
|
130
91
|
].filter(Boolean);
|
|
131
92
|
if (!parts.length) return "not run";
|
|
132
93
|
return parts.join(" · ");
|
|
@@ -145,6 +106,8 @@ function assertionOf(k) {
|
|
|
145
106
|
const line = k.outcome.assertion.trim().split("\n")[0]?.trim() ?? "";
|
|
146
107
|
return clip(line.startsWith(`${k.label}: `) ? line.slice(k.label.length + 2) : line, ASSERTION_MAX);
|
|
147
108
|
}
|
|
109
|
+
/** Where the eval made the check's first failure, `evals/a.eval.ts:7`. */
|
|
110
|
+
const atOf = (k) => k.outcome.status === "ran" ? k.outcome.at ?? "" : "";
|
|
148
111
|
/** A cell passed outright: every counted attempt, every check, nothing errored. */
|
|
149
112
|
const clean = (cell) => cell.outcome === "pass" && cell.failed === 0 && cell.infraError === 0 && cell.errors.length === 0 && !cell.checks.some(failing);
|
|
150
113
|
/** An error no check explains, by kind: a throw in the eval's body reads `error`. */
|
|
@@ -152,7 +115,7 @@ const ERROR_KIND = {
|
|
|
152
115
|
assertion: "error",
|
|
153
116
|
judge: "judge",
|
|
154
117
|
timeout: "timeout",
|
|
155
|
-
aborted: "
|
|
118
|
+
aborted: "canceled",
|
|
156
119
|
infra: "infra error",
|
|
157
120
|
turn: "turn error"
|
|
158
121
|
};
|
|
@@ -164,9 +127,14 @@ function evidence(c, cell, indent, more = true) {
|
|
|
164
127
|
const items = [];
|
|
165
128
|
for (const k of cell.checks.filter(failing)) {
|
|
166
129
|
const count = checkCount(k, single);
|
|
130
|
+
const at = atOf(k);
|
|
167
131
|
items.push({
|
|
168
132
|
indent,
|
|
169
|
-
cells: [
|
|
133
|
+
cells: [
|
|
134
|
+
checkLabel(k.label),
|
|
135
|
+
count === "not run" ? t(count, c.dim) : t(count, c.red),
|
|
136
|
+
...at ? [t(at, c.dim)] : []
|
|
137
|
+
]
|
|
170
138
|
});
|
|
171
139
|
const assertion = assertionOf(k);
|
|
172
140
|
if (assertion) items.push({
|
|
@@ -183,6 +151,15 @@ function evidence(c, cell, indent, more = true) {
|
|
|
183
151
|
cells: [errorKind(err), t(errorText(err), c.dim)]
|
|
184
152
|
});
|
|
185
153
|
items.push(...retriedItems(c, cell, indent));
|
|
154
|
+
const [snapshot, ...others] = cell.snapshots ?? [];
|
|
155
|
+
if (snapshot && items.length) {
|
|
156
|
+
const more = others.length ? ` (+${others.length} more in summary.md)` : "";
|
|
157
|
+
items.push({
|
|
158
|
+
indent,
|
|
159
|
+
cells: ["snapshot", t(`${snapshot}${more}`, c.dim)],
|
|
160
|
+
loose: true
|
|
161
|
+
});
|
|
162
|
+
}
|
|
186
163
|
const passed = cell.checks.filter((k) => !failing(k) && !k.retried).length;
|
|
187
164
|
if (more && passed && items.length) items.push({
|
|
188
165
|
indent,
|
|
@@ -198,7 +175,7 @@ const retriedItems = (c, cell, indent) => cell.checks.filter((k) => !failing(k)
|
|
|
198
175
|
/** A control with no environment of its own is the baseline the variants change. */
|
|
199
176
|
const armOf = (cell) => cell.variant ?? (cell.environment === "-" ? "baseline" : cell.environment);
|
|
200
177
|
/** Each cell's label: the parts that vary between them (harness, arm, sandbox). */
|
|
201
|
-
function
|
|
178
|
+
function labeler(cells, parts) {
|
|
202
179
|
const vary = parts.filter((p) => {
|
|
203
180
|
const of = (cell) => p === "harness" ? cell.harness : p === "arm" ? armOf(cell) : cell.sandbox;
|
|
204
181
|
return new Set(cells.map(of)).size > 1;
|
|
@@ -277,73 +254,9 @@ function effortTable(c, cells, label, indent, failed, width) {
|
|
|
277
254
|
]
|
|
278
255
|
}))], width);
|
|
279
256
|
}
|
|
280
|
-
const NO_SLOTS = {
|
|
281
|
-
row: () => null,
|
|
282
|
-
named: () => [],
|
|
283
|
-
subject: null
|
|
284
|
-
};
|
|
285
|
-
const triesOf = (cells) => cells.reduce((n, x) => n + countedOf(x), 0);
|
|
286
|
-
/** `named`, then a row per tool with the tries that named it. */
|
|
287
|
-
function namedItems(c, slots, cells, indent) {
|
|
288
|
-
const named = slots.named(cells);
|
|
289
|
-
if (!named.length) return [];
|
|
290
|
-
const of = triesOf(cells);
|
|
291
|
-
return [{
|
|
292
|
-
indent,
|
|
293
|
-
free: t("named", c.dim)
|
|
294
|
-
}, ...named.map((n) => ({
|
|
295
|
-
indent: indent + 2,
|
|
296
|
-
cells: [n.tool, `${n.tries} of ${of}`]
|
|
297
|
-
}))];
|
|
298
|
-
}
|
|
299
|
-
/** The tries that named the subject, over the cells' tries: `0 of 4`; empty with no subject. */
|
|
300
|
-
function subjectTries(slots, cells) {
|
|
301
|
-
if (!slots.subject) return "";
|
|
302
|
-
const subject = slots.subject.toLowerCase();
|
|
303
|
-
return `${slots.named(cells).find((x) => x.tool.toLowerCase() === subject)?.tries ?? 0} of ${triesOf(cells)}`;
|
|
304
|
-
}
|
|
305
|
-
/** Web searches or fetches over the tries, when every try reported a number. */
|
|
306
|
-
function webItems(cells, indent) {
|
|
307
|
-
const row = (name, counts) => counts.length && counts.every((x) => x !== null) ? [{
|
|
308
|
-
indent,
|
|
309
|
-
cells: [name, String(counts.reduce((a, b) => (a ?? 0) + (b ?? 0), 0))]
|
|
310
|
-
}] : [];
|
|
311
|
-
return [...row("searches", cells.flatMap((x) => x.searches)), ...row("fetches", cells.flatMap((x) => x.fetches))];
|
|
312
|
-
}
|
|
313
|
-
/** A choose case that missed: its failed checks, the tools named, its searches, then `+N more passed`. */
|
|
314
|
-
function chooseDetail(c, cells, indent, slots) {
|
|
315
|
-
const checks = cells.flatMap((cell) => evidence(c, cell, indent, false));
|
|
316
|
-
const passed = cells.flatMap((x) => x.checks.filter((k) => !failing(k))).length;
|
|
317
|
-
return [
|
|
318
|
-
...checks,
|
|
319
|
-
...namedItems(c, slots, cells, indent),
|
|
320
|
-
...webItems(cells, indent),
|
|
321
|
-
...passed ? [{
|
|
322
|
-
indent,
|
|
323
|
-
free: t(`+${passed} more passed`, c.dim)
|
|
324
|
-
}] : []
|
|
325
|
-
];
|
|
326
|
-
}
|
|
327
|
-
/** Choose a tool, on a single eval: the subject's count, and on a miss the detail. */
|
|
328
|
-
function chooseBody(c, cells, indent, slots) {
|
|
329
|
-
const items = slots.subject ? [{
|
|
330
|
-
indent,
|
|
331
|
-
cells: [slots.subject, subjectTries(slots, cells)]
|
|
332
|
-
}] : [];
|
|
333
|
-
if (!cells.every(clean)) items.push(...chooseDetail(c, cells, indent, slots));
|
|
334
|
-
return table(items);
|
|
335
|
-
}
|
|
336
|
-
/** A row's sentence, as a line under it. */
|
|
337
|
-
const rowLine = (slots, path, indent) => {
|
|
338
|
-
const text = slots.row(path);
|
|
339
|
-
return text ? [{
|
|
340
|
-
indent,
|
|
341
|
-
free: text
|
|
342
|
-
}] : [];
|
|
343
|
-
};
|
|
344
257
|
/** Measure a change: a row per arm with its effect, the check matrix and the effort table. */
|
|
345
258
|
function measureBody(c, cells, indent, o) {
|
|
346
|
-
const label =
|
|
259
|
+
const label = labeler(cells, ["arm", "sandbox"]);
|
|
347
260
|
const arms = [{
|
|
348
261
|
indent,
|
|
349
262
|
cells: ["arm", t("pass", c.dim)]
|
|
@@ -358,7 +271,6 @@ function measureBody(c, cells, indent, o) {
|
|
|
358
271
|
...effect.text ? [effect] : []
|
|
359
272
|
]
|
|
360
273
|
});
|
|
361
|
-
arms.push(...rowLine(o.slots, cell.path, indent + 2), ...namedItems(c, o.slots, [cell], indent + 2));
|
|
362
274
|
const stats = o.verbose && o.effects ? statsLine(cell, cells) : null;
|
|
363
275
|
if (stats) arms.push({
|
|
364
276
|
indent: indent + 2,
|
|
@@ -379,8 +291,8 @@ function measureBody(c, cells, indent, o) {
|
|
|
379
291
|
];
|
|
380
292
|
}
|
|
381
293
|
/** A row per cell (harness, or sandbox), the ones that didn't pass outright with their evidence. */
|
|
382
|
-
function cellRows(c, cells, heading, indent
|
|
383
|
-
const label =
|
|
294
|
+
function cellRows(c, cells, heading, indent) {
|
|
295
|
+
const label = labeler(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
|
|
384
296
|
const items = [{
|
|
385
297
|
indent,
|
|
386
298
|
cells: [heading, t("pass", c.dim)]
|
|
@@ -390,14 +302,13 @@ function cellRows(c, cells, heading, indent, slots) {
|
|
|
390
302
|
indent,
|
|
391
303
|
cells: [label(cell), cellCount(cell)]
|
|
392
304
|
});
|
|
393
|
-
items.push(...
|
|
394
|
-
if (!clean(cell)) items.push(...namedItems(c, slots, [cell], indent + 2), ...evidence(c, cell, indent + 2));
|
|
305
|
+
if (!clean(cell)) items.push(...evidence(c, cell, indent + 2));
|
|
395
306
|
}
|
|
396
307
|
return table(items);
|
|
397
308
|
}
|
|
398
309
|
/** Compare harnesses with arms: a heading per harness, its arms as Measure a change under it. */
|
|
399
310
|
function bothBody(c, cells, indent, o) {
|
|
400
|
-
const harnessOf =
|
|
311
|
+
const harnessOf = labeler(cells, ["harness", "sandbox"]);
|
|
401
312
|
const groups = /* @__PURE__ */ new Map();
|
|
402
313
|
for (const cell of cells) groups.set(harnessOf(cell), [...groups.get(harnessOf(cell)) ?? [], cell]);
|
|
403
314
|
return [...groups].flatMap(([harness, group]) => [`${" ".repeat(indent)}${harness}`, ...measureBody(c, group, indent + 2, o)]);
|
|
@@ -405,19 +316,19 @@ function bothBody(c, cells, indent, o) {
|
|
|
405
316
|
/** The rows under an eval's head (or a family case's), by what its cells compare. */
|
|
406
317
|
function cellsBody(c, kind, cells, indent, o) {
|
|
407
318
|
if (kind === "measure") return measureBody(c, cells, indent, o);
|
|
408
|
-
if (kind === "harnesses") return cellRows(c, cells, "harness", indent
|
|
319
|
+
if (kind === "harnesses") return cellRows(c, cells, "harness", indent);
|
|
409
320
|
if (kind === "both") return bothBody(c, cells, indent, o);
|
|
410
|
-
if (
|
|
411
|
-
if (cells.length > 1) return cellRows(c, cells, "sandbox", indent, o.slots);
|
|
321
|
+
if (cells.length > 1) return cellRows(c, cells, "sandbox", indent);
|
|
412
322
|
return resultsBody(c, cells[0], indent);
|
|
413
323
|
}
|
|
414
324
|
/** A single eval with one cell: its tries, and on anything but a clean pass, the evidence. */
|
|
415
325
|
function resultsBody(c, cell, indent) {
|
|
416
326
|
if (clean(cell)) return table(retriedItems(c, cell, indent));
|
|
417
327
|
const items = [];
|
|
418
|
-
|
|
328
|
+
const tries = cell.attempts - (cell.notNeeded ?? 0);
|
|
329
|
+
if (tries > 1) items.push({
|
|
419
330
|
indent,
|
|
420
|
-
cells: [plural(
|
|
331
|
+
cells: [plural(tries, "try", "tries"), cellCount(cell)]
|
|
421
332
|
});
|
|
422
333
|
return table([...items, ...evidence(c, cell, indent)]);
|
|
423
334
|
}
|
|
@@ -491,10 +402,9 @@ function pooledStats(p, label) {
|
|
|
491
402
|
/**
|
|
492
403
|
* A family: its template, then a row per case. With one cell a case is a row with its count;
|
|
493
404
|
* with more, the case is outer and its cells inner, laid out as a single eval's are, and a
|
|
494
|
-
* pooled `all cases` row sums each cell's counts with its variant's pooled effect.
|
|
495
|
-
* family's case row carries the subject's count instead, and a missed case the tools named.
|
|
405
|
+
* pooled `all cases` row sums each cell's counts with its variant's pooled effect.
|
|
496
406
|
*/
|
|
497
|
-
function familyBlock(c, e, kind, verbose
|
|
407
|
+
function familyBlock(c, e, kind, verbose) {
|
|
498
408
|
const cases = e.cases ?? [];
|
|
499
409
|
const columns = [...new Map(cases.flatMap((k) => k.cells).map((cell) => [cell.key, cell])).values()];
|
|
500
410
|
const pooled = e.pooled ?? [];
|
|
@@ -513,38 +423,6 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
513
423
|
});
|
|
514
424
|
const head = `${" ".repeat(17)}${e.template ?? e.name}`;
|
|
515
425
|
const questions = questionLines(c, e.questions, 2);
|
|
516
|
-
if (kind === "choose") {
|
|
517
|
-
const subject = slots.subject ? [t(slots.subject, c.dim)] : [];
|
|
518
|
-
const items = [{
|
|
519
|
-
indent: 2,
|
|
520
|
-
cells: [
|
|
521
|
-
"case",
|
|
522
|
-
t("pass", c.dim),
|
|
523
|
-
...subject
|
|
524
|
-
]
|
|
525
|
-
}];
|
|
526
|
-
for (const k of cases) {
|
|
527
|
-
const passed = k.cells.reduce((n, x) => n + x.passed, 0);
|
|
528
|
-
const tries = triesOf(k.cells);
|
|
529
|
-
const count = k.cells.length === 1 && k.cells[0] ? cellCount(k.cells[0]) : tries ? `${passed} of ${tries}` : "not run";
|
|
530
|
-
const named = slots.subject && tries ? [subjectTries(slots, k.cells)] : [];
|
|
531
|
-
items.push({
|
|
532
|
-
indent: 2,
|
|
533
|
-
cells: [
|
|
534
|
-
lead(caseWord(k), k.label),
|
|
535
|
-
count,
|
|
536
|
-
...named
|
|
537
|
-
]
|
|
538
|
-
});
|
|
539
|
-
items.push(...rowLine(slots, k.path, 4));
|
|
540
|
-
if (!k.cells.every(clean)) items.push(...chooseDetail(c, k.cells, 4, slots));
|
|
541
|
-
}
|
|
542
|
-
return [
|
|
543
|
-
head,
|
|
544
|
-
...table(items),
|
|
545
|
-
...questions
|
|
546
|
-
];
|
|
547
|
-
}
|
|
548
426
|
if (columns.length <= 1) {
|
|
549
427
|
const items = [];
|
|
550
428
|
for (const k of cases) {
|
|
@@ -553,7 +431,6 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
553
431
|
indent: 2,
|
|
554
432
|
cells: [lead(caseWord(k), k.label), cell ? cellCount(cell) : ""]
|
|
555
433
|
});
|
|
556
|
-
items.push(...rowLine(slots, k.path, 4));
|
|
557
434
|
if (cell && !clean(cell)) items.push(...evidence(c, cell, 4));
|
|
558
435
|
}
|
|
559
436
|
return [
|
|
@@ -565,18 +442,15 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
565
442
|
const out = [head];
|
|
566
443
|
for (const k of cases) {
|
|
567
444
|
out.push(` ${show(lead(caseWord(k), k.label))}`);
|
|
568
|
-
const sentence = slots.row(k.path);
|
|
569
|
-
if (sentence) out.push(` ${sentence}`);
|
|
570
445
|
const failed = k.outcome !== "pass";
|
|
571
446
|
out.push(...cellsBody(c, kind, k.cells, 4, {
|
|
572
447
|
failed,
|
|
573
448
|
verbose,
|
|
574
|
-
effects: false
|
|
575
|
-
slots
|
|
449
|
+
effects: false
|
|
576
450
|
}));
|
|
577
451
|
}
|
|
578
452
|
if (pooled.length) {
|
|
579
|
-
const label =
|
|
453
|
+
const label = labeler(columns, [
|
|
580
454
|
"harness",
|
|
581
455
|
"arm",
|
|
582
456
|
"sandbox"
|
|
@@ -597,7 +471,6 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
597
471
|
...effect.text ? [effect] : []
|
|
598
472
|
]
|
|
599
473
|
});
|
|
600
|
-
if (p) items.push(...rowLine(slots, p.path, 6));
|
|
601
474
|
if (p && verbose) items.push({
|
|
602
475
|
indent: 6,
|
|
603
476
|
free: t(pooledStats(p, label(col)), c.dim)
|
|
@@ -607,43 +480,40 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
607
480
|
}
|
|
608
481
|
return [...out, ...questions];
|
|
609
482
|
}
|
|
610
|
-
/**
|
|
611
|
-
function
|
|
612
|
-
|
|
613
|
-
const
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
}
|
|
483
|
+
/** `Anthropic 12 · Amazon Bedrock 3`: each upstream provider and how many calls it served, summed. */
|
|
484
|
+
function upstreamList(counts) {
|
|
485
|
+
const total = /* @__PURE__ */ new Map();
|
|
486
|
+
for (const u of counts) total.set(u.provider, (total.get(u.provider) ?? 0) + u.calls);
|
|
487
|
+
return [...total].map(([provider, calls]) => `${provider} ${calls}`).join(" · ");
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* Which upstream providers a gateway said served each harness's calls and its judge's, a line per
|
|
491
|
+
* harness, so a run OpenRouter spread over several shows it. None where the gateway named none.
|
|
492
|
+
*/
|
|
493
|
+
function upstreamLines(c, cells, indent) {
|
|
494
|
+
const by = /* @__PURE__ */ new Map();
|
|
495
|
+
for (const cell of cells) {
|
|
496
|
+
if (!cell.upstreams && !cell.judgeUpstreams) continue;
|
|
497
|
+
const seen = by.get(cell.harness) ?? {
|
|
498
|
+
agent: [],
|
|
499
|
+
judge: []
|
|
500
|
+
};
|
|
501
|
+
seen.agent.push(...cell.upstreams ?? []);
|
|
502
|
+
seen.judge.push(...cell.judgeUpstreams ?? []);
|
|
503
|
+
by.set(cell.harness, seen);
|
|
504
|
+
}
|
|
505
|
+
return [...by].map(([harness, { agent, judge }]) => {
|
|
506
|
+
const parts = [...agent.length ? [`served by ${upstreamList(agent)}`] : [], ...judge.length ? [`judge served by ${upstreamList(judge)}`] : []];
|
|
507
|
+
const label = by.size > 1 ? `${harness} ` : "";
|
|
508
|
+
return `${" ".repeat(indent)}${c.dim(`${label}${parts.join("; ")}`)}`;
|
|
509
|
+
});
|
|
632
510
|
}
|
|
633
|
-
/** One eval's block, laid out as `layout()` says for its
|
|
634
|
-
function evalBlock(c, e, verbose = false
|
|
635
|
-
const shape = layout(
|
|
636
|
-
const slots = slotsFor(insight);
|
|
637
|
-
const answer = insight?.answer ? [` ${oneLine(insight.answer.text)}`] : [];
|
|
638
|
-
const findings = (insight?.findings ?? []).map((f) => ` ${oneLine(f.text)}`);
|
|
511
|
+
/** One eval's block, laid out as `layout()` says for its shape. */
|
|
512
|
+
function evalBlock(c, e, verbose = false) {
|
|
513
|
+
const shape = layout(e.across, e.family, false);
|
|
639
514
|
if (shape.family) {
|
|
640
|
-
const
|
|
641
|
-
return [
|
|
642
|
-
head,
|
|
643
|
-
...answer,
|
|
644
|
-
...rest,
|
|
645
|
-
...findings
|
|
646
|
-
];
|
|
515
|
+
const served = upstreamLines(c, (e.cases ?? []).flatMap((k) => k.cells), 2);
|
|
516
|
+
return [...familyBlock(c, e, shape.kind, verbose), ...served];
|
|
647
517
|
}
|
|
648
518
|
const v = verdict(c, e);
|
|
649
519
|
const name = `${e.group.length ? `${e.group.join(" › ")} › ` : ""}${e.name}`;
|
|
@@ -651,22 +521,19 @@ function evalBlock(c, e, verbose = false, insight = null) {
|
|
|
651
521
|
const cells = e.cells ?? [];
|
|
652
522
|
if (e.todo || e.outcome === "skipped" || !cells.length) return [head];
|
|
653
523
|
const failed = e.outcome !== "pass";
|
|
654
|
-
const body = cellsBody(c, shape.kind, cells, 2, {
|
|
655
|
-
failed,
|
|
656
|
-
verbose,
|
|
657
|
-
effects: true,
|
|
658
|
-
slots
|
|
659
|
-
});
|
|
660
524
|
return [
|
|
661
525
|
head,
|
|
662
|
-
...
|
|
663
|
-
|
|
664
|
-
|
|
526
|
+
...cellsBody(c, shape.kind, cells, 2, {
|
|
527
|
+
failed,
|
|
528
|
+
verbose,
|
|
529
|
+
effects: true
|
|
530
|
+
}),
|
|
531
|
+
...upstreamLines(c, cells, 2),
|
|
665
532
|
...questionLines(c, e.questions, 2)
|
|
666
533
|
];
|
|
667
534
|
}
|
|
668
535
|
/**
|
|
669
|
-
* `5 fail · 2 pass · 1 not gated · 1
|
|
536
|
+
* `5 fail · 2 pass · 1 not gated · 1 canceled`: errors outside any eval and fails first, every
|
|
670
537
|
* case counting one, zeros left out.
|
|
671
538
|
*/
|
|
672
539
|
function tallyLine(c, tally) {
|
|
@@ -676,16 +543,15 @@ function tallyLine(c, tally) {
|
|
|
676
543
|
tally.pass ? c.green(`${tally.pass} pass`) : "",
|
|
677
544
|
tally.notGated ? c.dim(`${tally.notGated} not gated`) : "",
|
|
678
545
|
tally.infraError ? c.yellow(plural(tally.infraError, "infra error")) : "",
|
|
679
|
-
tally.
|
|
546
|
+
tally.canceled ? c.yellow(`${tally.canceled} canceled`) : "",
|
|
680
547
|
tally.skipped ? c.dim(`${tally.skipped} skipped`) : ""
|
|
681
548
|
].filter(Boolean);
|
|
682
549
|
return parts.length ? parts.join(c.dim(" · ")) : "no evals ran";
|
|
683
550
|
}
|
|
684
551
|
/** Every eval's block in file-path order, then source order; a blank line around each longer one. */
|
|
685
|
-
function renderBlocks(suite, opts
|
|
552
|
+
function renderBlocks(suite, opts) {
|
|
686
553
|
const c = colors(opts.color);
|
|
687
|
-
const
|
|
688
|
-
const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose, byEval.get(e.path) ?? null)));
|
|
554
|
+
const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose)));
|
|
689
555
|
const out = [];
|
|
690
556
|
blocks.forEach((block, i) => {
|
|
691
557
|
const prev = blocks[i - 1];
|
|
@@ -694,51 +560,20 @@ function renderBlocks(suite, opts, insights = null) {
|
|
|
694
560
|
});
|
|
695
561
|
return out;
|
|
696
562
|
}
|
|
697
|
-
/** Insights for these facts: null when there are none, they failed, or they were written for other facts. */
|
|
698
|
-
const freshInsights = (suite, insights) => insights && !insights.error && insights.factsHash === factsHash(suite) ? insights : null;
|
|
699
|
-
/** Why a run's insights didn't print, or with `-v` what wrote them. */
|
|
700
|
-
function insightNotes(c, suite, insights, fresh, verbose) {
|
|
701
|
-
if (!insights || !fromModel(insights)) return [];
|
|
702
|
-
if (insights.error) return [c.yellow(`insights failed (${clip(insights.error, 200)})`)];
|
|
703
|
-
if (!fresh) return [c.dim(`insights left out: insights.json was written for other facts than this run's; delete it and run reeval analyze ${suite.runId} --insights`)];
|
|
704
|
-
const skipped = insights.budget?.skipped.length ?? 0;
|
|
705
|
-
const failed = insights.failed ?? [];
|
|
706
|
-
const stopped = insights.stopped;
|
|
707
|
-
const finish = `reeval analyze ${suite.runId} --insights to finish`;
|
|
708
|
-
const budget = [
|
|
709
|
-
...skipped ? [c.yellow(`insights skipped for ${plural(skipped, "eval")}: analysis.maxTokens (${formatTokens(insights.budget?.maxTokens ?? 0)}) reached — raise it in reeval.config.ts, or ${finish}`)] : [],
|
|
710
|
-
...failed[0] ? [c.yellow(`insights failed for ${plural(failed.length, "eval")} (${clip(failed[0].error, 160)}), so they print their facts: ${finish}`)] : [],
|
|
711
|
-
...stopped ? [c.yellow(`insights stopped: ${plural(stopped.skipped.length, "eval")} skipped (${stopped.signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"}): ${finish}`)] : []
|
|
712
|
-
];
|
|
713
|
-
if (!verbose) return budget;
|
|
714
|
-
const { dropped, regenerated } = insights.validation;
|
|
715
|
-
return [...budget, c.dim([
|
|
716
|
-
`insights by ${insights.model} (${insights.promptVersion})`,
|
|
717
|
-
`${plural(dropped, "sentence")} removed`,
|
|
718
|
-
...regenerated ? [`${regenerated} asked again`] : [],
|
|
719
|
-
`${formatTokens(insights.usage.inputTokens)} in, ${formatTokens(insights.usage.outputTokens)} out${insights.usage.reasoningTokens ? ` (${formatTokens(insights.usage.reasoningTokens)} reasoning)` : ""}`
|
|
720
|
-
].join(" · "))];
|
|
721
|
-
}
|
|
722
563
|
/**
|
|
723
|
-
* A run's result: its errors outside any eval, its blocks
|
|
724
|
-
*
|
|
725
|
-
* other facts are left out with a note; remarks come from the record and history, not a model,
|
|
726
|
-
* so they print either way.
|
|
564
|
+
* A run's result: its errors outside any eval, its blocks, then the tally and the remarks, which
|
|
565
|
+
* come from the record and the history.
|
|
727
566
|
*/
|
|
728
|
-
function render(suite,
|
|
567
|
+
function render(suite, remarks, opts) {
|
|
729
568
|
const c = colors(opts.color);
|
|
730
|
-
const
|
|
731
|
-
const blocks = renderBlocks(suite, opts, fresh);
|
|
569
|
+
const blocks = renderBlocks(suite, opts);
|
|
732
570
|
return [
|
|
733
571
|
...suite.errors.flatMap((error) => [c.red(`error: ${error}`), ""]),
|
|
734
572
|
...blocks,
|
|
735
573
|
...blocks.length ? [""] : [],
|
|
736
|
-
...fresh?.headline ? [oneLine(fresh.headline.text)] : [],
|
|
737
|
-
...insightNotes(c, suite, insights, fresh, opts.verbose),
|
|
738
574
|
...opts.notes ?? [],
|
|
739
575
|
tallyLine(c, suite.tally),
|
|
740
|
-
...
|
|
741
|
-
...remarkLines(c, suite, insights, opts.verbose)
|
|
576
|
+
...remarkLines(c, suite, remarks, opts.verbose)
|
|
742
577
|
];
|
|
743
578
|
}
|
|
744
579
|
//#endregion
|
|
@@ -777,7 +612,7 @@ function stateRows(run) {
|
|
|
777
612
|
return rows;
|
|
778
613
|
}
|
|
779
614
|
/** Markdown for people and agents landing on a run cold. Links are relative to the artifact dir. */
|
|
780
|
-
function renderSummary(run, opts) {
|
|
615
|
+
function renderSummary(run, opts = {}) {
|
|
781
616
|
const lines = [
|
|
782
617
|
`# reeval run ${run.runId}`,
|
|
783
618
|
"",
|
|
@@ -788,10 +623,9 @@ function renderSummary(run, opts) {
|
|
|
788
623
|
...run.plan.sandboxWarnings.map((w) => `> **Warning:** ${w.message}\n`),
|
|
789
624
|
"## Results",
|
|
790
625
|
"",
|
|
791
|
-
...fenced(render(suiteResult(run), opts.
|
|
626
|
+
...fenced(render(suiteResult(run), opts.remarks ?? null, {
|
|
792
627
|
color: false,
|
|
793
|
-
verbose: false
|
|
794
|
-
hint: opts.hint
|
|
628
|
+
verbose: false
|
|
795
629
|
}), "text"),
|
|
796
630
|
"",
|
|
797
631
|
"## State",
|
|
@@ -844,9 +678,9 @@ function attemptDetail(lines, a) {
|
|
|
844
678
|
const act = a.activity;
|
|
845
679
|
const web = (n, what) => n === null ? `${what} not reported` : `${n} ${what}`;
|
|
846
680
|
lines.push(`- Agent: ${act.fileEdits} file edits · ${act.tools} other tool calls · ${web(act.webSearches, "web searches")} · ${web(act.webFetches, "fetches")} · ${formatDuration(act.reasoningMs)} reasoning${act.stalledTools ? ` · ${act.stalledTools} stalled tool${act.stalledTools === 1 ? "" : "s"} (≥2m)${act.maxToolMs ? `, longest ${formatDuration(act.maxToolMs)}` : ""}` : ""}`);
|
|
847
|
-
for (const {
|
|
681
|
+
for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
|
|
848
682
|
const retried = j.retries?.length ? `, after ${j.retries.length} ${j.retries.length === 1 ? "retry" : "retries"}` : "";
|
|
849
|
-
lines.push(`- Judge${
|
|
683
|
+
lines.push(`- Judge${labeled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
|
|
850
684
|
}
|
|
851
685
|
for (const q of a.answers ?? []) {
|
|
852
686
|
const said = q.error ? `failed: ${clip(cell(q.error), 200)}` : `${q.choice !== void 0 ? `**${cell(q.choice)}** ` : ""}${clip(cell(q.text), 300)}`;
|
|
@@ -871,8 +705,10 @@ function attemptDetail(lines, a) {
|
|
|
871
705
|
a.files.judgeInput && `[judge input](${a.files.judgeInput.path})`
|
|
872
706
|
].filter(Boolean);
|
|
873
707
|
if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
|
|
708
|
+
const snapshots = (a.snapshots ?? []).map((s) => s.path ? `[${s.name}](${s.path})` : `${s.name} not taken: ${cell(s.skipped?.message ?? "")}`);
|
|
709
|
+
if (snapshots.length) lines.push(`- Snapshots (in .reeval/runs, never recorded): ${snapshots.join(" · ")}`);
|
|
874
710
|
}
|
|
875
711
|
//#endregion
|
|
876
|
-
export {
|
|
712
|
+
export { table as _, assertionOf as a, checkLabel as c, evalBlock as d, labeler as f, t as g, show as h, stateRows as i, errorKind as l, renderBlocks as m, renderSummary as n, atOf as o, render as p, runTally as r, checkCount as s, OUTCOME_MARK as t, errorText as u };
|
|
877
713
|
|
|
878
|
-
//# sourceMappingURL=summary-
|
|
714
|
+
//# sourceMappingURL=summary-CSzyBVsF.js.map
|