reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
package/dist/view/index.d.ts
CHANGED
|
@@ -1,95 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
//#region src/stats.d.ts
|
|
3
|
-
/**
|
|
4
|
-
* Counted attempts a side before a pass-rate lift can be called: below it, every view says
|
|
5
|
-
* `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
|
|
6
|
-
* `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
|
|
7
|
-
* makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
|
|
8
|
-
* the time when nothing changed.
|
|
9
|
-
*/
|
|
10
|
-
export declare const EFFECT_MIN_COUNTED = 4;
|
|
11
|
-
//#endregion
|
|
12
|
-
//#region src/view/debug.d.ts
|
|
13
|
-
/**
|
|
14
|
-
* An attempt's local debug data, as `reeval run` writes it to `.reeval/debug/<runId>/<eval>/<cell>/<n>/`
|
|
15
|
-
* and `reeval studio` serves it. Browser-safe. Never part of a record or the history folder.
|
|
16
|
-
*
|
|
17
|
-
* debug.log the `-v` trace, one line per event, always written
|
|
18
|
-
* requests.ndjson one line per egress gateway request, always written
|
|
19
|
-
* trace.ndjson raw payloads (`-vv`), written when the attempt fails or with `--trace`
|
|
20
|
-
*/
|
|
21
|
-
export declare const DEBUG_LOG = "debug.log";
|
|
22
|
-
export declare const REQUESTS_FILE = "requests.ndjson";
|
|
23
|
-
export declare const TRACE_FILE = "trace.ndjson";
|
|
24
|
-
/** What part of the attempt a line is about, in the order an attempt goes through them. */
|
|
25
|
-
export declare const DEBUG_PHASES: readonly ['plan', 'image', 'container', 'env', 'broker', 'harness', 'check', 'judge', 'gateway', 'teardown', 'docker'];
|
|
26
|
-
export type DebugPhase = (typeof DEBUG_PHASES)[number];
|
|
27
|
-
/** One debug line: level 1 is `-v`, level 2 is `-vv` only. */
|
|
28
|
-
export type DebugLine = {
|
|
29
|
-
at: number;
|
|
30
|
-
level: 1 | 2;
|
|
31
|
-
phase: DebugPhase;
|
|
32
|
-
text: string;
|
|
33
|
-
/** More lines under it, e.g. a docker command's output or a raw payload. */
|
|
34
|
-
detail?: string;
|
|
35
|
-
};
|
|
36
|
-
/** One raw trace entry: `ms` is since the attempt started. */
|
|
37
|
-
export type TraceEntry = {
|
|
38
|
-
at: string;
|
|
39
|
-
ms: number;
|
|
40
|
-
phase: DebugPhase;
|
|
41
|
-
kind: string;
|
|
42
|
-
data: unknown;
|
|
43
|
-
};
|
|
44
|
-
/** One request through the egress gateway, or one tunnel it passed through without reading. */
|
|
45
|
-
export type DebugRequest = {
|
|
46
|
-
at: string;
|
|
47
|
-
method: string;
|
|
48
|
-
url: string;
|
|
49
|
-
host: string;
|
|
50
|
-
port: number;
|
|
51
|
-
via: 'tunnel' | 'decrypted' | 'http';
|
|
52
|
-
allowed: boolean;
|
|
53
|
-
status?: number;
|
|
54
|
-
reason?: string;
|
|
55
|
-
sent: number;
|
|
56
|
-
received: number;
|
|
57
|
-
/** Brokered secrets and the harness credential swapped in, by name. */
|
|
58
|
-
substituted: string[];
|
|
59
|
-
ms: number;
|
|
60
|
-
};
|
|
61
|
-
/** What the studio's Debug tab shows for one attempt. */
|
|
62
|
-
export type DebugView = {
|
|
63
|
-
lines: Array<{
|
|
64
|
-
at: string;
|
|
65
|
-
phase: string;
|
|
66
|
-
text: string;
|
|
67
|
-
}>;
|
|
68
|
-
requests: DebugRequest[];
|
|
69
|
-
/** Whether `trace.ndjson` was kept. */
|
|
70
|
-
trace: boolean;
|
|
71
|
-
};
|
|
72
|
-
/**
|
|
73
|
-
* The studio server's debug API (`reeval studio` only): an attempt's DebugView (null when it has no
|
|
74
|
-
* local data), and its raw trace.
|
|
75
|
-
*/
|
|
76
|
-
export declare const debugApiPath: (runId: string, attemptKey: string) => string;
|
|
77
|
-
export declare const traceApiPath: (runId: string, attemptKey: string) => string;
|
|
78
|
-
/** `12:50:01`, or `12:50:01.234` with milliseconds, in UTC like every reeval timestamp. */
|
|
79
|
-
export declare const debugTime: (at: number, ms: boolean) => string;
|
|
80
|
-
/** A line of debug.log: `2026-09-26T12:50:01.234Z plan text`. */
|
|
81
|
-
export declare const debugLogLine: (line: Pick<DebugLine, 'at' | 'phase' | 'text'>) => string;
|
|
82
|
-
/** debug.log's lines; a line that isn't one (a wrapped message) continues the one before. */
|
|
83
|
-
export declare function parseDebugLog(text: string): DebugView['lines'];
|
|
84
|
-
export declare function parseNdjson<T>(text: string): T[];
|
|
85
|
-
/** `512 B`, `1.2 KB`, `3.4 MB`. */
|
|
86
|
-
export declare function formatBytes(n: number): string;
|
|
87
|
-
/**
|
|
88
|
-
* The gateway line for a request: method, URL, status, allowed or blocked and why, bytes each way,
|
|
89
|
-
* time, and which secrets were swapped in (by name).
|
|
90
|
-
*/
|
|
91
|
-
export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
|
|
92
|
-
//#endregion
|
|
1
|
+
import { An as Spread, Bn as effortLabel, Cn as DIRECTIONAL_MIN, Dn as EffortMetric, En as EffortContrast, Fn as cellEffort, Gn as formatCount, Hn as effortSample, In as cellSamples, Jn as tokenValues, Jt as TranscriptEvent, Kn as formatRatio, Ln as contrastEffort, Mn as VERDICT_TEXT, Nn as bothUnderATenth, On as EffortSample, Pn as cellContrast, Rn as directional, Sn as CellEffort, Tn as EFFORT_MIN_RUNS, Tt as Outcome, U as ErrorKind, Un as effortSummary, Vn as effortOf, Vt as SandboxWarning, Wn as effortVerdict, Zt as TurnStatus, _n as JudgmentRecord, an as ArtifactRef, bn as StackSummary, cn as CheckTally, dn as FamilyLiftSummary, fn as FamilyRate, gn as HistoryEntry, hn as HistoryCell, jn as TOKENS_UNREPORTED, kn as EffortVerdict, ln as EvalEntry, mn as FileRef, on as AttemptRecord, pn as FamilyRecord, qn as formatSeconds, qt as TokenUsage, sn as CellRecord, un as FamilyLift, vn as Lift, wn as EFFORT_ALPHA, xn as UpstreamCount, yn as RunRecord, zn as effortChangeLabel } from "../types-DVvW66ci.js";
|
|
93
2
|
//#region src/view/analysis.d.ts
|
|
94
3
|
/** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
|
|
95
4
|
export declare const ANALYSIS_PROMPT_VERSION = "analysis-v11";
|
|
@@ -155,7 +64,7 @@ export type AnalysisValidation = {
|
|
|
155
64
|
export type AnalysisScope = {
|
|
156
65
|
kind: 'run';
|
|
157
66
|
head: string;
|
|
158
|
-
/** Resolved once, so a nearer ancestor recorded later doesn't change what this
|
|
67
|
+
/** Resolved once, so a nearer ancestor recorded later doesn't change what this analyzed. */
|
|
159
68
|
base: string | null;
|
|
160
69
|
/** An eval filter (substring of the eval id), or null for every eval. */
|
|
161
70
|
evals: string | null;
|
|
@@ -177,7 +86,7 @@ export type AnalysisDoc = {
|
|
|
177
86
|
promptVersion: string;
|
|
178
87
|
model: string;
|
|
179
88
|
resolvedModel: string | null;
|
|
180
|
-
route: 'direct' | 'gateway' | 'fake';
|
|
89
|
+
route: 'direct' | 'gateway' | 'openrouter' | 'vertex' | 'fake';
|
|
181
90
|
usage: {
|
|
182
91
|
inputTokens: number | null;
|
|
183
92
|
outputTokens: number | null;
|
|
@@ -193,7 +102,7 @@ export type AnalysisDoc = {
|
|
|
193
102
|
evalSlug: string;
|
|
194
103
|
attempt: string;
|
|
195
104
|
}>;
|
|
196
|
-
/** Each cited fact or evidence id,
|
|
105
|
+
/** Each cited fact or evidence id, summarized in one line. */
|
|
197
106
|
cited: Record<string, string>;
|
|
198
107
|
/**
|
|
199
108
|
* Why the analysis failed; its `analysis` is then empty. `reeval analyze` treats a cached one as
|
|
@@ -247,14 +156,27 @@ export declare function sortAnalyses(entries: AnalysisEntry[]): AnalysisEntry[];
|
|
|
247
156
|
export declare const isSuperseded: (doc: AnalysisDoc) => boolean;
|
|
248
157
|
//#endregion
|
|
249
158
|
//#region src/view/compare.d.ts
|
|
250
|
-
export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'drift' | 'added' | 'removed' | 'infra';
|
|
159
|
+
export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'gate' | 'drift' | 'added' | 'removed' | 'infra';
|
|
251
160
|
export type CompareRow = {
|
|
252
161
|
id: string;
|
|
253
162
|
status: CompareStatus;
|
|
254
163
|
base: Outcome | null;
|
|
255
164
|
head: Outcome | null;
|
|
256
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* For `drift`: what changed, e.g. `body changed`, `revision 2 → 3` or `harness config changed`. On
|
|
167
|
+
* any other row, `gate changed` when the cells' gate did.
|
|
168
|
+
*/
|
|
257
169
|
changes?: string[];
|
|
170
|
+
/**
|
|
171
|
+
* When the cells' gate changed but nothing else did: each side's gate, and base's outcome with
|
|
172
|
+
* its recorded attempts scored under head's gate. `status` compares that with head, so a flip
|
|
173
|
+
* the gate alone explains is `gate`, not `regressed` or `fixed`.
|
|
174
|
+
*/
|
|
175
|
+
gate?: {
|
|
176
|
+
base: string;
|
|
177
|
+
head: string;
|
|
178
|
+
rescored: Outcome;
|
|
179
|
+
};
|
|
258
180
|
/**
|
|
259
181
|
* When the runs ran different cells of the eval: the cells only head or only base has. Then
|
|
260
182
|
* `status`, `base` and `head` are over the cells both have.
|
|
@@ -266,10 +188,24 @@ export type CompareRow = {
|
|
|
266
188
|
/** Cells in both runs, with each side's lift (variants) and effort. */
|
|
267
189
|
perCell?: CompareCell[];
|
|
268
190
|
/**
|
|
269
|
-
* The sides whose `fail` is only attempts the run's interrupt
|
|
270
|
-
* reads as `
|
|
191
|
+
* The sides whose `fail` is only attempts the run's interrupt canceled, which `reeval show`
|
|
192
|
+
* reads as `canceled`. Such an eval is never `regressed` or `fixed`: it's `drift`.
|
|
271
193
|
*/
|
|
272
|
-
|
|
194
|
+
canceled?: Array<'base' | 'head'>;
|
|
195
|
+
/**
|
|
196
|
+
* `family`: a pooled `evaluate.each`'s gate, its `id` the family's. `base` and `head` are its
|
|
197
|
+
* pooled outcome (`fail` when a lift is below its `minLift`); its cases have rows of their own.
|
|
198
|
+
* For such a row, `cells` are the variants whose pooled lift only one run has.
|
|
199
|
+
*/
|
|
200
|
+
kind?: 'family';
|
|
201
|
+
/** On a family row: each variant's pooled lift on each side, null where that run has none. */
|
|
202
|
+
lifts?: CompareFamilyLift[];
|
|
203
|
+
};
|
|
204
|
+
export type CompareFamilyLift = {
|
|
205
|
+
key: string;
|
|
206
|
+
vs: string;
|
|
207
|
+
base: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
|
|
208
|
+
head: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
|
|
273
209
|
};
|
|
274
210
|
export type CompareCell = {
|
|
275
211
|
key: string;
|
|
@@ -303,17 +239,116 @@ export declare function stackChanges(base: StackSource, head: StackSource): Stac
|
|
|
303
239
|
* A cell `seen` doesn't have yet is new coverage, not a change.
|
|
304
240
|
*/
|
|
305
241
|
export declare function cellsChanged(seen: Map<string, string>, evalId: string, cells: ReadonlyArray<Pick<HistoryCell, 'key' | 'fingerprint'>>): boolean;
|
|
306
|
-
type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'cells'>;
|
|
307
|
-
type Comparable = Pick<HistoryEntry, 'stack'> & {
|
|
242
|
+
type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'source' | 'revision' | 'cells'>;
|
|
243
|
+
type Comparable = Pick<HistoryEntry, 'stack' | 'families'> & {
|
|
308
244
|
evals: ComparableEval[];
|
|
309
245
|
};
|
|
310
246
|
/** Which source files were added, removed or changed. */
|
|
311
247
|
export declare function sourceChanges(before: ComparableEval['sources'], after: ComparableEval['sources']): string[];
|
|
312
|
-
/**
|
|
248
|
+
/**
|
|
249
|
+
* Why two runs of an eval aren't comparable: what in its source differs (`body`, `options`,
|
|
250
|
+
* `case data`, `revision 2 → 3`; a record from before `source`, its files), then the fingerprint
|
|
251
|
+
* parts that differ. Each reason but the revision ends in ` changed`.
|
|
252
|
+
*/
|
|
313
253
|
export declare function whyChanged(base: ComparableEval, head: ComparableEval): string[];
|
|
314
254
|
/** An eval only head has. */
|
|
315
255
|
export declare const addedRow: (h: ComparableEval) => CompareRow;
|
|
256
|
+
/** The rows of a run with nothing to compare it with: every eval and gated family is new. */
|
|
257
|
+
export declare const newRows: (head: Comparable) => CompareRow[];
|
|
258
|
+
/** Each eval's row, then each family's that either run gated. */
|
|
316
259
|
export declare function compare(base: Comparable, head: Comparable): CompareRow[];
|
|
260
|
+
/** The key that tells a family's row from an eval's, whose ids can coincide. */
|
|
261
|
+
export declare const rowKey: (row: Pick<CompareRow, 'id' | 'kind'>) => string;
|
|
262
|
+
//#endregion
|
|
263
|
+
//#region src/view/debug.d.ts
|
|
264
|
+
/**
|
|
265
|
+
* An attempt's local debug data, as `reeval run` writes it to `.reeval/debug/<runId>/<eval>/<cell>/<n>/`
|
|
266
|
+
* and `reeval studio` serves it. Browser-safe. Never part of a record or the history folder.
|
|
267
|
+
*
|
|
268
|
+
* debug.log the `-v` trace, one line per event, always written
|
|
269
|
+
* requests.ndjson one line per egress gateway request, always written
|
|
270
|
+
* trace.ndjson raw payloads (`-vv`), written when the attempt fails or with `--trace`
|
|
271
|
+
*/
|
|
272
|
+
export declare const DEBUG_LOG = "debug.log";
|
|
273
|
+
export declare const REQUESTS_FILE = "requests.ndjson";
|
|
274
|
+
export declare const TRACE_FILE = "trace.ndjson";
|
|
275
|
+
/** What part of the attempt a line is about, in the order an attempt goes through them. */
|
|
276
|
+
export declare const DEBUG_PHASES: readonly ['plan', 'image', 'container', 'env', 'broker', 'harness', 'check', 'judge', 'gateway', 'teardown', 'docker'];
|
|
277
|
+
export type DebugPhase = (typeof DEBUG_PHASES)[number];
|
|
278
|
+
/** One debug line: level 1 is `-v`, level 2 is `-vv` only. */
|
|
279
|
+
export type DebugLine = {
|
|
280
|
+
at: number;
|
|
281
|
+
level: 1 | 2;
|
|
282
|
+
phase: DebugPhase;
|
|
283
|
+
text: string;
|
|
284
|
+
/** More lines under it, e.g. a docker command's output or a raw payload. */
|
|
285
|
+
detail?: string;
|
|
286
|
+
};
|
|
287
|
+
/** One raw trace entry: `ms` is since the attempt started. */
|
|
288
|
+
export type TraceEntry = {
|
|
289
|
+
at: string;
|
|
290
|
+
ms: number;
|
|
291
|
+
phase: DebugPhase;
|
|
292
|
+
kind: string;
|
|
293
|
+
data: unknown;
|
|
294
|
+
};
|
|
295
|
+
/** One request through the egress gateway, or one tunnel it passed through without reading. */
|
|
296
|
+
export type DebugRequest = {
|
|
297
|
+
at: string;
|
|
298
|
+
method: string;
|
|
299
|
+
url: string;
|
|
300
|
+
host: string;
|
|
301
|
+
port: number;
|
|
302
|
+
via: 'tunnel' | 'decrypted' | 'http';
|
|
303
|
+
allowed: boolean;
|
|
304
|
+
status?: number;
|
|
305
|
+
reason?: string;
|
|
306
|
+
sent: number;
|
|
307
|
+
received: number;
|
|
308
|
+
/** Brokered secrets and the harness credential swapped in, by name. */
|
|
309
|
+
substituted: string[];
|
|
310
|
+
ms: number;
|
|
311
|
+
/**
|
|
312
|
+
* On a request the gateway sent the harness's key on, or a model call whose response it reads,
|
|
313
|
+
* the reasoning settings its JSON body carried, as
|
|
314
|
+
* `<path>=<value>` joined by spaces: `reasoning_effort=high`,
|
|
315
|
+
* `generationConfig.thinkingConfig.thinkingLevel=LOW`.
|
|
316
|
+
*/
|
|
317
|
+
requestEffort?: string;
|
|
318
|
+
};
|
|
319
|
+
/** What the studio's Debug tab shows for one attempt. */
|
|
320
|
+
export type DebugView = {
|
|
321
|
+
lines: Array<{
|
|
322
|
+
at: string;
|
|
323
|
+
phase: string;
|
|
324
|
+
text: string;
|
|
325
|
+
}>;
|
|
326
|
+
requests: DebugRequest[];
|
|
327
|
+
/** Whether `trace.ndjson` was kept. */
|
|
328
|
+
trace: boolean;
|
|
329
|
+
};
|
|
330
|
+
/**
|
|
331
|
+
* The studio server's debug API (`reeval studio` only): an attempt's DebugView (null when it has no
|
|
332
|
+
* local data), and its raw trace.
|
|
333
|
+
*/
|
|
334
|
+
export declare const debugApiPath: (runId: string, attemptKey: string) => string;
|
|
335
|
+
export declare const traceApiPath: (runId: string, attemptKey: string) => string;
|
|
336
|
+
/** A snapshot from the run's directory in `.reeval/runs`, as a tar.gz (`reeval studio` only). */
|
|
337
|
+
export declare const snapshotApiPath: (runId: string, path: string) => string;
|
|
338
|
+
/** `12:50:01`, or `12:50:01.234` with milliseconds, in UTC like every reeval timestamp. */
|
|
339
|
+
export declare const debugTime: (at: number, ms: boolean) => string;
|
|
340
|
+
/** A line of debug.log: `2026-09-26T12:50:01.234Z plan text`. */
|
|
341
|
+
export declare const debugLogLine: (line: Pick<DebugLine, 'at' | 'phase' | 'text'>) => string;
|
|
342
|
+
/** debug.log's lines; a line that isn't one (a wrapped message) continues the one before. */
|
|
343
|
+
export declare function parseDebugLog(text: string): DebugView['lines'];
|
|
344
|
+
export declare function parseNdjson<T>(text: string): T[];
|
|
345
|
+
/** `512 B`, `1.2 KB`, `3.4 MB`. */
|
|
346
|
+
export declare function formatBytes(n: number): string;
|
|
347
|
+
/**
|
|
348
|
+
* The gateway line for a request: method, URL, status, allowed or blocked and why, bytes each way,
|
|
349
|
+
* time, and which secrets were swapped in (by name).
|
|
350
|
+
*/
|
|
351
|
+
export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
|
|
317
352
|
//#endregion
|
|
318
353
|
//#region src/view/family.d.ts
|
|
319
354
|
/** `pass@1 2/3 · 67% (95% 21…94)`, or `pass@1 67% over 3 cases (95% 30…100)` at several runs. */
|
|
@@ -342,7 +377,7 @@ export declare const familyCases: (family: Pick<FamilyRecord, 'id'>, evals: read
|
|
|
342
377
|
/** The derived header of one run: what listings, indexes and comparisons need without the record. */
|
|
343
378
|
export declare function runHeader(record: RunRecord, recorded: boolean): HistoryEntry;
|
|
344
379
|
//#endregion
|
|
345
|
-
//#region src/view/
|
|
380
|
+
//#region src/view/judgments.d.ts
|
|
346
381
|
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
347
382
|
export declare function judgeLabel(question: string): string;
|
|
348
383
|
/**
|
|
@@ -369,7 +404,7 @@ export type JudgeKeyRate = {
|
|
|
369
404
|
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
|
|
370
405
|
export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
|
|
371
406
|
/**
|
|
372
|
-
* The judge rates a cell's summary line adds to its checks: a
|
|
407
|
+
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
373
408
|
* of the cell's checks against its control is already there.
|
|
374
409
|
*/
|
|
375
410
|
export declare function summaryJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cell: Pick<CellRecord, 'key' | 'checks'>): JudgeKeyRate[];
|
|
@@ -383,20 +418,20 @@ export type CollapsedJudge = {
|
|
|
383
418
|
key: string;
|
|
384
419
|
};
|
|
385
420
|
/** Each key, or the lowest score plus how many judges there were. */
|
|
386
|
-
export declare function collapsedJudges(
|
|
421
|
+
export declare function collapsedJudges(judgments: readonly JudgmentRecord[]): {
|
|
387
422
|
shown: CollapsedJudge[];
|
|
388
423
|
extra: number;
|
|
389
424
|
};
|
|
390
425
|
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
391
|
-
export declare function collapsedJudgeLabel(
|
|
426
|
+
export declare function collapsedJudgeLabel(judgments: readonly JudgmentRecord[]): string | null;
|
|
392
427
|
/**
|
|
393
|
-
* Each
|
|
394
|
-
* failed judge call, which recorded no
|
|
395
|
-
* its
|
|
428
|
+
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
429
|
+
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
430
|
+
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
396
431
|
* reaches is named by its question.
|
|
397
432
|
*/
|
|
398
|
-
export declare function
|
|
399
|
-
|
|
433
|
+
export declare function labeledJudgments(a: Pick<AttemptRecord, 'judgements' | 'assertions'>): Array<{
|
|
434
|
+
judgment: JudgmentRecord;
|
|
400
435
|
label: string;
|
|
401
436
|
labelled: boolean;
|
|
402
437
|
check?: string;
|
|
@@ -457,7 +492,7 @@ export type LiveCounts = {
|
|
|
457
492
|
skipped: number;
|
|
458
493
|
running: number;
|
|
459
494
|
};
|
|
460
|
-
/** What the run is doing: running attempts,
|
|
495
|
+
/** What the run is doing: running attempts, canceling them, or assembling its record. */
|
|
461
496
|
export type LivePhase = 'running' | 'cancelling' | 'finishing';
|
|
462
497
|
export type LiveProgress = {
|
|
463
498
|
schemaVersion: 1;
|
|
@@ -741,11 +776,15 @@ export type CompareViewRow = {
|
|
|
741
776
|
/** See `CompareRow`: then pass@1 is over the cells both runs have too. */
|
|
742
777
|
cells?: CompareRow['cells'];
|
|
743
778
|
changes?: string[];
|
|
744
|
-
|
|
779
|
+
gate?: CompareRow['gate'];
|
|
780
|
+
/** Mean judge score per run; null without judgments or without the record. */
|
|
745
781
|
baseScore: number | null;
|
|
746
782
|
headScore: number | null;
|
|
747
783
|
/** Cells in both runs, with lift and effort on each side. */
|
|
748
784
|
perCell: CompareCell[];
|
|
785
|
+
/** See `CompareRow`: a pooled family's gate, with no slug, pass@1 or score of its own. */
|
|
786
|
+
kind?: 'family';
|
|
787
|
+
lifts?: CompareRow['lifts'];
|
|
749
788
|
};
|
|
750
789
|
export declare const COMPARE_ORDER: CompareStatus[];
|
|
751
790
|
/**
|
|
@@ -814,6 +853,12 @@ export type TranscriptItem = {
|
|
|
814
853
|
/** Parses `transcript.jsonl`, skipping lines that aren't JSON (a truncated tail, say). */
|
|
815
854
|
export declare function parseTranscript(text: string): TranscriptLine[];
|
|
816
855
|
export declare function transcriptItems(lines: TranscriptLine[]): TranscriptItem[];
|
|
856
|
+
export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
|
|
857
|
+
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
858
|
+
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
859
|
+
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
860
|
+
//#endregion
|
|
861
|
+
//#region src/view/patch.d.ts
|
|
817
862
|
export type PatchFile = {
|
|
818
863
|
path: string;
|
|
819
864
|
status: 'added' | 'deleted' | 'modified' | 'renamed' | 'binary';
|
|
@@ -821,12 +866,13 @@ export type PatchFile = {
|
|
|
821
866
|
removed: number;
|
|
822
867
|
text: string;
|
|
823
868
|
};
|
|
869
|
+
/**
|
|
870
|
+
* The path one file's part of a git patch changes: the new path, or the old one for a deletion.
|
|
871
|
+
* Read from its header lines, before the first `@@`, where git C-quotes an awkward name.
|
|
872
|
+
*/
|
|
873
|
+
export declare function patchPath(part: string): string | null;
|
|
824
874
|
/** Splits a git patch into one entry per file, with line counts. */
|
|
825
875
|
export declare function splitPatch(patch: string): PatchFile[];
|
|
826
|
-
export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
|
|
827
|
-
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
828
|
-
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
829
|
-
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
830
876
|
//#endregion
|
|
831
877
|
//#region src/view/paths.d.ts
|
|
832
878
|
/**
|
|
@@ -875,6 +921,57 @@ export declare function parseDataPath(path: string): DataTarget | null;
|
|
|
875
921
|
/** Artifacts are untrusted agent output: JSON stays JSON, PNGs are images, else plain text. */
|
|
876
922
|
export declare const artifactType: (name: string) => string;
|
|
877
923
|
//#endregion
|
|
924
|
+
//#region src/view/remarks.d.ts
|
|
925
|
+
/** `<history>/<run>/remarks.json`, beside the record. */
|
|
926
|
+
export declare const REMARKS_FILE = "remarks.json";
|
|
927
|
+
/** A run's remarks, `<history>/<run>/remarks.json`. Written only when the run raised one. */
|
|
928
|
+
export type RunRemarks = {
|
|
929
|
+
kind: 'reeval-remarks';
|
|
930
|
+
schemaVersion: 1;
|
|
931
|
+
runId: string;
|
|
932
|
+
createdAt: string;
|
|
933
|
+
/**
|
|
934
|
+
* Every remark this run raised, ranked, including those an earlier run printed first
|
|
935
|
+
* (`firstSeenRun`), which don't print again.
|
|
936
|
+
*/
|
|
937
|
+
remarks: Remark[];
|
|
938
|
+
};
|
|
939
|
+
/** Each remark kind's severity. */
|
|
940
|
+
export declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
941
|
+
severity: 'concern' | 'suggestion';
|
|
942
|
+
}>;
|
|
943
|
+
/** A remark about the eval's quality, not about the agent. */
|
|
944
|
+
export type Remark = {
|
|
945
|
+
/**
|
|
946
|
+
* sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
|
|
947
|
+
* per kind (and server) per eval version.
|
|
948
|
+
*/
|
|
949
|
+
id: string;
|
|
950
|
+
evalId: string;
|
|
951
|
+
kind: 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret' | 'snapshot-too-large';
|
|
952
|
+
/** Fixed by kind (`REMARK_KINDS`). */
|
|
953
|
+
severity: 'concern' | 'suggestion';
|
|
954
|
+
/** One plain sentence: what's wrong. */
|
|
955
|
+
text: string;
|
|
956
|
+
/** Facts from this run (or history facts) showing it mattered now. */
|
|
957
|
+
evidence: string[];
|
|
958
|
+
/** An edit at a source location (`source:<file>#L<n>`) in the eval. */
|
|
959
|
+
fix: {
|
|
960
|
+
at: string;
|
|
961
|
+
edit: string;
|
|
962
|
+
};
|
|
963
|
+
/**
|
|
964
|
+
* The first run that printed this `id`, or this run when none has: it prints only in that run,
|
|
965
|
+
* except the kinds about the environment, which print in every run they happen.
|
|
966
|
+
*/
|
|
967
|
+
firstSeenRun: string;
|
|
968
|
+
/**
|
|
969
|
+
* Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
|
|
970
|
+
* when the cap or a run without `-v` left it out, so a later run prints it.
|
|
971
|
+
*/
|
|
972
|
+
printed?: boolean;
|
|
973
|
+
};
|
|
974
|
+
//#endregion
|
|
878
975
|
//#region src/view/record.d.ts
|
|
879
976
|
/** A file from a newer reeval, whose schemaVersion this reeval doesn't know. */
|
|
880
977
|
export declare class NewerSchemaError extends Error {
|
|
@@ -893,8 +990,338 @@ export declare function readableRecord(record: RunRecord, what?: string): RunRec
|
|
|
893
990
|
export declare const readableTranscriptLine: (line: TranscriptLine) => TranscriptLine;
|
|
894
991
|
/** An analysis document as every reader sees it; one from a newer reeval throws. */
|
|
895
992
|
export declare const readableAnalysis: (doc: AnalysisDoc, what?: string) => AnalysisDoc;
|
|
896
|
-
/** A run's
|
|
897
|
-
export declare const
|
|
993
|
+
/** A run's remarks as every reader sees them; a file from a newer reeval throws. */
|
|
994
|
+
export declare const readableRemarks: (doc: RunRemarks, what?: string) => RunRemarks;
|
|
995
|
+
//#endregion
|
|
996
|
+
//#region src/stats.d.ts
|
|
997
|
+
/**
|
|
998
|
+
* Counted attempts a side before a pass-rate lift can be called: below it, every view says
|
|
999
|
+
* `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
|
|
1000
|
+
* `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
|
|
1001
|
+
* makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
|
|
1002
|
+
* the time when nothing changed.
|
|
1003
|
+
*/
|
|
1004
|
+
export declare const EFFECT_MIN_COUNTED = 4;
|
|
1005
|
+
//#endregion
|
|
1006
|
+
//#region src/view/suite.d.ts
|
|
1007
|
+
/**
|
|
1008
|
+
* A variant against its baseline: `better` or `worse` when the stored 95% interval on the
|
|
1009
|
+
* difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
|
|
1010
|
+
* including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
|
|
1011
|
+
*/
|
|
1012
|
+
export type Effect = 'better' | 'worse' | 'same' | 'unclear';
|
|
1013
|
+
/**
|
|
1014
|
+
* What an eval's cells compare: `change` when it has variants (or several environments),
|
|
1015
|
+
* `harness` when it has several harnesses, `both`, or `none`.
|
|
1016
|
+
*/
|
|
1017
|
+
export type Across = 'none' | 'change' | 'harness' | 'both';
|
|
1018
|
+
/**
|
|
1019
|
+
* One finished run's facts. Every node has a `path`, a stable id that remarks cite it by.
|
|
1020
|
+
*
|
|
1021
|
+
* A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
|
|
1022
|
+
*
|
|
1023
|
+
* - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
|
|
1024
|
+
* (`evals/a.eval.ts › fixes $id`);
|
|
1025
|
+
* - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
|
|
1026
|
+
* - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
|
|
1027
|
+
* - `<cell>/checks/<label>`: a check, by its assertion label;
|
|
1028
|
+
* - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
|
|
1029
|
+
*
|
|
1030
|
+
* A case segment never has a raw `+` and a cell segment always has two, so the two can't be
|
|
1031
|
+
* confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
|
|
1032
|
+
* fact in every run that has it.
|
|
1033
|
+
*/
|
|
1034
|
+
export type SuiteResult = {
|
|
1035
|
+
runId: string;
|
|
1036
|
+
status: RunRecord['status'];
|
|
1037
|
+
/**
|
|
1038
|
+
* The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
|
|
1039
|
+
* an attempt, and `skipped` on an interrupted run.
|
|
1040
|
+
*/
|
|
1041
|
+
outcome: Outcome;
|
|
1042
|
+
/**
|
|
1043
|
+
* Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
|
|
1044
|
+
* awaited, a worker that died. Any one fails the run.
|
|
1045
|
+
*/
|
|
1046
|
+
errors: string[];
|
|
1047
|
+
/**
|
|
1048
|
+
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
1049
|
+
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
|
|
1050
|
+
* `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
|
|
1051
|
+
*/
|
|
1052
|
+
tally: SuiteTally;
|
|
1053
|
+
/** Sorted by path. */
|
|
1054
|
+
files: ResultFile[];
|
|
1055
|
+
};
|
|
1056
|
+
export type SuiteTally = {
|
|
1057
|
+
errors: number;
|
|
1058
|
+
pass: number;
|
|
1059
|
+
notGated: number;
|
|
1060
|
+
fail: number;
|
|
1061
|
+
infraError: number;
|
|
1062
|
+
canceled: number;
|
|
1063
|
+
skipped: number;
|
|
1064
|
+
};
|
|
1065
|
+
/**
|
|
1066
|
+
* An eval's or a family case's outcome: the record's, except `canceled` where its only failed
|
|
1067
|
+
* gated attempts are ones the run's interrupt canceled, since the agent didn't fail those.
|
|
1068
|
+
*/
|
|
1069
|
+
export type ResultOutcome = Outcome | 'canceled';
|
|
1070
|
+
export type ResultFile = {
|
|
1071
|
+
path: string;
|
|
1072
|
+
evals: EvalResult[];
|
|
1073
|
+
};
|
|
1074
|
+
export type EvalResult = {
|
|
1075
|
+
path: string;
|
|
1076
|
+
/** The eval's id, or the family's. */
|
|
1077
|
+
id: string;
|
|
1078
|
+
group: string[];
|
|
1079
|
+
/** The eval's name; on a family, its template. */
|
|
1080
|
+
name: string;
|
|
1081
|
+
/** A family's name template (`fixes $id`); null on a single eval. */
|
|
1082
|
+
template: string | null;
|
|
1083
|
+
/** Source order in the file. */
|
|
1084
|
+
order: number;
|
|
1085
|
+
across: Across;
|
|
1086
|
+
/**
|
|
1087
|
+
* A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
|
|
1088
|
+
* unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
|
|
1089
|
+
*/
|
|
1090
|
+
family: boolean;
|
|
1091
|
+
/** A placeholder (`evaluate.todo`) that never ran. */
|
|
1092
|
+
todo: boolean;
|
|
1093
|
+
/** On a family: `fail` when a case failed or its pooled gate did. */
|
|
1094
|
+
outcome: ResultOutcome;
|
|
1095
|
+
cells: CellResult[] | null;
|
|
1096
|
+
cases: CaseResult[] | null;
|
|
1097
|
+
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
1098
|
+
pooled: PooledArm[] | null;
|
|
1099
|
+
/**
|
|
1100
|
+
* Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
|
|
1101
|
+
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
1102
|
+
*/
|
|
1103
|
+
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
1104
|
+
};
|
|
1105
|
+
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
1106
|
+
export type QuestionResult = {
|
|
1107
|
+
kind: 'pairs';
|
|
1108
|
+
question: string;
|
|
1109
|
+
/** Pairs voted, each counted once per order it was asked in. */
|
|
1110
|
+
votes: number;
|
|
1111
|
+
rates: Array<{
|
|
1112
|
+
cut: 'within' | 'across';
|
|
1113
|
+
name: string;
|
|
1114
|
+
wins: number;
|
|
1115
|
+
n: number;
|
|
1116
|
+
}>;
|
|
1117
|
+
/** Why some or all of its votes are missing: the model call failed. */
|
|
1118
|
+
error: string | null;
|
|
1119
|
+
};
|
|
1120
|
+
/** A question asked of each attempt on its own: what each one answered. Never gates. */
|
|
1121
|
+
export type AttemptQuestionResult = {
|
|
1122
|
+
kind: 'attempts';
|
|
1123
|
+
question: string;
|
|
1124
|
+
/** Attempts it was asked of, failed calls included. */
|
|
1125
|
+
asked: number;
|
|
1126
|
+
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
1127
|
+
choices: Array<{
|
|
1128
|
+
choice: string;
|
|
1129
|
+
n: number;
|
|
1130
|
+
}> | null;
|
|
1131
|
+
/** Each attempt's answer, in attempt order; failed calls left out. */
|
|
1132
|
+
answers: Array<{
|
|
1133
|
+
attempt: string;
|
|
1134
|
+
choice: string | null;
|
|
1135
|
+
text: string;
|
|
1136
|
+
}>;
|
|
1137
|
+
/** Calls that failed, and the first one's error. */
|
|
1138
|
+
failed: number;
|
|
1139
|
+
error: string | null;
|
|
1140
|
+
};
|
|
1141
|
+
export type CaseResult = {
|
|
1142
|
+
path: string;
|
|
1143
|
+
/** The case's own eval id. */
|
|
1144
|
+
id: string;
|
|
1145
|
+
/** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
|
|
1146
|
+
label: string;
|
|
1147
|
+
name: string;
|
|
1148
|
+
/** Its place in the case list. */
|
|
1149
|
+
order: number;
|
|
1150
|
+
outcome: ResultOutcome;
|
|
1151
|
+
cells: CellResult[];
|
|
1152
|
+
};
|
|
1153
|
+
export type CellResult = {
|
|
1154
|
+
path: string;
|
|
1155
|
+
key: string;
|
|
1156
|
+
harness: string;
|
|
1157
|
+
/** `-` when no environment is set. */
|
|
1158
|
+
environment: string;
|
|
1159
|
+
sandbox: string;
|
|
1160
|
+
/** The variant's name; null on a control. */
|
|
1161
|
+
variant: string | null;
|
|
1162
|
+
/** The control of an eval that has variants: the arm the variants are measured against. */
|
|
1163
|
+
baseline: boolean;
|
|
1164
|
+
/** On a variant, its baseline cell's key; else null. */
|
|
1165
|
+
vs: string | null;
|
|
1166
|
+
/** The cell's place in the eval's cells. */
|
|
1167
|
+
order: number;
|
|
1168
|
+
/** Its failures fail the eval (its gate isn't `'none'`). */
|
|
1169
|
+
gated: boolean;
|
|
1170
|
+
outcome: Outcome;
|
|
1171
|
+
/** Every attempt: `passed + failed + infraError + skipped`. */
|
|
1172
|
+
attempts: number;
|
|
1173
|
+
passed: number;
|
|
1174
|
+
failed: number;
|
|
1175
|
+
/** Infrastructure errors, which are neither passes nor fails. */
|
|
1176
|
+
infraError: number;
|
|
1177
|
+
skipped: number;
|
|
1178
|
+
/** Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed. */
|
|
1179
|
+
bestOf?: number;
|
|
1180
|
+
/** Of `skipped`, the attempts a best-of cell's majority left unrun. */
|
|
1181
|
+
notNeeded?: number;
|
|
1182
|
+
fingerprint: string;
|
|
1183
|
+
/** The skill sources and files the cell's environment placed, as written. */
|
|
1184
|
+
context: string[];
|
|
1185
|
+
/**
|
|
1186
|
+
* Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
|
|
1187
|
+
* attempt (every one an infra error or skipped) lists each as `not run`.
|
|
1188
|
+
*/
|
|
1189
|
+
checks: CheckResult[];
|
|
1190
|
+
/** On a variant, against its baseline; null on a control. */
|
|
1191
|
+
effect: Effect | null;
|
|
1192
|
+
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1193
|
+
interval: Interval | null;
|
|
1194
|
+
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1195
|
+
minLift: number | null;
|
|
1196
|
+
/** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
|
|
1197
|
+
effort: EffortCounters | null;
|
|
1198
|
+
/** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
|
|
1199
|
+
effortDiffers: boolean;
|
|
1200
|
+
/** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
|
|
1201
|
+
errors: CellError[];
|
|
1202
|
+
/**
|
|
1203
|
+
* Where each failed attempt's final snapshot is, from the project's root
|
|
1204
|
+
* (`.reeval/runs/<run>/…/snapshots/final`); absent when none has one.
|
|
1205
|
+
*/
|
|
1206
|
+
snapshots?: string[];
|
|
1207
|
+
/** Web searches and fetches per counted attempt; null where the harness can't report them. */
|
|
1208
|
+
searches: Array<number | null>;
|
|
1209
|
+
fetches: Array<number | null>;
|
|
1210
|
+
/**
|
|
1211
|
+
* The upstream providers a gateway (OpenRouter) said served the cell's agent calls, with how
|
|
1212
|
+
* many, summed over its attempts in first-seen order; absent when it named none.
|
|
1213
|
+
*/
|
|
1214
|
+
upstreams?: UpstreamCount[];
|
|
1215
|
+
/** The same for the cell's judge calls; absent when the gateway named none. */
|
|
1216
|
+
judgeUpstreams?: UpstreamCount[];
|
|
1217
|
+
};
|
|
1218
|
+
/** A cell's attempts the run's interrupt canceled: recorded as fails, but not the agent's. */
|
|
1219
|
+
export declare const canceledOf: (cell: CellResult) => number;
|
|
1220
|
+
/** A cell's counted tries: its passes and fails, less the ones the interrupt canceled. */
|
|
1221
|
+
export declare const countedOf: (cell: CellResult) => number;
|
|
1222
|
+
export type CheckResult = {
|
|
1223
|
+
path: string;
|
|
1224
|
+
label: string;
|
|
1225
|
+
/** First-seen order across the eval's attempts. */
|
|
1226
|
+
order: number;
|
|
1227
|
+
outcome: CheckOutcome;
|
|
1228
|
+
/**
|
|
1229
|
+
* On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
|
|
1230
|
+
* null on a control or without one.
|
|
1231
|
+
*/
|
|
1232
|
+
effect: Effect | null;
|
|
1233
|
+
/** Set when the check is a judge verdict. */
|
|
1234
|
+
judge: JudgeResult | null;
|
|
1235
|
+
/**
|
|
1236
|
+
* A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
|
|
1237
|
+
* it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
|
|
1238
|
+
* when none did.
|
|
1239
|
+
*/
|
|
1240
|
+
retried?: string;
|
|
1241
|
+
};
|
|
1242
|
+
/**
|
|
1243
|
+
* `not run`: no counted attempt reached the check, because an `expect.require` before it failed or
|
|
1244
|
+
* none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
|
|
1245
|
+
* reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
|
|
1246
|
+
* first failure's message and `path:line`.
|
|
1247
|
+
*/
|
|
1248
|
+
export type CheckOutcome = {
|
|
1249
|
+
status: 'not run';
|
|
1250
|
+
} | {
|
|
1251
|
+
status: 'ran';
|
|
1252
|
+
passed: number;
|
|
1253
|
+
failed: number;
|
|
1254
|
+
errored: number;
|
|
1255
|
+
notRun: number;
|
|
1256
|
+
assertion: string | null;
|
|
1257
|
+
at: string | null;
|
|
1258
|
+
};
|
|
1259
|
+
/**
|
|
1260
|
+
* One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
|
|
1261
|
+
* an error), its message's first line, and how often.
|
|
1262
|
+
*/
|
|
1263
|
+
export type CellError = {
|
|
1264
|
+
kind: ErrorKind | 'turn';
|
|
1265
|
+
message: string;
|
|
1266
|
+
attempts: number;
|
|
1267
|
+
};
|
|
1268
|
+
export type JudgeResult = {
|
|
1269
|
+
question: string;
|
|
1270
|
+
threshold: number;
|
|
1271
|
+
/** One per counted attempt that judged it. */
|
|
1272
|
+
scores: number[];
|
|
1273
|
+
};
|
|
1274
|
+
export type Interval = {
|
|
1275
|
+
low: number;
|
|
1276
|
+
high: number;
|
|
1277
|
+
};
|
|
1278
|
+
/** A null counter is one the harness didn't record; a recorded zero stays zero. */
|
|
1279
|
+
export type EffortCounters = {
|
|
1280
|
+
tools: number | null;
|
|
1281
|
+
agentMs: number | null;
|
|
1282
|
+
tokens: number | null;
|
|
1283
|
+
};
|
|
1284
|
+
export type PooledArm = {
|
|
1285
|
+
path: string;
|
|
1286
|
+
/** The variant cell's key, the same in every case. */
|
|
1287
|
+
key: string;
|
|
1288
|
+
vs: string;
|
|
1289
|
+
effect: Effect;
|
|
1290
|
+
/** Cases with counted attempts on both sides. */
|
|
1291
|
+
pairs: number;
|
|
1292
|
+
interval: Interval;
|
|
1293
|
+
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
1294
|
+
minLift: number | null;
|
|
1295
|
+
outcome: 'pass' | 'fail';
|
|
1296
|
+
};
|
|
1297
|
+
type Side = {
|
|
1298
|
+
pass: number;
|
|
1299
|
+
counted: number;
|
|
1300
|
+
};
|
|
1301
|
+
/**
|
|
1302
|
+
* The effect of `variant` against `control`, from the interval on their difference. A side
|
|
1303
|
+
* with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
|
|
1304
|
+
*/
|
|
1305
|
+
export declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
|
|
1306
|
+
/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
|
|
1307
|
+
export declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
|
|
1308
|
+
/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
|
|
1309
|
+
export declare const factSegment: (s: string) => string;
|
|
1310
|
+
export declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
|
|
1311
|
+
/**
|
|
1312
|
+
* An error's one line: its first line with any letter or digit in it, joined with the next one
|
|
1313
|
+
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1314
|
+
*/
|
|
1315
|
+
export declare function messageLine(text: string): string;
|
|
1316
|
+
/** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
|
|
1317
|
+
export declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
|
|
1318
|
+
/**
|
|
1319
|
+
* A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
|
|
1320
|
+
* gating it, so it reads `not gated`, never `pass`.
|
|
1321
|
+
*/
|
|
1322
|
+
export declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
|
|
1323
|
+
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1324
|
+
export declare function suiteResult(record: RunRecord): SuiteResult;
|
|
898
1325
|
//#endregion
|
|
899
|
-
export {
|
|
1326
|
+
export { type ArtifactRef, type AttemptRecord, CellEffort, type CellRecord, DIRECTIONAL_MIN, EFFORT_ALPHA, EFFORT_MIN_RUNS, EffortContrast, EffortMetric, EffortSample, EffortVerdict, type EvalEntry, type FamilyLift, type FamilyRate, type FamilyRecord, type FileRef, type HistoryCell, type HistoryEntry, type JudgmentRecord, type Outcome, type RunRecord, type SandboxWarning, Spread, type StackSummary, TOKENS_UNREPORTED, type TranscriptEvent, VERDICT_TEXT, bothUnderATenth, cellContrast, cellEffort, cellSamples, contrastEffort, directional, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, formatCount, formatRatio, formatSeconds, tokenValues };
|
|
900
1327
|
//# sourceMappingURL=index.d.ts.map
|