reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,3 +1,63 @@
|
|
|
1
|
+
//#region src/declaration.d.ts
|
|
2
|
+
/** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
|
|
3
|
+
type DeclaredMode = 'run' | 'skip' | 'only';
|
|
4
|
+
/** A `group`: options its evals and groups inherit, and its children in declaration order. */
|
|
5
|
+
type DeclaredGroup = {
|
|
6
|
+
type: 'group';
|
|
7
|
+
/** The group's name, as passed to `group`. */
|
|
8
|
+
name: string;
|
|
9
|
+
mode: DeclaredMode;
|
|
10
|
+
/** The group's own options, before the cascade. */
|
|
11
|
+
options: EvalOptionsData;
|
|
12
|
+
children: DeclaredNode[];
|
|
13
|
+
};
|
|
14
|
+
/** An `evaluate`: one eval, planned into a test per attempt. */
|
|
15
|
+
type DeclaredEval = {
|
|
16
|
+
type: 'eval';
|
|
17
|
+
/** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
|
|
18
|
+
name: string;
|
|
19
|
+
mode: DeclaredMode;
|
|
20
|
+
/** The eval's own options, before the cascade. */
|
|
21
|
+
options: EvalOptionsData;
|
|
22
|
+
/** Names the eval's body to whoever runs it. */
|
|
23
|
+
body: string;
|
|
24
|
+
source: DeclaredSource;
|
|
25
|
+
/** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
|
|
26
|
+
family?: DeclaredFamily;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
|
|
30
|
+
* text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
|
|
31
|
+
* whitespace, non-ASCII characters as themselves).
|
|
32
|
+
*/
|
|
33
|
+
type DeclaredSource = {
|
|
34
|
+
/** sha256 of the body function's source text. */
|
|
35
|
+
body: string;
|
|
36
|
+
/** sha256 of the case's data; set on each case of an `evaluate.each`. */
|
|
37
|
+
case?: string;
|
|
38
|
+
};
|
|
39
|
+
/** Where a case of a pooled `evaluate.each` sits in its list. */
|
|
40
|
+
type DeclaredFamily = {
|
|
41
|
+
/** The name template passed to `evaluate.each`. */
|
|
42
|
+
name: string;
|
|
43
|
+
/** The case's index, from 0. */
|
|
44
|
+
case: number;
|
|
45
|
+
/** How many cases the list has. */
|
|
46
|
+
cases: number;
|
|
47
|
+
};
|
|
48
|
+
/** `evaluate.todo` or `group.todo`. */
|
|
49
|
+
type DeclaredTodo = {
|
|
50
|
+
type: 'todo';
|
|
51
|
+
/** The name passed to `.todo`. */
|
|
52
|
+
name: string;
|
|
53
|
+
};
|
|
54
|
+
/** One node of the tree: a group, an eval or a todo. */
|
|
55
|
+
type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
|
|
56
|
+
/** An eval file's top-level groups, evals and todos, in declaration order. */
|
|
57
|
+
type DeclarationTree = {
|
|
58
|
+
children: DeclaredNode[];
|
|
59
|
+
};
|
|
60
|
+
//#endregion
|
|
1
61
|
//#region src/view/effort.d.ts
|
|
2
62
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
3
63
|
declare const EFFORT_ALPHA = 0.05;
|
|
@@ -127,7 +187,7 @@ type FileRef = {
|
|
|
127
187
|
sha256: string;
|
|
128
188
|
bytes: number;
|
|
129
189
|
};
|
|
130
|
-
/** A generic attempt file from `c.capture()
|
|
190
|
+
/** A generic attempt file from `c.capture()`. */
|
|
131
191
|
type ArtifactRef = FileRef & {
|
|
132
192
|
name: string;
|
|
133
193
|
mediaType: 'image/png';
|
|
@@ -157,15 +217,18 @@ type GitInfo = {
|
|
|
157
217
|
*/
|
|
158
218
|
sha: string;
|
|
159
219
|
branch: string | null;
|
|
160
|
-
/** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
|
|
220
|
+
/** Uncommitted changes, outside the `.reeval/` directory and the history folder, when the run started. */
|
|
161
221
|
dirty: boolean;
|
|
162
222
|
/** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
|
|
163
223
|
diffHash: string | null;
|
|
164
|
-
/**
|
|
224
|
+
/**
|
|
225
|
+
* Those changes are stored as the `worktree.patch` file in the artifact directory. Applying it
|
|
226
|
+
* to `sha` reproduces the workspace.
|
|
227
|
+
*/
|
|
165
228
|
worktreePatch: FileRef | null;
|
|
166
229
|
/**
|
|
167
|
-
*
|
|
168
|
-
* a secret (then where, and which rule, never the value).
|
|
230
|
+
* This explains why a dirty run has no `worktree.patch` file: it was too large, `artifacts` was
|
|
231
|
+
* `'none'`, or it looked like it held a secret (then where, and which rule, never the value).
|
|
169
232
|
*/
|
|
170
233
|
worktreePatchNote: string | null;
|
|
171
234
|
commitTime: string;
|
|
@@ -192,7 +255,7 @@ type JudgeRetry = {
|
|
|
192
255
|
error: string;
|
|
193
256
|
delayMs: number;
|
|
194
257
|
};
|
|
195
|
-
type
|
|
258
|
+
type JudgmentRecord = {
|
|
196
259
|
question: string;
|
|
197
260
|
verdict: 'pass' | 'fail';
|
|
198
261
|
score: number;
|
|
@@ -202,9 +265,15 @@ type JudgementRecord = {
|
|
|
202
265
|
resolvedModel?: string;
|
|
203
266
|
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
204
267
|
gateway?: string;
|
|
268
|
+
/** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
|
|
269
|
+
upstream?: string;
|
|
270
|
+
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
271
|
+
costUsd?: number;
|
|
205
272
|
/**
|
|
206
|
-
* `<index>:<sha256(question) prefix>`: the same question at the same place in the
|
|
207
|
-
* verdict can be followed across runs (and an edited question starts a new series).
|
|
273
|
+
* `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
|
|
274
|
+
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
275
|
+
* is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
|
|
276
|
+
* whichever finishes first.
|
|
208
277
|
*/
|
|
209
278
|
key: string;
|
|
210
279
|
/** Transient provider errors retried before this verdict, with the backoff before each retry. */
|
|
@@ -213,33 +282,44 @@ type JudgementRecord = {
|
|
|
213
282
|
artifacts?: JudgeArtifactRef[];
|
|
214
283
|
};
|
|
215
284
|
type AttemptFiles = {
|
|
216
|
-
/** Each environment setup step's command, exit code, duration and output. */
|
|
285
|
+
/** Each environment setup step's command, exit code, duration, and output. */
|
|
217
286
|
setupLog?: FileRef;
|
|
218
287
|
transcript?: FileRef;
|
|
219
|
-
/**
|
|
288
|
+
/** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
|
|
220
289
|
rawTranscript?: FileRef;
|
|
221
290
|
diff?: FileRef;
|
|
222
291
|
judgeInput?: FileRef;
|
|
223
|
-
/** PNGs from `c.capture()`, under `artifacts/<name>.png
|
|
292
|
+
/** PNGs from `c.capture()`, under the `artifacts/<name>.png` path. */
|
|
224
293
|
artifacts?: ArtifactRef[];
|
|
225
294
|
};
|
|
226
295
|
type ContextFile = {
|
|
227
296
|
path: string;
|
|
228
297
|
sha256: string;
|
|
229
298
|
};
|
|
299
|
+
/** An upstream provider behind a gateway, and how many of the attempt's calls it served. */
|
|
300
|
+
type UpstreamCount = {
|
|
301
|
+
provider: string;
|
|
302
|
+
calls: number;
|
|
303
|
+
};
|
|
230
304
|
type AttemptRecord = {
|
|
231
305
|
/** `<eval key> # <cell key>#<n>`: unique in the run. */
|
|
232
306
|
key: string;
|
|
233
|
-
/** `<cell key>#<n>`: unique in its
|
|
307
|
+
/** `<cell key>#<n>`: unique in its evaluation. */
|
|
234
308
|
id: string;
|
|
235
309
|
/** The cell's key; see `EvalEntry.cells`. */
|
|
236
310
|
cell: string;
|
|
237
311
|
/** Run number within the cell, from 1. */
|
|
238
312
|
n: number;
|
|
313
|
+
/**
|
|
314
|
+
* In a `runs: { bestOf }` cell: `added` when it ran past the majority's first attempts (the
|
|
315
|
+
* earlier ones split, or an infra error left them short), `skipped` when it didn't run because
|
|
316
|
+
* the earlier ones already had a majority. Unset on the attempts that always run.
|
|
317
|
+
*/
|
|
318
|
+
bestOf?: 'added' | 'skipped';
|
|
239
319
|
/** Start order across the run, from 1; null for attempts that never started. */
|
|
240
320
|
seq: number | null;
|
|
241
321
|
startedAt: string | null;
|
|
242
|
-
/** The image the container started from, in `stack.images`; null
|
|
322
|
+
/** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
|
|
243
323
|
imageId: string | null;
|
|
244
324
|
/** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
|
|
245
325
|
env: {
|
|
@@ -247,16 +327,17 @@ type AttemptRecord = {
|
|
|
247
327
|
secrets: string[];
|
|
248
328
|
brokered: string[];
|
|
249
329
|
};
|
|
250
|
-
/** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
|
|
330
|
+
/** Skills, rules, and MCP config visible to the agent in its sandbox, by content hash. */
|
|
251
331
|
context: ContextFile[];
|
|
252
332
|
/** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
|
|
253
333
|
egress: EgressRecord | null;
|
|
254
334
|
/**
|
|
255
|
-
* The models the harness's
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
335
|
+
* The models served by the harness's API calls, as observed by the egress gateway. Values may
|
|
336
|
+
* come from requests or responses and include reported effort; null when no model was named or
|
|
337
|
+
* recorded. The gateway reads the request URL for Antigravity, Goose, pi, and OpenCode on a
|
|
338
|
+
* `google/` model, so a request for `<model>-low` records `<model>`, and the API response for
|
|
339
|
+
* Codex, Grok Build, and OpenCode on another provider. Null for Cursor and Claude Code, whose
|
|
340
|
+
* API requests name no model.
|
|
260
341
|
*/
|
|
261
342
|
servedModels: string[] | null;
|
|
262
343
|
/**
|
|
@@ -265,13 +346,24 @@ type AttemptRecord = {
|
|
|
265
346
|
* gateway's, so never part of `servedModels`. Absent when there's none.
|
|
266
347
|
*/
|
|
267
348
|
askedModels?: string[];
|
|
349
|
+
/**
|
|
350
|
+
* The upstream providers a gateway said served the harness's own API calls (OpenRouter's
|
|
351
|
+
* `provider`), each with how many it served, in the order first seen; more than one means the
|
|
352
|
+
* gateway spread or failed the calls over between them. Absent when the gateway names none.
|
|
353
|
+
*/
|
|
354
|
+
upstreams?: UpstreamCount[];
|
|
355
|
+
/**
|
|
356
|
+
* What a gateway said the harness's own API calls cost, in US dollars, summed (OpenRouter's
|
|
357
|
+
* `usage.cost`); absent when it names none.
|
|
358
|
+
*/
|
|
359
|
+
reportedCostUsd?: number;
|
|
268
360
|
/**
|
|
269
361
|
* The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
|
|
270
362
|
* null when the factory left it unset. Fast is not this.
|
|
271
363
|
*/
|
|
272
364
|
effort: string | null;
|
|
273
365
|
/**
|
|
274
|
-
* The reasoning effort the harness's API said it ran at, from `servedModels`
|
|
366
|
+
* The reasoning effort the harness's API said it ran at, from `servedModels` or
|
|
275
367
|
* each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
|
|
276
368
|
* labels name none. Fast (`[fast=true]`) is not this.
|
|
277
369
|
*/
|
|
@@ -280,7 +372,8 @@ type AttemptRecord = {
|
|
|
280
372
|
error: AttemptError | null;
|
|
281
373
|
durationMs: number;
|
|
282
374
|
/**
|
|
283
|
-
* `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl
|
|
375
|
+
* `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
|
|
376
|
+
* file.
|
|
284
377
|
* `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
|
|
285
378
|
*/
|
|
286
379
|
turns: Array<{
|
|
@@ -303,8 +396,8 @@ type AttemptRecord = {
|
|
|
303
396
|
/** Tokens across the attempt's turns; null when the harness reported none. */
|
|
304
397
|
usage: TokenUsage | null;
|
|
305
398
|
/**
|
|
306
|
-
* Tokens the capture vision loop used. Infrastructure only: never
|
|
307
|
-
*
|
|
399
|
+
* Tokens the capture vision loop used. Infrastructure only: never included
|
|
400
|
+
* in `usage`, effort, compare, or lift.
|
|
308
401
|
*/
|
|
309
402
|
captureUse?: {
|
|
310
403
|
model: string;
|
|
@@ -312,7 +405,7 @@ type AttemptRecord = {
|
|
|
312
405
|
usage: TokenUsage | null;
|
|
313
406
|
};
|
|
314
407
|
timings: PhaseTimings;
|
|
315
|
-
/** Counts from the transcript, so
|
|
408
|
+
/** Counts from the transcript, so the attempt's activity is available without reading the transcript. `webSearches` includes refused searches. */
|
|
316
409
|
activity: {
|
|
317
410
|
tools: number;
|
|
318
411
|
fileEdits: number;
|
|
@@ -325,9 +418,34 @@ type AttemptRecord = {
|
|
|
325
418
|
/** Longest tool `durationMs` in the transcript; null when none reported a duration. */
|
|
326
419
|
maxToolMs: number | null;
|
|
327
420
|
};
|
|
328
|
-
judgements:
|
|
421
|
+
judgements: JudgmentRecord[];
|
|
329
422
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
330
423
|
startRetries?: StartRetry[];
|
|
424
|
+
/**
|
|
425
|
+
* Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
|
|
426
|
+
* without `hosts` in their environment, so they hold its real value while the agent works and
|
|
427
|
+
* the agent, as the same user, can read it from them. Found by the secret's variable name, so a
|
|
428
|
+
* copy under another name or in a command's arguments isn't. At most 20, each
|
|
429
|
+
* `{ command, secrets }`: its command line, masked and cut at 200 characters, and those secrets'
|
|
430
|
+
* names. Absent when there were none.
|
|
431
|
+
*/
|
|
432
|
+
setupProcesses?: SetupProcess[];
|
|
433
|
+
/**
|
|
434
|
+
* Copies of the attempt's workspace in its directory in `.reeval/runs` on the machine that ran
|
|
435
|
+
* it (never in the history folder), in the order taken: `turn-<n>` after each `c.send` with
|
|
436
|
+
* `snapshot: 'turns'`, then `final` after the checks. Each has its form, path and size, its
|
|
437
|
+
* file count, the directories left out by name, the files whose secret-looking values were
|
|
438
|
+
* masked and the binary ones left out for holding one, or why it wasn't taken. Absent with
|
|
439
|
+
* `snapshot: false`, and when the sandbox was gone first (a timeout or a cancel).
|
|
440
|
+
*/
|
|
441
|
+
snapshots?: Snapshot[];
|
|
442
|
+
/**
|
|
443
|
+
* Each `c.serve` call that listened, in call order: `{ command, port, portEnv, detected,
|
|
444
|
+
* address, readyMs }`. `port` is where it listens; `detected` is true when that isn't the
|
|
445
|
+
* `PORT` it was given (`portEnv`), so reeval took the one port that started listening after
|
|
446
|
+
* the command did. Absent when the attempt served nothing.
|
|
447
|
+
*/
|
|
448
|
+
serves?: ServeRecord[];
|
|
331
449
|
/**
|
|
332
450
|
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
333
451
|
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
@@ -342,9 +460,9 @@ type AttemptRecord = {
|
|
|
342
460
|
/** Every try, in order, when it was given `retries`; its log has each try's output. */
|
|
343
461
|
tries?: ExecTry[];
|
|
344
462
|
}>;
|
|
345
|
-
/** Every `expect` and `expect.
|
|
463
|
+
/** Every `expect` and `expect.require` matcher call that settled, in call order; unset with none. */
|
|
346
464
|
assertions?: AssertionRecord[];
|
|
347
|
-
/** Files under a dependency directory
|
|
465
|
+
/** Files under a dependency directory, such as the `node_modules/` directory or `.venv/` directory, collapse to one entry with `count`. */
|
|
348
466
|
changedFiles: Array<FileChange & {
|
|
349
467
|
count?: number;
|
|
350
468
|
}>;
|
|
@@ -360,7 +478,7 @@ type AttemptRecord = {
|
|
|
360
478
|
type EvalEntry = {
|
|
361
479
|
/** `file › group › … › name`, as shown. */
|
|
362
480
|
id: string;
|
|
363
|
-
/**
|
|
481
|
+
/** Used to match an evaluation across runs; equal to `id`. */
|
|
364
482
|
key: string;
|
|
365
483
|
/** Directory name inside the artifact directory, from `key`. */
|
|
366
484
|
slug: string;
|
|
@@ -369,18 +487,28 @@ type EvalEntry = {
|
|
|
369
487
|
name: string;
|
|
370
488
|
/** Results are comparable only when fingerprints match. */
|
|
371
489
|
fingerprint: string;
|
|
372
|
-
/**
|
|
490
|
+
/**
|
|
491
|
+
* The evaluation file, by content hash: where the evaluation lives. Records from before `source`
|
|
492
|
+
* also list the local files it imported, and were fingerprinted over them.
|
|
493
|
+
*/
|
|
373
494
|
sources: Array<{
|
|
374
495
|
path: string;
|
|
375
496
|
sha256: string;
|
|
376
497
|
}>;
|
|
498
|
+
/**
|
|
499
|
+
* Hashes of what the `sources` fingerprint part covers but `revision`: the eval's body, its
|
|
500
|
+
* case data and its options with its groups'. Unset on records from before it.
|
|
501
|
+
*/
|
|
502
|
+
source?: EvalSource;
|
|
503
|
+
/** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
|
|
504
|
+
revision?: Revision;
|
|
377
505
|
tags: string[];
|
|
378
|
-
/** The baseline environment, when the
|
|
506
|
+
/** The baseline environment, when the evaluation uses `compare`. */
|
|
379
507
|
compare?: EvalCompareMeta;
|
|
380
508
|
/**
|
|
381
|
-
* Over gated cells
|
|
382
|
-
* only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
383
|
-
* failures don't fail the
|
|
509
|
+
* Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
|
|
510
|
+
* `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
511
|
+
* cell's failures don't fail the evaluation.
|
|
384
512
|
*/
|
|
385
513
|
outcome: Outcome;
|
|
386
514
|
/** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
|
|
@@ -393,9 +521,9 @@ type EvalEntry = {
|
|
|
393
521
|
* The eval's pairwise questions (those with `across`), answered after its attempts by votes,
|
|
394
522
|
* one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
395
523
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
396
|
-
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
397
|
-
* `<eval-slug>/questions/<n>.json
|
|
398
|
-
* masked and screenshots as refs; unset where the
|
|
524
|
+
* (rates by cut, votes, ties, a failed call, and a sample of reasons). Each answer's `input` is
|
|
525
|
+
* the `<eval-slug>/questions/<n>.json` file: every call its model was sent, per pair and order,
|
|
526
|
+
* with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
|
|
399
527
|
*/
|
|
400
528
|
answers?: CompareAnswer[];
|
|
401
529
|
/** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
|
|
@@ -412,7 +540,7 @@ type FamilyInterval = {
|
|
|
412
540
|
hi: number;
|
|
413
541
|
level: 0.95;
|
|
414
542
|
/**
|
|
415
|
-
* `wilson`
|
|
543
|
+
* `wilson` or `newcombe-paired`: one counted attempt a case and side, so each case is one
|
|
416
544
|
* Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
|
|
417
545
|
* attempts within a case never counted as independent.
|
|
418
546
|
*/
|
|
@@ -487,17 +615,17 @@ type FamilyLift = {
|
|
|
487
615
|
outcome: 'pass' | 'fail';
|
|
488
616
|
effort?: FamilyEffort;
|
|
489
617
|
};
|
|
490
|
-
/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an
|
|
618
|
+
/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
|
|
491
619
|
type FamilyRecord = {
|
|
492
620
|
/** `file › group › … › template`. */
|
|
493
621
|
id: string;
|
|
494
622
|
file: string;
|
|
495
623
|
group: string[];
|
|
496
|
-
/** The name template,
|
|
624
|
+
/** The name template, for example `fixes $id`. */
|
|
497
625
|
name: string;
|
|
498
626
|
/** Cases in the list. */
|
|
499
627
|
cases: number;
|
|
500
|
-
/** Cases with an
|
|
628
|
+
/** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
|
|
501
629
|
ran: number;
|
|
502
630
|
/** One per cell. */
|
|
503
631
|
rates: FamilyRate[];
|
|
@@ -535,26 +663,34 @@ type Lift = {
|
|
|
535
663
|
};
|
|
536
664
|
/** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
|
|
537
665
|
type FingerprintParts = {
|
|
538
|
-
/** Over `EvalEntry.sources`. */
|
|
666
|
+
/** Over `EvalEntry.source` and `EvalEntry.revision`; on records from before them, over `EvalEntry.sources`. */
|
|
539
667
|
sources: string;
|
|
540
668
|
/** Over the harness descriptor, less concurrency caps. */
|
|
541
669
|
harness: string;
|
|
542
670
|
environment: string | null;
|
|
543
671
|
image: string | null;
|
|
544
672
|
agentCli: string | null;
|
|
545
|
-
/** Over the judge's model, threshold and version
|
|
673
|
+
/** Over the judge's model, threshold and version. */
|
|
546
674
|
judge: string;
|
|
547
675
|
/** Over the attempt's context files. */
|
|
548
676
|
context: string;
|
|
549
677
|
/** The gateway its model calls went through; unset on `direct`, each provider's own key. */
|
|
550
678
|
gateway?: string;
|
|
679
|
+
/**
|
|
680
|
+
* The cell's gate (`none`, `passRate 0.5`); unset on the default `all`. Compare reads a change
|
|
681
|
+
* in it as a policy change, not a reason the runs can't be compared.
|
|
682
|
+
*/
|
|
683
|
+
gate?: string;
|
|
551
684
|
};
|
|
552
|
-
/** One harness and environment of an
|
|
685
|
+
/** One harness and environment of an evaluation, and how its attempts went. */
|
|
553
686
|
type CellRecord = Cell & {
|
|
554
687
|
/** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
|
|
555
688
|
fingerprint: string;
|
|
556
689
|
fingerprintParts: FingerprintParts;
|
|
557
|
-
/**
|
|
690
|
+
/**
|
|
691
|
+
* The environment's digest; null for none. With `environment`, the key into
|
|
692
|
+
* `RunRecord.environments`: environments with the same contents share a digest.
|
|
693
|
+
*/
|
|
558
694
|
environmentDigest: string | null;
|
|
559
695
|
harnessKind: string;
|
|
560
696
|
/**
|
|
@@ -569,10 +705,15 @@ type CellRecord = Cell & {
|
|
|
569
705
|
* Fast is not this.
|
|
570
706
|
*/
|
|
571
707
|
effort: string | null;
|
|
572
|
-
/** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the
|
|
708
|
+
/** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the evaluation. */
|
|
573
709
|
gate: CellGate;
|
|
574
710
|
/** Environment keys the variant replaced on the control's (or a child on its parent's). */
|
|
575
711
|
overrides: string[];
|
|
712
|
+
/**
|
|
713
|
+
* Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed, and
|
|
714
|
+
* gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
|
|
715
|
+
*/
|
|
716
|
+
bestOf?: number;
|
|
576
717
|
runs: number;
|
|
577
718
|
pass: number;
|
|
578
719
|
fail: number;
|
|
@@ -588,7 +729,7 @@ type CellRecord = Cell & {
|
|
|
588
729
|
outcome: Outcome;
|
|
589
730
|
lift?: Lift;
|
|
590
731
|
/**
|
|
591
|
-
* Each
|
|
732
|
+
* Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
|
|
592
733
|
* attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
|
|
593
734
|
*/
|
|
594
735
|
checks?: CellCheck[];
|
|
@@ -601,10 +742,10 @@ type CheckTally = {
|
|
|
601
742
|
notRun: number;
|
|
602
743
|
};
|
|
603
744
|
/**
|
|
604
|
-
* One check over a cell's counted attempts. `verdict` (variant cells)
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
745
|
+
* One check over a cell's counted attempts. `verdict` (variant cells) is `discriminates` when
|
|
746
|
+
* every answered instance passes on one side and fails on the other, `both-pass` when every
|
|
747
|
+
* answered instance passes on both sides, `both-fail` when every answered instance fails on both
|
|
748
|
+
* sides, or `mixed` otherwise. It is absent when either side has no pass or fail.
|
|
608
749
|
*/
|
|
609
750
|
type CellCheck = CheckTally & {
|
|
610
751
|
label: string;
|
|
@@ -664,7 +805,10 @@ type RunRecord = {
|
|
|
664
805
|
harness?: string[];
|
|
665
806
|
environment?: string;
|
|
666
807
|
sandbox?: string;
|
|
808
|
+
/** With `bestOf`, the most attempts a cell runs. */
|
|
667
809
|
runs?: number;
|
|
810
|
+
/** Set by `defaults.runs: { bestOf }`. */
|
|
811
|
+
bestOf?: number;
|
|
668
812
|
timeout?: string | number;
|
|
669
813
|
env?: string[];
|
|
670
814
|
};
|
|
@@ -698,7 +842,13 @@ type RunRecord = {
|
|
|
698
842
|
config: {
|
|
699
843
|
sandbox: string;
|
|
700
844
|
environment: string | null;
|
|
845
|
+
/**
|
|
846
|
+
* The config's `runs` (with `bestOf`, the most a cell runs); 1 when it sets none, though a
|
|
847
|
+
* compare's cells then run 4 each.
|
|
848
|
+
*/
|
|
701
849
|
runs: number;
|
|
850
|
+
/** Set by the config's `runs: { bestOf }`. */
|
|
851
|
+
bestOf?: number;
|
|
702
852
|
judgeModel: string | null;
|
|
703
853
|
threshold: number;
|
|
704
854
|
timeoutMs: number;
|
|
@@ -717,28 +867,26 @@ type RunRecord = {
|
|
|
717
867
|
/** Each pooled `evaluate.each` that ran; unset when none did. */
|
|
718
868
|
families?: FamilyRecord[];
|
|
719
869
|
/**
|
|
720
|
-
*
|
|
721
|
-
* attempt awaited, a worker that died. Any one fails the run. Unset when there
|
|
870
|
+
* These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
|
|
871
|
+
* or throw no attempt awaited, or a worker that died. Any one fails the run. Unset when there
|
|
872
|
+
* were none.
|
|
722
873
|
*/
|
|
723
874
|
errors?: string[];
|
|
724
875
|
};
|
|
725
|
-
/**
|
|
876
|
+
/** These fields describe what ran and graded the agents: the machine, images, harnesses, and judge. */
|
|
726
877
|
type Stack = {
|
|
727
878
|
/** The reeval version that wrote the run. */
|
|
728
879
|
reeval: string;
|
|
729
|
-
/** `process.versions.node
|
|
880
|
+
/** `process.versions.node`, or the compatible Node version reported by another runtime. */
|
|
730
881
|
node: string;
|
|
731
|
-
/**
|
|
732
|
-
* The runtime reeval ran on, `node`, `bun`, `deno` or another, and its own version. Absent in
|
|
733
|
-
* runs recorded before it, which ran on Node.
|
|
734
|
-
*/
|
|
882
|
+
/** The runtime reeval ran on, `node`, `bun`, `deno`, or another, and its version. */
|
|
735
883
|
runtime?: {
|
|
736
884
|
name: string;
|
|
737
885
|
version: string;
|
|
738
886
|
};
|
|
739
887
|
/**
|
|
740
|
-
* The language
|
|
741
|
-
*
|
|
888
|
+
* The evaluation language, `typescript` for `*.eval.ts` files, and the version of its toolchain
|
|
889
|
+
* when the runner reports one.
|
|
742
890
|
*/
|
|
743
891
|
language?: {
|
|
744
892
|
name: string;
|
|
@@ -746,16 +894,17 @@ type Stack = {
|
|
|
746
894
|
};
|
|
747
895
|
os: string;
|
|
748
896
|
arch: string;
|
|
749
|
-
/** Every image an attempt started from. `id` is content-addressed; `repoDigests` only
|
|
897
|
+
/** Every image an attempt started from. `id` is content-addressed; `repoDigests` exist only after the image is pushed. */
|
|
750
898
|
images: Array<AttemptImage & {
|
|
751
899
|
repoDigests: string[];
|
|
752
900
|
}>;
|
|
753
901
|
/** Exact versions of the packages that run and grade the agent. */
|
|
754
902
|
packages: Record<string, string>;
|
|
755
903
|
/**
|
|
756
|
-
*
|
|
757
|
-
* reported serving, with the variant it resolved), and
|
|
758
|
-
* suffixes those labels named). Fast (`[fast=true]`) is not
|
|
904
|
+
* For each harness, the record includes its configured `model` and requested `effort`,
|
|
905
|
+
* `resolvedModels` (what it reported serving, with the variant it resolved), and
|
|
906
|
+
* `resolvedEffort` (reasoning-effort suffixes those labels named). Fast (`[fast=true]`) is not
|
|
907
|
+
* `resolvedEffort`.
|
|
759
908
|
*/
|
|
760
909
|
harnesses: Array<{
|
|
761
910
|
id: string;
|
|
@@ -791,12 +940,12 @@ type Stack = {
|
|
|
791
940
|
threshold: number;
|
|
792
941
|
/** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
|
|
793
942
|
version: string;
|
|
794
|
-
/**
|
|
943
|
+
/** The config's default judge inputs, every key resolved. Unset when the config names none, which reeval requires: each `c.judge()` call names its own. */
|
|
795
944
|
state?: ResolvedJudgeStateOptions;
|
|
796
945
|
};
|
|
797
946
|
/** Union of every attempt's context files. */
|
|
798
947
|
context: ContextFile[];
|
|
799
|
-
/** What
|
|
948
|
+
/** What evaluation fingerprints hash; fingerprints from different scopes aren't comparable. */
|
|
800
949
|
fingerprintScope: string[];
|
|
801
950
|
};
|
|
802
951
|
/** The parts of a run's stack that `reeval compare` checks for drift. */
|
|
@@ -824,8 +973,15 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
|
|
|
824
973
|
};
|
|
825
974
|
/** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
|
|
826
975
|
effort: CellEffort | null;
|
|
827
|
-
/** Of `fail`, the attempts the run's interrupt
|
|
828
|
-
|
|
976
|
+
/** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
|
|
977
|
+
canceled?: number;
|
|
978
|
+
};
|
|
979
|
+
/** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
|
|
980
|
+
type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
|
|
981
|
+
lo: number;
|
|
982
|
+
hi: number;
|
|
983
|
+
/** Unset when the compare has no `minLift`, so the lift can't fail. */
|
|
984
|
+
minLift?: number;
|
|
829
985
|
};
|
|
830
986
|
/**
|
|
831
987
|
* One line of the derived history listing: cached locally by the CLI, and the run header in the
|
|
@@ -860,636 +1016,24 @@ type HistoryEntry = {
|
|
|
860
1016
|
slug: string;
|
|
861
1017
|
fingerprint: string;
|
|
862
1018
|
sources: EvalEntry['sources'];
|
|
1019
|
+
source?: EvalEntry['source'];
|
|
1020
|
+
revision?: EvalEntry['revision'];
|
|
863
1021
|
outcome: Outcome;
|
|
864
1022
|
passRate: number | null;
|
|
865
1023
|
cells: HistoryCell[];
|
|
866
1024
|
}>;
|
|
867
1025
|
/** The run's pooled families; unset when none ran. */
|
|
868
|
-
families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'
|
|
1026
|
+
families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
|
|
1027
|
+
/** The eval ids of its cases in this run. */
|
|
1028
|
+
caseIds: string[];
|
|
1029
|
+
lifts: FamilyLiftSummary[];
|
|
1030
|
+
}>;
|
|
869
1031
|
/** The run's errors outside any attempt; unset when there were none. */
|
|
870
1032
|
errors?: string[];
|
|
871
1033
|
};
|
|
872
1034
|
//#endregion
|
|
873
|
-
//#region src/view/suite.d.ts
|
|
874
|
-
/**
|
|
875
|
-
* A variant against its baseline: `better` or `worse` when the stored 95% interval on the
|
|
876
|
-
* difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
|
|
877
|
-
* including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
|
|
878
|
-
*/
|
|
879
|
-
type Effect = 'better' | 'worse' | 'same' | 'unclear';
|
|
880
|
-
/**
|
|
881
|
-
* What an eval's cells compare: `change` when it has variants (or several environments),
|
|
882
|
-
* `harness` when it has several harnesses, `both`, or `none`.
|
|
883
|
-
*/
|
|
884
|
-
type Across = 'none' | 'change' | 'harness' | 'both';
|
|
885
|
-
/**
|
|
886
|
-
* One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
|
|
887
|
-
* lists them all in the order the nodes appear.
|
|
888
|
-
*
|
|
889
|
-
* A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
|
|
890
|
-
*
|
|
891
|
-
* - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
|
|
892
|
-
* (`evals/a.eval.ts › fixes $id`);
|
|
893
|
-
* - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
|
|
894
|
-
* - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
|
|
895
|
-
* - `<cell>/checks/<label>`: a check, by its assertion label;
|
|
896
|
-
* - `<cell>/effort`: the cell's effort, only when it was tested;
|
|
897
|
-
* - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
|
|
898
|
-
*
|
|
899
|
-
* A case segment never has a raw `+` and a cell segment always has two, so the two can't be
|
|
900
|
-
* confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
|
|
901
|
-
* fact in every run that has it.
|
|
902
|
-
*/
|
|
903
|
-
type SuiteResult = {
|
|
904
|
-
runId: string;
|
|
905
|
-
status: RunRecord['status'];
|
|
906
|
-
/**
|
|
907
|
-
* The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
|
|
908
|
-
* an attempt, and `skipped` on an interrupted run.
|
|
909
|
-
*/
|
|
910
|
-
outcome: Outcome;
|
|
911
|
-
/**
|
|
912
|
-
* Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
|
|
913
|
-
* awaited, a worker that died. Any one fails the run.
|
|
914
|
-
*/
|
|
915
|
-
errors: string[];
|
|
916
|
-
/**
|
|
917
|
-
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
918
|
-
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
|
|
919
|
-
* `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
|
|
920
|
-
*/
|
|
921
|
-
tally: SuiteTally;
|
|
922
|
-
/** Sorted by path. */
|
|
923
|
-
files: ResultFile[];
|
|
924
|
-
/** Every node's `path`, in the order they appear. */
|
|
925
|
-
paths: string[];
|
|
926
|
-
};
|
|
927
|
-
type SuiteTally = {
|
|
928
|
-
errors: number;
|
|
929
|
-
pass: number;
|
|
930
|
-
notGated: number;
|
|
931
|
-
fail: number;
|
|
932
|
-
infraError: number;
|
|
933
|
-
cancelled: number;
|
|
934
|
-
skipped: number;
|
|
935
|
-
};
|
|
936
|
-
/**
|
|
937
|
-
* An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
|
|
938
|
-
* gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
|
|
939
|
-
*/
|
|
940
|
-
type ResultOutcome = Outcome | 'cancelled';
|
|
941
|
-
type ResultFile = {
|
|
942
|
-
path: string;
|
|
943
|
-
evals: EvalResult[];
|
|
944
|
-
};
|
|
945
|
-
type EvalResult = {
|
|
946
|
-
path: string;
|
|
947
|
-
/** The eval's id, or the family's. */
|
|
948
|
-
id: string;
|
|
949
|
-
group: string[];
|
|
950
|
-
/** The eval's name; on a family, its template. */
|
|
951
|
-
name: string;
|
|
952
|
-
/** A family's name template (`fixes $id`); null on a single eval. */
|
|
953
|
-
template: string | null;
|
|
954
|
-
/** Source order in the file. */
|
|
955
|
-
order: number;
|
|
956
|
-
across: Across;
|
|
957
|
-
/**
|
|
958
|
-
* A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
|
|
959
|
-
* unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
|
|
960
|
-
*/
|
|
961
|
-
family: boolean;
|
|
962
|
-
/** A placeholder (`evaluate.todo`) that never ran. */
|
|
963
|
-
todo: boolean;
|
|
964
|
-
/** On a family: `fail` when a case failed or its pooled gate did. */
|
|
965
|
-
outcome: ResultOutcome;
|
|
966
|
-
cells: CellResult[] | null;
|
|
967
|
-
cases: CaseResult[] | null;
|
|
968
|
-
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
969
|
-
pooled: PooledArm[] | null;
|
|
970
|
-
/**
|
|
971
|
-
* Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
|
|
972
|
-
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
973
|
-
*/
|
|
974
|
-
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
975
|
-
};
|
|
976
|
-
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
977
|
-
type QuestionResult = {
|
|
978
|
-
kind: 'pairs';
|
|
979
|
-
question: string;
|
|
980
|
-
/** Pairs voted, each counted once per order it was asked in. */
|
|
981
|
-
votes: number;
|
|
982
|
-
rates: Array<{
|
|
983
|
-
cut: 'within' | 'across';
|
|
984
|
-
name: string;
|
|
985
|
-
wins: number;
|
|
986
|
-
n: number;
|
|
987
|
-
}>;
|
|
988
|
-
/** Why some or all of its votes are missing: the model call failed. */
|
|
989
|
-
error: string | null;
|
|
990
|
-
};
|
|
991
|
-
/** A question asked of each attempt on its own: what each one answered. Never gates. */
|
|
992
|
-
type AttemptQuestionResult = {
|
|
993
|
-
kind: 'attempts';
|
|
994
|
-
question: string;
|
|
995
|
-
/** Attempts it was asked of, failed calls included. */
|
|
996
|
-
asked: number;
|
|
997
|
-
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
998
|
-
choices: Array<{
|
|
999
|
-
choice: string;
|
|
1000
|
-
n: number;
|
|
1001
|
-
}> | null;
|
|
1002
|
-
/** Each attempt's answer, in attempt order; failed calls left out. */
|
|
1003
|
-
answers: Array<{
|
|
1004
|
-
attempt: string;
|
|
1005
|
-
choice: string | null;
|
|
1006
|
-
text: string;
|
|
1007
|
-
}>;
|
|
1008
|
-
/** Calls that failed, and the first one's error. */
|
|
1009
|
-
failed: number;
|
|
1010
|
-
error: string | null;
|
|
1011
|
-
};
|
|
1012
|
-
type CaseResult = {
|
|
1013
|
-
path: string;
|
|
1014
|
-
/** The case's own eval id. */
|
|
1015
|
-
id: string;
|
|
1016
|
-
/** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
|
|
1017
|
-
label: string;
|
|
1018
|
-
name: string;
|
|
1019
|
-
/** Its place in the case list. */
|
|
1020
|
-
order: number;
|
|
1021
|
-
outcome: ResultOutcome;
|
|
1022
|
-
cells: CellResult[];
|
|
1023
|
-
};
|
|
1024
|
-
type CellResult = {
|
|
1025
|
-
path: string;
|
|
1026
|
-
key: string;
|
|
1027
|
-
harness: string;
|
|
1028
|
-
/** `-` when no environment is set. */
|
|
1029
|
-
environment: string;
|
|
1030
|
-
sandbox: string;
|
|
1031
|
-
/** The variant's name; null on a control. */
|
|
1032
|
-
variant: string | null;
|
|
1033
|
-
/** The control of an eval that has variants: the arm the variants are measured against. */
|
|
1034
|
-
baseline: boolean;
|
|
1035
|
-
/** On a variant, its baseline cell's key; else null. */
|
|
1036
|
-
vs: string | null;
|
|
1037
|
-
/** The cell's place in the eval's cells. */
|
|
1038
|
-
order: number;
|
|
1039
|
-
/** Its failures fail the eval (its gate isn't `'none'`). */
|
|
1040
|
-
gated: boolean;
|
|
1041
|
-
outcome: Outcome;
|
|
1042
|
-
/** Every attempt: `passed + failed + infraError + skipped`. */
|
|
1043
|
-
attempts: number;
|
|
1044
|
-
passed: number;
|
|
1045
|
-
failed: number;
|
|
1046
|
-
/** Infrastructure errors, which are neither passes nor fails. */
|
|
1047
|
-
infraError: number;
|
|
1048
|
-
skipped: number;
|
|
1049
|
-
fingerprint: string;
|
|
1050
|
-
/** The skill sources and files the cell's environment placed, as written. */
|
|
1051
|
-
context: string[];
|
|
1052
|
-
/**
|
|
1053
|
-
* Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
|
|
1054
|
-
* attempt (every one an infra error or skipped) lists each as `not run`.
|
|
1055
|
-
*/
|
|
1056
|
-
checks: CheckResult[];
|
|
1057
|
-
/** On a variant, against its baseline; null on a control. */
|
|
1058
|
-
effect: Effect | null;
|
|
1059
|
-
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1060
|
-
interval: Interval | null;
|
|
1061
|
-
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1062
|
-
minLift: number | null;
|
|
1063
|
-
/** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
|
|
1064
|
-
effort: EffortCounters | null;
|
|
1065
|
-
/** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
|
|
1066
|
-
effortDiffers: boolean;
|
|
1067
|
-
/** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
|
|
1068
|
-
errors: CellError[];
|
|
1069
|
-
/** Web searches and fetches per counted attempt; null where the harness can't report them. */
|
|
1070
|
-
searches: Array<number | null>;
|
|
1071
|
-
fetches: Array<number | null>;
|
|
1072
|
-
};
|
|
1073
|
-
/** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
|
|
1074
|
-
declare const cancelledOf: (cell: CellResult) => number;
|
|
1075
|
-
/** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
|
|
1076
|
-
declare const countedOf: (cell: CellResult) => number;
|
|
1077
|
-
type CheckResult = {
|
|
1078
|
-
path: string;
|
|
1079
|
-
label: string;
|
|
1080
|
-
/** First-seen order across the eval's attempts. */
|
|
1081
|
-
order: number;
|
|
1082
|
-
outcome: CheckOutcome;
|
|
1083
|
-
/**
|
|
1084
|
-
* On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
|
|
1085
|
-
* null on a control or without one.
|
|
1086
|
-
*/
|
|
1087
|
-
effect: Effect | null;
|
|
1088
|
-
/** Set when the check is a judge verdict. */
|
|
1089
|
-
judge: JudgeResult | null;
|
|
1090
|
-
/**
|
|
1091
|
-
* A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
|
|
1092
|
-
* it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
|
|
1093
|
-
* when none did.
|
|
1094
|
-
*/
|
|
1095
|
-
retried?: string;
|
|
1096
|
-
};
|
|
1097
|
-
/**
|
|
1098
|
-
* `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
|
|
1099
|
-
* none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
|
|
1100
|
-
* reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
|
|
1101
|
-
* first failure's message.
|
|
1102
|
-
*/
|
|
1103
|
-
type CheckOutcome = {
|
|
1104
|
-
status: 'not run';
|
|
1105
|
-
} | {
|
|
1106
|
-
status: 'ran';
|
|
1107
|
-
passed: number;
|
|
1108
|
-
failed: number;
|
|
1109
|
-
errored: number;
|
|
1110
|
-
notRun: number;
|
|
1111
|
-
assertion: string | null;
|
|
1112
|
-
};
|
|
1113
|
-
/**
|
|
1114
|
-
* One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
|
|
1115
|
-
* an error), its message's first line, and how often.
|
|
1116
|
-
*/
|
|
1117
|
-
type CellError = {
|
|
1118
|
-
kind: ErrorKind | 'turn';
|
|
1119
|
-
message: string;
|
|
1120
|
-
attempts: number;
|
|
1121
|
-
};
|
|
1122
|
-
type JudgeResult = {
|
|
1123
|
-
question: string;
|
|
1124
|
-
threshold: number;
|
|
1125
|
-
/** One per counted attempt that judged it. */
|
|
1126
|
-
scores: number[];
|
|
1127
|
-
};
|
|
1128
|
-
type Interval = {
|
|
1129
|
-
low: number;
|
|
1130
|
-
high: number;
|
|
1131
|
-
};
|
|
1132
|
-
/** A null counter is one the harness didn't record; a recorded zero stays zero. */
|
|
1133
|
-
type EffortCounters = {
|
|
1134
|
-
tools: number | null;
|
|
1135
|
-
agentMs: number | null;
|
|
1136
|
-
tokens: number | null;
|
|
1137
|
-
};
|
|
1138
|
-
type PooledArm = {
|
|
1139
|
-
path: string;
|
|
1140
|
-
/** The variant cell's key, the same in every case. */
|
|
1141
|
-
key: string;
|
|
1142
|
-
vs: string;
|
|
1143
|
-
effect: Effect;
|
|
1144
|
-
/** Cases with counted attempts on both sides. */
|
|
1145
|
-
pairs: number;
|
|
1146
|
-
interval: Interval;
|
|
1147
|
-
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
1148
|
-
minLift: number | null;
|
|
1149
|
-
outcome: 'pass' | 'fail';
|
|
1150
|
-
};
|
|
1151
|
-
type Side = {
|
|
1152
|
-
pass: number;
|
|
1153
|
-
counted: number;
|
|
1154
|
-
};
|
|
1155
|
-
/**
|
|
1156
|
-
* The effect of `variant` against `control`, from the interval on their difference. A side
|
|
1157
|
-
* with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
|
|
1158
|
-
*/
|
|
1159
|
-
declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
|
|
1160
|
-
/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
|
|
1161
|
-
declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
|
|
1162
|
-
/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
|
|
1163
|
-
declare const factSegment: (s: string) => string;
|
|
1164
|
-
declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
|
|
1165
|
-
/**
|
|
1166
|
-
* An error's one line: its first line with any letter or digit in it, joined with the next one
|
|
1167
|
-
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1168
|
-
*/
|
|
1169
|
-
declare function messageLine(text: string): string;
|
|
1170
|
-
/** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
|
|
1171
|
-
declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
|
|
1172
|
-
/**
|
|
1173
|
-
* A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
|
|
1174
|
-
* gating it, so it reads `not gated`, never `pass`.
|
|
1175
|
-
*/
|
|
1176
|
-
declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
|
|
1177
|
-
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1178
|
-
declare function suiteResult(record: RunRecord): SuiteResult;
|
|
1179
|
-
//#endregion
|
|
1180
|
-
//#region src/view/insights.d.ts
|
|
1181
|
-
/** `<history>/<run>/insights.json`, beside the record. */
|
|
1182
|
-
declare const INSIGHTS_FILE = "insights.json";
|
|
1183
|
-
/** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
|
|
1184
|
-
declare const NO_MODEL = "none";
|
|
1185
|
-
/** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
|
|
1186
|
-
declare const fromModel: (insights: RunInsights) => boolean;
|
|
1187
|
-
/** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
|
|
1188
|
-
type Job = 'choose' | 'finish' | 'other';
|
|
1189
|
-
/**
|
|
1190
|
-
* A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
|
|
1191
|
-
* They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
|
|
1192
|
-
*/
|
|
1193
|
-
type RunInsights = {
|
|
1194
|
-
kind: 'reeval-insights';
|
|
1195
|
-
schemaVersion: 1;
|
|
1196
|
-
runId: string;
|
|
1197
|
-
/**
|
|
1198
|
-
* The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
|
|
1199
|
-
* reeval wrote the file without one, to hold the run's deterministic remarks only.
|
|
1200
|
-
*/
|
|
1201
|
-
model: string;
|
|
1202
|
-
/** The prompts' version: insights from another version were asked differently. */
|
|
1203
|
-
promptVersion: string;
|
|
1204
|
-
/** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
|
|
1205
|
-
factsHash: string;
|
|
1206
|
-
createdAt: string;
|
|
1207
|
-
/**
|
|
1208
|
-
* Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
|
|
1209
|
-
* reports them. `costUsd` is null when reeval doesn't know the model's price.
|
|
1210
|
-
*/
|
|
1211
|
-
usage: {
|
|
1212
|
-
inputTokens: number;
|
|
1213
|
-
outputTokens: number;
|
|
1214
|
-
reasoningTokens?: number;
|
|
1215
|
-
costUsd: number | null;
|
|
1216
|
-
};
|
|
1217
|
-
/**
|
|
1218
|
-
* `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
|
|
1219
|
-
* without insights. Absent when no budget applied.
|
|
1220
|
-
*/
|
|
1221
|
-
budget?: {
|
|
1222
|
-
maxTokens: number;
|
|
1223
|
-
skipped: string[];
|
|
1224
|
-
};
|
|
1225
|
-
/**
|
|
1226
|
-
* Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
|
|
1227
|
-
* timeout), with the last error: they print without insights. Absent when none failed.
|
|
1228
|
-
*/
|
|
1229
|
-
failed?: Array<{
|
|
1230
|
-
evalId: string;
|
|
1231
|
-
error: string;
|
|
1232
|
-
}>;
|
|
1233
|
-
/**
|
|
1234
|
-
* Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
|
|
1235
|
-
* the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
|
|
1236
|
-
* during the headline call, which leaves `headline` null.
|
|
1237
|
-
*/
|
|
1238
|
-
stopped?: {
|
|
1239
|
-
signal: 'SIGINT' | 'SIGTERM';
|
|
1240
|
-
skipped: string[];
|
|
1241
|
-
};
|
|
1242
|
-
/** One sentence for the whole run, citing facts; null when the model's was dropped. */
|
|
1243
|
-
headline: Sentence | null;
|
|
1244
|
-
/** One per eval (a family is one), in the order the output prints them. */
|
|
1245
|
-
evals: EvalInsight[];
|
|
1246
|
-
/**
|
|
1247
|
-
* Remarks about the evals' own quality, ranked: every one this run raised, including those an
|
|
1248
|
-
* earlier run printed first (`firstSeenRun`), which don't print again.
|
|
1249
|
-
*/
|
|
1250
|
-
remarks: Remark[];
|
|
1251
|
-
/**
|
|
1252
|
-
* What the checks dropped, and how many sentences were asked for again, so the output can say
|
|
1253
|
-
* so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
|
|
1254
|
-
* its text and why.
|
|
1255
|
-
*/
|
|
1256
|
-
validation: {
|
|
1257
|
-
dropped: number;
|
|
1258
|
-
regenerated: number;
|
|
1259
|
-
drops?: Array<{
|
|
1260
|
-
slot: string;
|
|
1261
|
-
text: string;
|
|
1262
|
-
reason: string;
|
|
1263
|
-
}>;
|
|
1264
|
-
};
|
|
1265
|
-
/** Set when insights failed; the facts still print. */
|
|
1266
|
-
error?: string;
|
|
1267
|
-
};
|
|
1268
|
-
type EvalInsight = {
|
|
1269
|
-
/** The eval's `SuiteResult` path; for a family, the family's. */
|
|
1270
|
-
evalId: string;
|
|
1271
|
-
intent: {
|
|
1272
|
-
/** What one attempt is testing. Can't be read from the cell shape. */
|
|
1273
|
-
job: Job;
|
|
1274
|
-
/** What the cells compare. Deterministic, copied from the facts. */
|
|
1275
|
-
across: Across;
|
|
1276
|
-
/** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
|
|
1277
|
-
question: string;
|
|
1278
|
-
/** The tool a choose eval asks about, and the change a measure eval makes. */
|
|
1279
|
-
subject: {
|
|
1280
|
-
tool?: string;
|
|
1281
|
-
change?: string;
|
|
1282
|
-
};
|
|
1283
|
-
/** How the eval name relates to what the eval checks. */
|
|
1284
|
-
name: 'states-intent' | 'no-intent' | 'contradicts';
|
|
1285
|
-
/**
|
|
1286
|
-
* sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
|
|
1287
|
-
* the model and the prompt version: the classification's cache key.
|
|
1288
|
-
*/
|
|
1289
|
-
key: string;
|
|
1290
|
-
};
|
|
1291
|
-
/** The answer to `question`, at most two sentences; null when it was dropped. */
|
|
1292
|
-
answer: Sentence | null;
|
|
1293
|
-
/** Up to three supporting points: what moved, what broke, what the agent did. */
|
|
1294
|
-
findings: Sentence[];
|
|
1295
|
-
/**
|
|
1296
|
-
* At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
|
|
1297
|
-
* Only rows the eval's layout opens a slot on are kept.
|
|
1298
|
-
*/
|
|
1299
|
-
rows?: Array<Sentence & {
|
|
1300
|
-
row: string;
|
|
1301
|
-
}>;
|
|
1302
|
-
/**
|
|
1303
|
-
* Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
|
|
1304
|
-
* verbatim in that attempt's final message. Only on `choose` evals.
|
|
1305
|
-
*/
|
|
1306
|
-
namedTools?: Array<{
|
|
1307
|
-
attempt: string;
|
|
1308
|
-
tools: string[];
|
|
1309
|
-
}>;
|
|
1310
|
-
};
|
|
1311
|
-
/**
|
|
1312
|
-
* A plain sentence citing the facts it states: `SuiteResult` paths
|
|
1313
|
-
* (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
|
|
1314
|
-
* (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
|
|
1315
|
-
*/
|
|
1316
|
-
type Sentence = {
|
|
1317
|
-
text: string;
|
|
1318
|
-
cites: string[];
|
|
1319
|
-
};
|
|
1320
|
-
/**
|
|
1321
|
-
* Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
|
|
1322
|
-
* gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
|
|
1323
|
-
* severity is the kind's, never the model's.
|
|
1324
|
-
*/
|
|
1325
|
-
declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
1326
|
-
origin: Remark['origin'];
|
|
1327
|
-
severity: 'concern' | 'suggestion';
|
|
1328
|
-
}>;
|
|
1329
|
-
/** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
|
|
1330
|
-
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
|
|
1331
|
-
/** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
|
|
1332
|
-
type Remark = {
|
|
1333
|
-
/**
|
|
1334
|
-
* sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
|
|
1335
|
-
* per kind (and server) per eval version.
|
|
1336
|
-
*/
|
|
1337
|
-
id: string;
|
|
1338
|
-
evalId: string;
|
|
1339
|
-
/** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
|
|
1340
|
-
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
|
|
1341
|
-
/** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
|
|
1342
|
-
severity: 'concern' | 'suggestion';
|
|
1343
|
-
/** One plain sentence: what's wrong. */
|
|
1344
|
-
text: string;
|
|
1345
|
-
/**
|
|
1346
|
-
* Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
|
|
1347
|
-
* file at the run's source hash. Every model remark has one.
|
|
1348
|
-
*/
|
|
1349
|
-
source?: {
|
|
1350
|
-
at: string;
|
|
1351
|
-
quote: string;
|
|
1352
|
-
};
|
|
1353
|
-
/** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
|
|
1354
|
-
evidence: string[];
|
|
1355
|
-
/**
|
|
1356
|
-
* An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
|
|
1357
|
-
* empty when the model gave none, and its `at` is the quoted line when it named no line.
|
|
1358
|
-
*/
|
|
1359
|
-
fix: {
|
|
1360
|
-
at: string;
|
|
1361
|
-
edit: string;
|
|
1362
|
-
};
|
|
1363
|
-
origin: 'model' | 'deterministic';
|
|
1364
|
-
/**
|
|
1365
|
-
* The first run that printed this `id`, or this run when none has: it prints only in that run,
|
|
1366
|
-
* except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
|
|
1367
|
-
* every run it happens.
|
|
1368
|
-
*/
|
|
1369
|
-
firstSeenRun: string;
|
|
1370
|
-
/**
|
|
1371
|
-
* Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
|
|
1372
|
-
* when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
|
|
1373
|
-
* `firstSeenRun` only.
|
|
1374
|
-
*/
|
|
1375
|
-
printed?: boolean;
|
|
1376
|
-
/** Set on later runs once the cited span changed: the remark was acted on. */
|
|
1377
|
-
actedOn?: {
|
|
1378
|
-
run: string;
|
|
1379
|
-
};
|
|
1380
|
-
};
|
|
1381
|
-
//#endregion
|
|
1382
|
-
//#region src/insights/input.d.ts
|
|
1383
|
-
/** One fact the model may cite, by its path. */
|
|
1384
|
-
type InsightFact = {
|
|
1385
|
-
path: string;
|
|
1386
|
-
kind: string;
|
|
1387
|
-
} & Record<string, unknown>;
|
|
1388
|
-
type InsightAttempt = {
|
|
1389
|
-
/** `<cell path>#<n>`. */
|
|
1390
|
-
id: string;
|
|
1391
|
-
cell: string;
|
|
1392
|
-
outcome: string;
|
|
1393
|
-
/** The agent's final message, clipped. */
|
|
1394
|
-
lastText: string;
|
|
1395
|
-
/** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
|
|
1396
|
-
lines: string[];
|
|
1397
|
-
};
|
|
1398
|
-
/** Where an insight sentence can go, as the eval's layout opens them. */
|
|
1399
|
-
type InsightSlots = {
|
|
1400
|
-
/** Row paths that take one sentence each. */
|
|
1401
|
-
rows: string[];
|
|
1402
|
-
findings: boolean;
|
|
1403
|
-
namedTools: boolean;
|
|
1404
|
-
};
|
|
1405
|
-
/** One eval as the classifier and the insight model see it. */
|
|
1406
|
-
type InsightInput = {
|
|
1407
|
-
call: 'classify' | 'insights';
|
|
1408
|
-
eval: {
|
|
1409
|
-
path: string;
|
|
1410
|
-
/** The eval's name, or a family's template: the author's statement of what it asks. */
|
|
1411
|
-
name: string;
|
|
1412
|
-
group: string[];
|
|
1413
|
-
file: string;
|
|
1414
|
-
across: Across;
|
|
1415
|
-
family: boolean;
|
|
1416
|
-
verdict: string;
|
|
1417
|
-
};
|
|
1418
|
-
/** The first attempt's prompts, in order. */
|
|
1419
|
-
prompts: string[];
|
|
1420
|
-
judgeQuestions: string[];
|
|
1421
|
-
checks: string[];
|
|
1422
|
-
/** Set on the insights call: the classification. */
|
|
1423
|
-
intent?: {
|
|
1424
|
-
job: Job;
|
|
1425
|
-
question: string;
|
|
1426
|
-
subject: {
|
|
1427
|
-
tool?: string;
|
|
1428
|
-
change?: string;
|
|
1429
|
-
};
|
|
1430
|
-
};
|
|
1431
|
-
/** Set on the insights call. */
|
|
1432
|
-
slots?: InsightSlots;
|
|
1433
|
-
facts: InsightFact[];
|
|
1434
|
-
attempts: InsightAttempt[];
|
|
1435
|
-
/**
|
|
1436
|
-
* Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
|
|
1437
|
-
* numbers it, which a remark's `source` and `fix` cite.
|
|
1438
|
-
*/
|
|
1439
|
-
source?: {
|
|
1440
|
-
file: string;
|
|
1441
|
-
lines: string[];
|
|
1442
|
-
};
|
|
1443
|
-
/** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
|
|
1444
|
-
trimmed?: string[];
|
|
1445
|
-
/** On a regeneration: why the earlier sentences were refused. */
|
|
1446
|
-
retry?: string[];
|
|
1447
|
-
};
|
|
1448
|
-
/** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
|
|
1449
|
-
type HeadlineInput = {
|
|
1450
|
-
call: 'headline';
|
|
1451
|
-
/**
|
|
1452
|
-
* Counts results, not evals: each case of a family counts one, as the tally line does, and
|
|
1453
|
-
* `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
|
|
1454
|
-
* results and stay out of it. The eval count is `evals.length`.
|
|
1455
|
-
*/
|
|
1456
|
-
tally: SuiteResult['tally'] & {
|
|
1457
|
-
total: number;
|
|
1458
|
-
unit: 'results';
|
|
1459
|
-
};
|
|
1460
|
-
evals: Array<{
|
|
1461
|
-
path: string;
|
|
1462
|
-
name: string;
|
|
1463
|
-
verdict: string;
|
|
1464
|
-
answer: string | null;
|
|
1465
|
-
}>;
|
|
1466
|
-
retry?: string[];
|
|
1467
|
-
};
|
|
1468
|
-
/** A fact as the model sees it: its short id (`F1`), which is what it cites. */
|
|
1469
|
-
type ModelFact = InsightFact & {
|
|
1470
|
-
id: string;
|
|
1471
|
-
};
|
|
1472
|
-
/** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
|
|
1473
|
-
type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
|
|
1474
|
-
id: string;
|
|
1475
|
-
cell: string;
|
|
1476
|
-
};
|
|
1477
|
-
/** One eval's call as the model sees it: facts, attempts and row slots by short id. */
|
|
1478
|
-
type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
|
|
1479
|
-
facts: ModelFact[];
|
|
1480
|
-
attempts: ModelAttempt[];
|
|
1481
|
-
slots?: Omit<InsightSlots, 'rows'> & {
|
|
1482
|
-
rows: string[];
|
|
1483
|
-
};
|
|
1484
|
-
};
|
|
1485
|
-
/** The headline call as the model sees it: each eval by its short id. */
|
|
1486
|
-
type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
|
|
1487
|
-
evals: Array<HeadlineInput['evals'][number] & {
|
|
1488
|
-
id: string;
|
|
1489
|
-
}>;
|
|
1490
|
-
};
|
|
1491
|
-
//#endregion
|
|
1492
1035
|
//#region src/types.d.ts
|
|
1036
|
+
/** More than zero and at most 596h: `90s`, `15m`, `1h`, or a number of milliseconds. */
|
|
1493
1037
|
type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
|
|
1494
1038
|
/** When an event started: wall clock, and ms since its turn started. */
|
|
1495
1039
|
type EventTime = {
|
|
@@ -1500,13 +1044,17 @@ type FileEditEvent = {
|
|
|
1500
1044
|
type: 'file-edit';
|
|
1501
1045
|
toolCallId: string;
|
|
1502
1046
|
path: string;
|
|
1503
|
-
|
|
1047
|
+
/**
|
|
1048
|
+
* `write` is a whole-file write whose result doesn't say whether the file existed before
|
|
1049
|
+
* (Antigravity's, fx's, Goose's, Grok Build's and pi's write tools, for example).
|
|
1050
|
+
*/
|
|
1051
|
+
kind: 'create' | 'modify' | 'write' | 'delete';
|
|
1504
1052
|
/** Line counts, or null when the harness didn't send the content (e.g. deletes). */
|
|
1505
1053
|
added: number | null;
|
|
1506
1054
|
removed: number | null;
|
|
1507
1055
|
};
|
|
1508
1056
|
/**
|
|
1509
|
-
*
|
|
1057
|
+
* Normalized agent events. Each tool call is one event, merged with its result; `durationMs`
|
|
1510
1058
|
* is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
|
|
1511
1059
|
* when the harness never reported the end. Long shell or edit calls set `stalled: true`.
|
|
1512
1060
|
*/
|
|
@@ -1599,16 +1147,21 @@ type Wait = CaptureWait | {
|
|
|
1599
1147
|
type CaptureInstruction = string | {
|
|
1600
1148
|
wait: Wait;
|
|
1601
1149
|
};
|
|
1150
|
+
/** The keys a capture's driver presses. In a focused `<select>`, the arrows change the choice. */
|
|
1151
|
+
type CaptureKey = 'Tab' | 'Enter' | 'Escape' | 'Backspace' | 'Space' | 'ArrowUp' | 'ArrowDown' | 'ArrowLeft' | 'ArrowRight';
|
|
1602
1152
|
type CaptureUseAction = {
|
|
1603
1153
|
type: 'click';
|
|
1604
1154
|
x: number;
|
|
1605
1155
|
y: number;
|
|
1606
|
-
} |
|
|
1156
|
+
} |
|
|
1157
|
+
/** `replace` selects what the focused field holds first, so the text replaces it. */
|
|
1158
|
+
{
|
|
1607
1159
|
type: 'type';
|
|
1608
1160
|
text: string;
|
|
1161
|
+
replace?: boolean;
|
|
1609
1162
|
} | {
|
|
1610
1163
|
type: 'key';
|
|
1611
|
-
key:
|
|
1164
|
+
key: CaptureKey;
|
|
1612
1165
|
} | {
|
|
1613
1166
|
type: 'scroll';
|
|
1614
1167
|
x: number;
|
|
@@ -1643,7 +1196,7 @@ type CaptureUse = {
|
|
|
1643
1196
|
type CaptureOptions = {
|
|
1644
1197
|
/** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
|
|
1645
1198
|
viewport?: Viewport;
|
|
1646
|
-
/** How long to poll before failing. Default `'60s'`. */
|
|
1199
|
+
/** How long to poll before failing; a 502, 503 or 504 is polled past until then. Default `'60s'`. */
|
|
1647
1200
|
timeout?: Duration;
|
|
1648
1201
|
/** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
|
|
1649
1202
|
browser?: boolean;
|
|
@@ -1657,7 +1210,7 @@ type CaptureOptions = {
|
|
|
1657
1210
|
type BackgroundHandle = {
|
|
1658
1211
|
kill(): Promise<void>;
|
|
1659
1212
|
};
|
|
1660
|
-
/** `c.serve`: a background process plus the port it
|
|
1213
|
+
/** `c.serve`: a background process plus the port it listens on. */
|
|
1661
1214
|
type ServeHandle = BackgroundHandle & {
|
|
1662
1215
|
port: number;
|
|
1663
1216
|
};
|
|
@@ -1683,6 +1236,7 @@ type Capture = {
|
|
|
1683
1236
|
* default `name` is derived from the path. `wait` and `instructions` are call-only.
|
|
1684
1237
|
*/
|
|
1685
1238
|
type CaptureCall = {
|
|
1239
|
+
/** The `artifacts/<name>.png` file name: 1 to 100 of `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. Unique within the attempt. */
|
|
1686
1240
|
name?: string;
|
|
1687
1241
|
path?: string;
|
|
1688
1242
|
viewport?: Viewport;
|
|
@@ -1769,10 +1323,10 @@ type JudgeState = JudgeJsonState | Array<{
|
|
|
1769
1323
|
type: 'json';
|
|
1770
1324
|
value: JudgeJsonState;
|
|
1771
1325
|
} | JudgeFilePart>;
|
|
1772
|
-
/** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png
|
|
1326
|
+
/** A capture this judge call sent as a file part. Bytes live in the `artifacts/<name>.png` file. */
|
|
1773
1327
|
type JudgeArtifactRef = {
|
|
1774
1328
|
name: string;
|
|
1775
|
-
/** `artifacts/<name>.png` under the attempt directory. */
|
|
1329
|
+
/** The `artifacts/<name>.png` path under the attempt directory. */
|
|
1776
1330
|
path: string;
|
|
1777
1331
|
sha256: string;
|
|
1778
1332
|
bytes: number;
|
|
@@ -1804,11 +1358,6 @@ type JudgeOptions = {
|
|
|
1804
1358
|
providerOptions?: Record<string, Record<string, JSONValue>>;
|
|
1805
1359
|
/** The most judge calls to this model's provider at once, across the run. */
|
|
1806
1360
|
maxConcurrent?: number;
|
|
1807
|
-
/**
|
|
1808
|
-
* The judge's state: what the judge is given beside the question, not the workspace's or the
|
|
1809
|
-
* attempt's state. Merged key by key down the cascade.
|
|
1810
|
-
*/
|
|
1811
|
-
state?: JudgeStateOptions;
|
|
1812
1361
|
/**
|
|
1813
1362
|
* The gateway the judge's calls go through, by name or by its helper. It skips the config's
|
|
1814
1363
|
* `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
|
|
@@ -1816,43 +1365,61 @@ type JudgeOptions = {
|
|
|
1816
1365
|
gateway?: GatewayName | GatewayDescriptor;
|
|
1817
1366
|
};
|
|
1818
1367
|
/**
|
|
1819
|
-
* What
|
|
1820
|
-
*
|
|
1368
|
+
* What one `c.judge()` call shows the judge beside the question. Every input is off unless the
|
|
1369
|
+
* call turns it on, and a call turns on at least one; each is a key of the state in
|
|
1370
|
+
* `judge-input.json`, or the captures sent as file parts.
|
|
1821
1371
|
*/
|
|
1822
|
-
type
|
|
1823
|
-
/** The graded turn's prompt.
|
|
1372
|
+
type JudgeInputs = {
|
|
1373
|
+
/** The graded turn's prompt. */
|
|
1824
1374
|
task?: boolean;
|
|
1825
|
-
/** The graded turn's final message, or `on`'s value.
|
|
1375
|
+
/** The graded turn's final message, or `on`'s value. */
|
|
1826
1376
|
output?: boolean;
|
|
1827
|
-
/** `status path` lines for the changed files, under `diff`'s patterns.
|
|
1377
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. */
|
|
1828
1378
|
files?: boolean;
|
|
1829
1379
|
/**
|
|
1830
1380
|
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
1831
|
-
* the last matching pattern wins and `!` excludes.
|
|
1381
|
+
* the last matching pattern wins and `!` excludes.
|
|
1832
1382
|
*/
|
|
1833
1383
|
diff?: boolean | string[];
|
|
1834
1384
|
/**
|
|
1835
1385
|
* Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
|
|
1836
1386
|
* patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
|
|
1837
1387
|
* the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
|
|
1838
|
-
* These files leave the diff.
|
|
1388
|
+
* These files leave the diff.
|
|
1839
1389
|
*/
|
|
1840
1390
|
contents?: string[];
|
|
1841
1391
|
/**
|
|
1842
1392
|
* The graded turn's transcript as text lines: `true` is messages and tool calls with tool
|
|
1843
1393
|
* output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
|
|
1844
|
-
* Default false.
|
|
1845
1394
|
*/
|
|
1846
1395
|
transcript?: boolean | 'tools' | 'full';
|
|
1396
|
+
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
|
|
1397
|
+
checks?: boolean | string[];
|
|
1398
|
+
/**
|
|
1399
|
+
* Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
|
|
1400
|
+
* capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
|
|
1401
|
+
*/
|
|
1402
|
+
artifacts?: true | Array<Capture | string>;
|
|
1403
|
+
};
|
|
1404
|
+
/** What a judge call's inputs read: which turn, and whether the turns before it too. */
|
|
1405
|
+
type JudgeSubject = {
|
|
1406
|
+
/** The turn `task`, `output`, `transcript` and `diff` read instead of the latest, or a value used as the output. */
|
|
1407
|
+
on?: Turn | JSONValue;
|
|
1847
1408
|
/**
|
|
1848
1409
|
* `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
|
|
1849
1410
|
* `transcript`) transcript. Default `'last'`.
|
|
1850
1411
|
*/
|
|
1851
1412
|
turns?: 'last' | 'all';
|
|
1852
|
-
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
1853
|
-
checks?: boolean | string[];
|
|
1854
1413
|
};
|
|
1855
|
-
/** `
|
|
1414
|
+
/** One input turned on: `true`, a mode or a non-empty selection, not `false`. */
|
|
1415
|
+
type OneInput<I, K extends keyof I> = { [P in K]-?: Exclude<I[P], false | undefined>; };
|
|
1416
|
+
/** At least one of `I`'s keys turned on. */
|
|
1417
|
+
type AtLeastOneInput<I> = { [K in keyof I]-?: OneInput<I, K>; }[keyof I];
|
|
1418
|
+
/** `c.judge()`'s options, before the at-least-one-input rule that `JudgeCallOptions` adds. */
|
|
1419
|
+
type JudgeCallFields = JudgeOptions & JudgeInputs & JudgeSubject;
|
|
1420
|
+
/** `c.judge()`'s options: the judge options, the inputs (at least one) and what they read. */
|
|
1421
|
+
type JudgeCallOptions = JudgeCallFields & AtLeastOneInput<JudgeInputs>;
|
|
1422
|
+
/** The inputs one judge call saw, every key set (`false` when off); `artifacts` is recorded apart. */
|
|
1856
1423
|
type ResolvedJudgeStateOptions = {
|
|
1857
1424
|
task: boolean;
|
|
1858
1425
|
output: boolean;
|
|
@@ -1863,16 +1430,6 @@ type ResolvedJudgeStateOptions = {
|
|
|
1863
1430
|
turns: 'last' | 'all';
|
|
1864
1431
|
checks: boolean | string[];
|
|
1865
1432
|
};
|
|
1866
|
-
type JudgeCallOptions = JudgeOptions & {
|
|
1867
|
-
/** What to grade instead of the latest turn: another turn, or a value used as the output. */
|
|
1868
|
-
on?: Turn | JSONValue;
|
|
1869
|
-
/**
|
|
1870
|
-
* Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
|
|
1871
|
-
* `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
|
|
1872
|
-
* captures; a provider that cannot take files fails the call.
|
|
1873
|
-
*/
|
|
1874
|
-
artifacts?: true | Array<Capture | string>;
|
|
1875
|
-
};
|
|
1876
1433
|
type Verdict = {
|
|
1877
1434
|
verdict: 'pass' | 'fail';
|
|
1878
1435
|
/** The judge's probability that the answer is yes, from 0 to 1. */
|
|
@@ -1884,6 +1441,10 @@ type Verdict = {
|
|
|
1884
1441
|
resolvedModel?: string;
|
|
1885
1442
|
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
1886
1443
|
gateway?: GatewayName;
|
|
1444
|
+
/** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
|
|
1445
|
+
upstream?: string;
|
|
1446
|
+
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
1447
|
+
costUsd?: number;
|
|
1887
1448
|
/** Captures this call sent the judge as file parts, when any. */
|
|
1888
1449
|
artifacts?: JudgeArtifactRef[];
|
|
1889
1450
|
};
|
|
@@ -1891,7 +1452,7 @@ type Verdict = {
|
|
|
1891
1452
|
* What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
|
|
1892
1453
|
* so the judges and checks after it still run.
|
|
1893
1454
|
*/
|
|
1894
|
-
type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
|
|
1455
|
+
type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel' | 'upstream' | 'costUsd'> & {
|
|
1895
1456
|
verdict: 'error';
|
|
1896
1457
|
score: null;
|
|
1897
1458
|
error: string;
|
|
@@ -1904,10 +1465,11 @@ type AttemptContext = {
|
|
|
1904
1465
|
/** Sends the next turn of this eval's agent session. */
|
|
1905
1466
|
send(prompt: string): Promise<Turn>;
|
|
1906
1467
|
/**
|
|
1907
|
-
* Asks the judge a yes/no question about
|
|
1908
|
-
*
|
|
1468
|
+
* Asks the judge a yes/no question about what `options` turns on, and nothing else: at least
|
|
1469
|
+
* one of `task`, `output`, `files`, `diff`, `contents`, `transcript`, `checks` or `artifacts`.
|
|
1470
|
+
* They read the latest turn, or `options.on`.
|
|
1909
1471
|
*/
|
|
1910
|
-
judge(question: string, options
|
|
1472
|
+
judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1911
1473
|
/**
|
|
1912
1474
|
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1913
1475
|
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
@@ -1922,8 +1484,10 @@ type AttemptContext = {
|
|
|
1922
1484
|
background?: boolean;
|
|
1923
1485
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1924
1486
|
/**
|
|
1925
|
-
* Starts `command` in the background
|
|
1926
|
-
* and
|
|
1487
|
+
* Starts `command` in the background with `PORT` set to `port`, or to a port reeval allocates,
|
|
1488
|
+
* and returns once it listens. Without `port`, a command that listens on one new port instead
|
|
1489
|
+
* (Vite's 5173) gets that one; several new ports, none of them `PORT`, or an exit first, throw
|
|
1490
|
+
* with its last output.
|
|
1927
1491
|
*/
|
|
1928
1492
|
serve(command: string, port?: number): Promise<ServeHandle>;
|
|
1929
1493
|
/**
|
|
@@ -1966,7 +1530,7 @@ type FakeHarnessDescriptor = {
|
|
|
1966
1530
|
files?: Record<string, string>;
|
|
1967
1531
|
/** The fake agent's final message on every turn; default `Done.`. */
|
|
1968
1532
|
text?: string;
|
|
1969
|
-
/** Simulated turn length,
|
|
1533
|
+
/** Simulated turn length, honors the abort signal. */
|
|
1970
1534
|
delayMs?: number;
|
|
1971
1535
|
/** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
|
|
1972
1536
|
ignoreAbort?: boolean;
|
|
@@ -2243,7 +1807,7 @@ type CodexHarnessDescriptor = {
|
|
|
2243
1807
|
};
|
|
2244
1808
|
/**
|
|
2245
1809
|
* fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
|
|
2246
|
-
* OpenAI's API by the model id, and reports each turn's tokens.
|
|
1810
|
+
* OpenAI's API by the model id, or on OpenRouter or Vertex AI, and reports each turn's tokens.
|
|
2247
1811
|
*/
|
|
2248
1812
|
type FxHarnessDescriptor = {
|
|
2249
1813
|
kind: 'fx';
|
|
@@ -2255,10 +1819,18 @@ type FxHarnessDescriptor = {
|
|
|
2255
1819
|
/**
|
|
2256
1820
|
* A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
|
|
2257
1821
|
* `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
|
|
2258
|
-
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
|
|
1822
|
+
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`. On `openrouter()`, either
|
|
1823
|
+
* runs on OpenRouter's Chat Completions API with `OPENROUTER_API_KEY`, a bare id as `openai/<id>`.
|
|
1824
|
+
* On `vertex()`, a Gemini model id, `google/<id>` or a bare `gemini-…` id, run on Vertex AI's
|
|
1825
|
+
* Chat Completions endpoint, e.g. `google/gemini-3.8-flash`.
|
|
2259
1826
|
*/
|
|
2260
1827
|
model: string;
|
|
2261
|
-
/**
|
|
1828
|
+
/**
|
|
1829
|
+
* Reasoning effort, sent as `reasoning_effort` on each model call; only on `vertex()`, since fx
|
|
1830
|
+
* sends none itself. Values are what the model takes there, e.g. `low`, `medium` or `high`.
|
|
1831
|
+
*/
|
|
1832
|
+
effort?: string;
|
|
1833
|
+
/** Cell id; defaults to `fx:<model>`, or `fx:<model>[effort=<value>]` when `effort` is set. */
|
|
2262
1834
|
id?: string;
|
|
2263
1835
|
/** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
|
|
2264
1836
|
maxConcurrent?: number;
|
|
@@ -2298,12 +1870,21 @@ type GrokBuildHarnessDescriptor = {
|
|
|
2298
1870
|
};
|
|
2299
1871
|
/**
|
|
2300
1872
|
* A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
|
|
2301
|
-
* `vercel` is Vercel AI Gateway
|
|
1873
|
+
* `vercel` is Vercel AI Gateway; `openrouter` is OpenRouter; `vertex` is
|
|
1874
|
+
* Google Cloud's Vertex AI.
|
|
1875
|
+
*/
|
|
1876
|
+
type GatewayName = 'direct' | 'vercel' | 'openrouter' | 'vertex';
|
|
1877
|
+
/**
|
|
1878
|
+
* Each provider's own key: made by `direct()` in `reeval/gateway`. Every `gateway` list ends with
|
|
1879
|
+
* it unless it's listed elsewhere, where it's tried in that place.
|
|
2302
1880
|
*/
|
|
2303
|
-
type GatewayName = 'direct' | 'vercel';
|
|
2304
|
-
/** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
|
|
2305
1881
|
type DirectGatewayDescriptor = {
|
|
2306
1882
|
kind: 'direct';
|
|
1883
|
+
/**
|
|
1884
|
+
* `false` turns off the fallback to each provider's own key, so a cell no other gateway in the
|
|
1885
|
+
* list serves is refused at plan time. Default `true`.
|
|
1886
|
+
*/
|
|
1887
|
+
enabled?: boolean;
|
|
2307
1888
|
};
|
|
2308
1889
|
/**
|
|
2309
1890
|
* Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
|
|
@@ -2312,8 +1893,37 @@ type DirectGatewayDescriptor = {
|
|
|
2312
1893
|
type VercelGatewayDescriptor = {
|
|
2313
1894
|
kind: 'vercel';
|
|
2314
1895
|
};
|
|
1896
|
+
/**
|
|
1897
|
+
* OpenRouter, with `OPENROUTER_API_KEY`: made by `openrouter()` in `reeval/gateway`. It serves
|
|
1898
|
+
* Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi.
|
|
1899
|
+
*/
|
|
1900
|
+
type OpenRouterGatewayDescriptor = {
|
|
1901
|
+
kind: 'openrouter';
|
|
1902
|
+
/**
|
|
1903
|
+
* Let OpenRouter fail over to another upstream provider when the one it picked can't serve a
|
|
1904
|
+
* call. Off by default, so reeval's own calls (the judge, analysis, compare questions and
|
|
1905
|
+
* captures) send `allow_fallbacks: false`. The harnesses' calls take OpenRouter's own routing
|
|
1906
|
+
* either way.
|
|
1907
|
+
*/
|
|
1908
|
+
fallbacks?: boolean;
|
|
1909
|
+
};
|
|
1910
|
+
/** A model publisher on Vertex AI, which may serve its models in a region of its own. */
|
|
1911
|
+
type VertexPublisher = 'anthropic' | 'google' | 'xai';
|
|
1912
|
+
/**
|
|
1913
|
+
* Google Cloud's Vertex AI, on Application Default Credentials: made by `vertex()` in
|
|
1914
|
+
* `reeval/gateway`.
|
|
1915
|
+
*/
|
|
1916
|
+
type VertexGatewayDescriptor = {
|
|
1917
|
+
kind: 'vertex';
|
|
1918
|
+
/** The Google Cloud project the calls are billed to; defaults to `GOOGLE_CLOUD_PROJECT`. */
|
|
1919
|
+
project?: string;
|
|
1920
|
+
/** The region the models are called in, e.g. `us-east5`; defaults to `global`. */
|
|
1921
|
+
location?: string;
|
|
1922
|
+
/** A region per publisher, where it differs from `location`, e.g. `{ anthropic: 'us-east5' }`. */
|
|
1923
|
+
regions?: Partial<Record<VertexPublisher, string>>;
|
|
1924
|
+
};
|
|
2315
1925
|
/** One entry of the config's `gateway` list. */
|
|
2316
|
-
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
|
|
1926
|
+
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor | OpenRouterGatewayDescriptor | VertexGatewayDescriptor;
|
|
2317
1927
|
/** A real coding agent, driven through its AI SDK harness adapter. */
|
|
2318
1928
|
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
|
|
2319
1929
|
/** The agent an attempt runs: a real harness, or the fake one. */
|
|
@@ -2818,9 +2428,11 @@ type CompareOptions<E = Environment> = {
|
|
|
2818
2428
|
* With `across`, it picks between two attempts that differ only in that factor, in both orders;
|
|
2819
2429
|
* without it, it's asked of each attempt on its own. Without a model, the cascaded
|
|
2820
2430
|
* `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
|
|
2821
|
-
*
|
|
2431
|
+
* It's shown only the inputs it turns on, at least one.
|
|
2822
2432
|
*/
|
|
2823
|
-
type Question =
|
|
2433
|
+
type Question = QuestionFields & AtLeastOneInput<QuestionInputs>;
|
|
2434
|
+
/** A question's options, before the at-least-one-input rule that `Question` adds. */
|
|
2435
|
+
type QuestionFields = QuestionInputs & {
|
|
2824
2436
|
question: string;
|
|
2825
2437
|
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
2826
2438
|
model?: string;
|
|
@@ -2831,43 +2443,39 @@ type Question = string | {
|
|
|
2831
2443
|
across?: PairFactor | PairFactor[];
|
|
2832
2444
|
/** The answers a question asked of each attempt picks from, with a short reason. */
|
|
2833
2445
|
choices?: string[];
|
|
2834
|
-
/**
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2446
|
+
/**
|
|
2447
|
+
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2448
|
+
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2449
|
+
*/
|
|
2450
|
+
turns?: 'last' | 'all';
|
|
2838
2451
|
};
|
|
2839
2452
|
/**
|
|
2840
|
-
* What a question is shown of each attempt:
|
|
2841
|
-
*
|
|
2842
|
-
*
|
|
2843
|
-
* message and the transcript.
|
|
2453
|
+
* What a question is shown of each attempt: `c.judge()`'s inputs but `contents` (the attempts'
|
|
2454
|
+
* workspaces are gone when questions are asked). Every input is off unless the question turns it
|
|
2455
|
+
* on, and a question turns on at least one.
|
|
2844
2456
|
*/
|
|
2845
|
-
type
|
|
2846
|
-
/** The prompts the attempt was sent, once for both sides when they match.
|
|
2457
|
+
type QuestionInputs = {
|
|
2458
|
+
/** The prompts the attempt was sent, once for both sides when they match. */
|
|
2847
2459
|
task?: boolean;
|
|
2848
|
-
/** The final message: the agent's own account of its work.
|
|
2460
|
+
/** The final message: the agent's own account of its work. */
|
|
2849
2461
|
output?: boolean;
|
|
2850
|
-
/** `status path` lines for the changed files, under `diff`'s patterns.
|
|
2462
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. */
|
|
2851
2463
|
files?: boolean;
|
|
2852
2464
|
/**
|
|
2853
2465
|
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
2854
|
-
* the last matching pattern wins and `!` excludes.
|
|
2466
|
+
* the last matching pattern wins and `!` excludes.
|
|
2855
2467
|
*/
|
|
2856
2468
|
diff?: boolean | string[];
|
|
2857
2469
|
/**
|
|
2858
2470
|
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
2859
2471
|
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2860
|
-
* harness's tools, so a pair of two harnesses isn't blind.
|
|
2861
|
-
* attempt.
|
|
2472
|
+
* harness's tools, so a pair of two harnesses isn't blind.
|
|
2862
2473
|
*/
|
|
2863
2474
|
transcript?: boolean | 'tools' | 'full';
|
|
2864
|
-
/**
|
|
2865
|
-
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2866
|
-
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2867
|
-
*/
|
|
2868
|
-
turns?: 'last' | 'all';
|
|
2869
|
-
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
2475
|
+
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
|
|
2870
2476
|
checks?: boolean | string[];
|
|
2477
|
+
/** Each attempt's own captures, by name; `true` sends all of them. */
|
|
2478
|
+
artifacts?: true | string[];
|
|
2871
2479
|
};
|
|
2872
2480
|
/** `QuestionStateOptions` with every key set: what each side of one question showed. */
|
|
2873
2481
|
type ResolvedQuestionStateOptions = {
|
|
@@ -2879,20 +2487,16 @@ type ResolvedQuestionStateOptions = {
|
|
|
2879
2487
|
turns: 'last' | 'all';
|
|
2880
2488
|
checks: boolean | string[];
|
|
2881
2489
|
};
|
|
2882
|
-
/** Defaults for every question
|
|
2490
|
+
/** Defaults for every question below it. What a question is shown is its own, never a default. */
|
|
2883
2491
|
type QuestionOptions = {
|
|
2884
2492
|
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2885
2493
|
model?: string;
|
|
2886
|
-
/** What it's shown of each attempt. */
|
|
2887
|
-
state?: QuestionStateOptions;
|
|
2888
|
-
/** Each attempt's own captures, by name; `true` sends all of them. Default none. */
|
|
2889
|
-
artifacts?: true | string[];
|
|
2890
2494
|
};
|
|
2891
2495
|
/** A matrix factor a pairwise question varies; the others are held. */
|
|
2892
2496
|
type PairFactor = 'environment' | 'harness';
|
|
2893
2497
|
/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
|
|
2894
2498
|
type CompareCut = 'within' | 'across';
|
|
2895
|
-
/** One
|
|
2499
|
+
/** One judgment of one pair of attempts, in one order. */
|
|
2896
2500
|
type PairVote = {
|
|
2897
2501
|
question: string;
|
|
2898
2502
|
cut: CompareCut;
|
|
@@ -2924,7 +2528,7 @@ type CompareAnswer = {
|
|
|
2924
2528
|
model: string;
|
|
2925
2529
|
/** The model id the provider says served the calls, when it reports one. */
|
|
2926
2530
|
resolvedModel?: string;
|
|
2927
|
-
/**
|
|
2531
|
+
/** The inputs each side showed, every key resolved; unset on answers recorded before reeval wrote it, which showed the task and the diff. */
|
|
2928
2532
|
stateOptions?: ResolvedQuestionStateOptions;
|
|
2929
2533
|
/** The captures it sent of each side, as asked: `true` for all, or their names. */
|
|
2930
2534
|
artifacts?: true | string[];
|
|
@@ -2977,8 +2581,21 @@ type GatePolicy = 'all' | {
|
|
|
2977
2581
|
type CellGate = GatePolicy | 'none';
|
|
2978
2582
|
/** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
|
|
2979
2583
|
type CompareSpec<E = Environment> = CompareOptions<E>;
|
|
2584
|
+
type OddDigit = '1' | '3' | '5' | '7' | '9';
|
|
2585
|
+
/**
|
|
2586
|
+
* `N` when it's an odd whole number of 3 or more, else a sentence the compiler shows as the type
|
|
2587
|
+
* it expected. A `number` that isn't a literal stays `number`: only planning can check it.
|
|
2588
|
+
*/
|
|
2589
|
+
type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` | '1' ? 'an odd whole number of 3 or more, e.g. 3 or 5' : `${N}` extends `${string}${OddDigit}` ? N : 'an odd whole number of 3 or more, e.g. 3 or 5';
|
|
2590
|
+
/**
|
|
2591
|
+
* Attempts per cell: a fixed count, or `{ bestOf }`, an odd number of 3 or more, which stops a
|
|
2592
|
+
* cell once one side (pass or fail) has a majority: `{ bestOf: 5 }` runs 3 to 5 attempts.
|
|
2593
|
+
*/
|
|
2594
|
+
type Runs<N extends number = number> = number | {
|
|
2595
|
+
bestOf: OddAtLeast3<N>;
|
|
2596
|
+
};
|
|
2980
2597
|
/** Options other than `environment` and `compare`, which are exclusive of each other. */
|
|
2981
|
-
type EvalOptionFields = {
|
|
2598
|
+
type EvalOptionFields<N extends number = number> = {
|
|
2982
2599
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2983
2600
|
harness?: HarnessOption;
|
|
2984
2601
|
/**
|
|
@@ -2986,8 +2603,13 @@ type EvalOptionFields = {
|
|
|
2986
2603
|
* Not an environment: it does not invent skills, files or a name.
|
|
2987
2604
|
*/
|
|
2988
2605
|
sandbox?: SandboxDescriptor;
|
|
2989
|
-
/**
|
|
2990
|
-
|
|
2606
|
+
/**
|
|
2607
|
+
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
|
|
2608
|
+
* a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
|
|
2609
|
+
* Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
|
|
2610
|
+
* a verdict, and anything else runs once, a pooled `evaluate.each` case included.
|
|
2611
|
+
*/
|
|
2612
|
+
runs?: Runs<N>;
|
|
2991
2613
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2992
2614
|
gate?: GatePolicy;
|
|
2993
2615
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
@@ -3001,7 +2623,7 @@ type EvalOptionFields = {
|
|
|
3001
2623
|
judge?: JudgeOptions;
|
|
3002
2624
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
3003
2625
|
capture?: CaptureOptions;
|
|
3004
|
-
/**
|
|
2626
|
+
/** The model every question that names none of its own is answered by, down the cascade. */
|
|
3005
2627
|
question?: QuestionOptions;
|
|
3006
2628
|
/**
|
|
3007
2629
|
* What a language model is asked about the attempts, always a list; they never change pass,
|
|
@@ -3012,11 +2634,32 @@ type EvalOptionFields = {
|
|
|
3012
2634
|
questions?: Question[];
|
|
3013
2635
|
/** Per attempt. */
|
|
3014
2636
|
timeout?: Duration;
|
|
2637
|
+
/**
|
|
2638
|
+
* When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
|
|
2639
|
+
* the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
|
|
2640
|
+
*/
|
|
2641
|
+
snapshot?: SnapshotOption;
|
|
3015
2642
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
3016
2643
|
tags?: string[];
|
|
2644
|
+
/**
|
|
2645
|
+
* The eval's version, part of its fingerprint: bump it when a change the eval file doesn't show
|
|
2646
|
+
* (a helper, a fixture, data it reads) changes what it measures. A group's applies to the evals
|
|
2647
|
+
* inside it that set none.
|
|
2648
|
+
*/
|
|
2649
|
+
revision?: Revision;
|
|
3017
2650
|
/** Keys a profile's `limit` may not narrow for this group or eval. */
|
|
3018
2651
|
pin?: PinnableKey[];
|
|
3019
2652
|
};
|
|
2653
|
+
/** An eval's `revision`: a whole number or a non-empty string, compared exactly. */
|
|
2654
|
+
type Revision = number | string;
|
|
2655
|
+
/**
|
|
2656
|
+
* An eval's own source as hashes: its body and case data as declared, and its options and its
|
|
2657
|
+
* enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
|
|
2658
|
+
*/
|
|
2659
|
+
type EvalSource = DeclaredSource & {
|
|
2660
|
+
/** sha256 of the canonical options. */
|
|
2661
|
+
options: string;
|
|
2662
|
+
};
|
|
3020
2663
|
/**
|
|
3021
2664
|
* `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
|
|
3022
2665
|
* A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
|
|
@@ -3042,8 +2685,9 @@ type EnvironmentOrCompare<E> = {
|
|
|
3042
2685
|
baseline?: undefined;
|
|
3043
2686
|
};
|
|
3044
2687
|
};
|
|
3045
|
-
|
|
3046
|
-
type
|
|
2688
|
+
/** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
|
|
2689
|
+
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
|
|
2690
|
+
type GroupOptions<N extends number = number> = EvalOptions<N>;
|
|
3047
2691
|
/**
|
|
3048
2692
|
* `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
|
|
3049
2693
|
* by name or a fake judge, and environments without `extend`. The same `environment` / compare
|
|
@@ -3065,7 +2709,7 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
|
3065
2709
|
type HistoryConfig = {
|
|
3066
2710
|
/** Committed history folder (default: `.history` in the directory the evals live in). */
|
|
3067
2711
|
dir?: string;
|
|
3068
|
-
/** Artifacts recorded next to each run: every
|
|
2712
|
+
/** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
|
|
3069
2713
|
artifacts?: ArtifactLevel;
|
|
3070
2714
|
};
|
|
3071
2715
|
/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
|
|
@@ -3083,8 +2727,8 @@ type ProfileDefaults = {
|
|
|
3083
2727
|
sandbox?: SandboxDescriptor;
|
|
3084
2728
|
/** What every eval runs in when it names no environment. */
|
|
3085
2729
|
environment?: Environment;
|
|
3086
|
-
/** Attempts per cell. */
|
|
3087
|
-
runs?:
|
|
2730
|
+
/** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
|
|
2731
|
+
runs?: Runs;
|
|
3088
2732
|
/** How each cell's attempts decide its outcome. */
|
|
3089
2733
|
gate?: GatePolicy;
|
|
3090
2734
|
/** Variables and secrets for the sandbox, merged by name over the config's. */
|
|
@@ -3099,6 +2743,8 @@ type ProfileDefaults = {
|
|
|
3099
2743
|
capture?: CaptureOptions;
|
|
3100
2744
|
/** Defaults for every question, merged over the config's. */
|
|
3101
2745
|
question?: QuestionOptions;
|
|
2746
|
+
/** When each attempt's workspace is copied; merged over the config's. */
|
|
2747
|
+
snapshot?: SnapshotOption;
|
|
3102
2748
|
};
|
|
3103
2749
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
3104
2750
|
type ProfileLimit = {
|
|
@@ -3148,74 +2794,6 @@ type FakeAnalysisModel = {
|
|
|
3148
2794
|
answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
|
|
3149
2795
|
error: string;
|
|
3150
2796
|
};
|
|
3151
|
-
/**
|
|
3152
|
-
* What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
|
|
3153
|
-
* function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
|
|
3154
|
-
* or `headline` once; it may return a promise), or `{ error }` to fail every call. It counts a
|
|
3155
|
-
* token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
|
|
3156
|
-
*/
|
|
3157
|
-
insights?: FakeInsights | ((input: FakeInsightsInput) => FakeInsights | Promise<FakeInsights>) | {
|
|
3158
|
-
error: string;
|
|
3159
|
-
};
|
|
3160
|
-
};
|
|
3161
|
-
/** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
|
|
3162
|
-
type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
|
|
3163
|
-
/**
|
|
3164
|
-
* A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
|
|
3165
|
-
* `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
|
|
3166
|
-
* left out is empty, and the intent defaults to `job: 'other'`.
|
|
3167
|
-
*/
|
|
3168
|
-
type FakeInsights = {
|
|
3169
|
-
intent?: {
|
|
3170
|
-
job: 'choose' | 'finish' | 'other';
|
|
3171
|
-
question?: string;
|
|
3172
|
-
subject?: {
|
|
3173
|
-
tool?: string;
|
|
3174
|
-
change?: string;
|
|
3175
|
-
};
|
|
3176
|
-
name?: 'states-intent' | 'no-intent' | 'contradicts';
|
|
3177
|
-
/** `low` falls back to `job: 'other'`, as a real classification does. */
|
|
3178
|
-
confidence?: 'high' | 'medium' | 'low';
|
|
3179
|
-
};
|
|
3180
|
-
answer?: {
|
|
3181
|
-
text: string;
|
|
3182
|
-
cites: string[];
|
|
3183
|
-
} | null;
|
|
3184
|
-
findings?: Array<{
|
|
3185
|
-
text: string;
|
|
3186
|
-
cites: string[];
|
|
3187
|
-
}>;
|
|
3188
|
-
rows?: Array<{
|
|
3189
|
-
row: string;
|
|
3190
|
-
text: string;
|
|
3191
|
-
cites: string[];
|
|
3192
|
-
}>;
|
|
3193
|
-
namedTools?: Array<{
|
|
3194
|
-
attempt: string;
|
|
3195
|
-
tools: string[];
|
|
3196
|
-
}>;
|
|
3197
|
-
/**
|
|
3198
|
-
* The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
|
|
3199
|
-
* `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
|
|
3200
|
-
* They go through the same gates as a model's, so one missing a part is dropped.
|
|
3201
|
-
*/
|
|
3202
|
-
remarks?: Array<{
|
|
3203
|
-
kind: string;
|
|
3204
|
-
text: string;
|
|
3205
|
-
source?: {
|
|
3206
|
-
at: string;
|
|
3207
|
-
quote: string;
|
|
3208
|
-
} | null;
|
|
3209
|
-
evidence?: string[];
|
|
3210
|
-
fix?: {
|
|
3211
|
-
at: string;
|
|
3212
|
-
edit: string;
|
|
3213
|
-
} | null;
|
|
3214
|
-
}>;
|
|
3215
|
-
headline?: {
|
|
3216
|
-
text: string;
|
|
3217
|
-
cites: string[];
|
|
3218
|
-
} | null;
|
|
3219
2797
|
};
|
|
3220
2798
|
/** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
|
|
3221
2799
|
type FakeVote = 'A' | 'B' | 'tie';
|
|
@@ -3243,15 +2821,15 @@ type BlindAttempt = {
|
|
|
3243
2821
|
choices?: string[];
|
|
3244
2822
|
attempt: BlindSide;
|
|
3245
2823
|
};
|
|
3246
|
-
/** A turn of one side, with the
|
|
2824
|
+
/** A turn of one side, with the inputs its question turned on. */
|
|
3247
2825
|
type BlindTurn = {
|
|
3248
2826
|
task?: string;
|
|
3249
2827
|
output?: string;
|
|
3250
2828
|
transcript?: string;
|
|
3251
2829
|
};
|
|
3252
2830
|
/**
|
|
3253
|
-
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the
|
|
3254
|
-
* its question
|
|
2831
|
+
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the inputs
|
|
2832
|
+
* its question turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
|
|
3255
2833
|
* under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
|
|
3256
2834
|
*/
|
|
3257
2835
|
type BlindSide = BlindTurn & {
|
|
@@ -3269,7 +2847,7 @@ type BlindSide = BlindTurn & {
|
|
|
3269
2847
|
command: string;
|
|
3270
2848
|
notRun: true;
|
|
3271
2849
|
}>;
|
|
3272
|
-
/** Its captures, in the order sent,
|
|
2850
|
+
/** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
|
|
3273
2851
|
artifacts?: Array<{
|
|
3274
2852
|
label: string;
|
|
3275
2853
|
mediaType: 'image/png';
|
|
@@ -3279,24 +2857,13 @@ type BlindSide = BlindTurn & {
|
|
|
3279
2857
|
missing: true;
|
|
3280
2858
|
}>;
|
|
3281
2859
|
};
|
|
3282
|
-
/**
|
|
2860
|
+
/** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
|
|
3283
2861
|
type AnalysisConfig = {
|
|
3284
2862
|
/**
|
|
3285
|
-
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`.
|
|
3286
|
-
*
|
|
2863
|
+
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Unset, `reeval analyze` and questions
|
|
2864
|
+
* fall back to the judge's when it's a language model.
|
|
3287
2865
|
*/
|
|
3288
2866
|
model?: string | FakeAnalysisModel;
|
|
3289
|
-
/**
|
|
3290
|
-
* `reeval run` classifies each eval and explains its result with `model` once the attempts
|
|
3291
|
-
* finish (default true when `model` is set; `--no-insights` turns it off for one run).
|
|
3292
|
-
*/
|
|
3293
|
-
insights?: boolean;
|
|
3294
|
-
/**
|
|
3295
|
-
* Tokens (in and out) that one run's insights may use, over every classification, insight and
|
|
3296
|
-
* headline call (default 250000). Past it, the evals left print without insights; the run's
|
|
3297
|
-
* outcome and exit code never change.
|
|
3298
|
-
*/
|
|
3299
|
-
maxTokens?: number;
|
|
3300
2867
|
/** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
|
|
3301
2868
|
instructions?: string;
|
|
3302
2869
|
/** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
|
|
@@ -3316,8 +2883,11 @@ type ReevalConfig = {
|
|
|
3316
2883
|
sandbox?: SandboxDescriptor;
|
|
3317
2884
|
/** What every eval runs in when it names no environment, and a baseline-less compare's control. */
|
|
3318
2885
|
environment?: Environment;
|
|
3319
|
-
/**
|
|
3320
|
-
|
|
2886
|
+
/**
|
|
2887
|
+
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
|
|
2888
|
+
* variants runs 4 a side and anything else once.
|
|
2889
|
+
*/
|
|
2890
|
+
runs?: Runs;
|
|
3321
2891
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
3322
2892
|
gate?: GatePolicy;
|
|
3323
2893
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
@@ -3335,6 +2905,11 @@ type ReevalConfig = {
|
|
|
3335
2905
|
capture?: CaptureOptions;
|
|
3336
2906
|
/** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
|
|
3337
2907
|
question?: QuestionOptions;
|
|
2908
|
+
/**
|
|
2909
|
+
* When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
|
|
2910
|
+
* the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
|
|
2911
|
+
*/
|
|
2912
|
+
snapshot?: SnapshotOption;
|
|
3338
2913
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
3339
2914
|
concurrency?: number;
|
|
3340
2915
|
/** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
|
|
@@ -3350,7 +2925,7 @@ type ReevalConfig = {
|
|
|
3350
2925
|
ci?: string;
|
|
3351
2926
|
local?: string;
|
|
3352
2927
|
};
|
|
3353
|
-
/** `unbrokered: 'error'` refuses Docker secrets without `hosts
|
|
2928
|
+
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
|
|
3354
2929
|
secrets?: {
|
|
3355
2930
|
unbrokered?: 'warn' | 'error';
|
|
3356
2931
|
};
|
|
@@ -3362,11 +2937,12 @@ type ReevalConfig = {
|
|
|
3362
2937
|
keep?: number;
|
|
3363
2938
|
traceBufferMb?: number;
|
|
3364
2939
|
};
|
|
3365
|
-
/**
|
|
2940
|
+
/** `reeval analyze`, and questions without a model of their own. */
|
|
3366
2941
|
analysis?: AnalysisConfig;
|
|
3367
2942
|
/**
|
|
3368
2943
|
* The gateways model calls may go through, in the order you prefer them: each harness takes
|
|
3369
|
-
* the first that can serve it.
|
|
2944
|
+
* the first that can serve it. `direct()`, each provider's own key, ends the list unless it's listed
|
|
2945
|
+
* in another place or turned off with `direct({ enabled: false })`. Default `[direct()]`.
|
|
3370
2946
|
*/
|
|
3371
2947
|
gateway?: GatewayDescriptor[];
|
|
3372
2948
|
};
|
|
@@ -3418,9 +2994,9 @@ type AttemptError = {
|
|
|
3418
2994
|
};
|
|
3419
2995
|
/**
|
|
3420
2996
|
* One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
|
|
3421
|
-
* reeval's MCP timer, which every stdio server runs through. An agent CLI that
|
|
3422
|
-
* turn
|
|
3423
|
-
*
|
|
2997
|
+
* reeval's MCP timer, which every stdio server runs through. An agent CLI that starts one process
|
|
2998
|
+
* per turn starts its servers on every turn; a CLI that keeps a session starts them on the first
|
|
2999
|
+
* turn only. A server that doesn't answer within 60s fails the attempt as an infra
|
|
3424
3000
|
* error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
|
|
3425
3001
|
* environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
|
|
3426
3002
|
*/
|
|
@@ -3433,7 +3009,7 @@ type McpStartup = {
|
|
|
3433
3009
|
*/
|
|
3434
3010
|
ms: number | null;
|
|
3435
3011
|
};
|
|
3436
|
-
/**
|
|
3012
|
+
/** The duration of each evaluation phase, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
|
|
3437
3013
|
type PhaseTimings = {
|
|
3438
3014
|
setupMs: number;
|
|
3439
3015
|
agentMs: number;
|
|
@@ -3443,16 +3019,28 @@ type PhaseTimings = {
|
|
|
3443
3019
|
};
|
|
3444
3020
|
type CellRole = 'control' | 'variant';
|
|
3445
3021
|
/**
|
|
3446
|
-
*
|
|
3447
|
-
*
|
|
3448
|
-
*
|
|
3449
|
-
*
|
|
3022
|
+
* A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
|
|
3023
|
+
* environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
|
|
3024
|
+
* environment, or an empty `baseline`) and each variant is that environment on its own,
|
|
3025
|
+
* including its sandbox.
|
|
3450
3026
|
*/
|
|
3451
3027
|
type Cell = {
|
|
3028
|
+
/** The harness id, the environment name, and the sandbox id, joined with `/`. */
|
|
3452
3029
|
key: string;
|
|
3030
|
+
/**
|
|
3031
|
+
* The harness id: `id` when set, otherwise `<kind>:<model>` (`<kind>` alone without a model),
|
|
3032
|
+
* with `[effort=<value>]` appended whenever `effort` is set.
|
|
3033
|
+
*/
|
|
3453
3034
|
harness: string;
|
|
3454
|
-
/** The environment's own name, also on a variant cell (`with-
|
|
3035
|
+
/** The environment's own name, also on a variant cell (`with-skill`, with no prefix). */
|
|
3455
3036
|
environment: string;
|
|
3037
|
+
/**
|
|
3038
|
+
* The sandbox id is `id` when set, or `fake` for the fake sandbox. A Docker sandbox starts with
|
|
3039
|
+
* the engine (`docker` by default). It adds `:` and the sanitized final component of `image` or
|
|
3040
|
+
* `from`, or for `dockerfile` the sanitized filename (the parent directory name when that
|
|
3041
|
+
* filename is `Dockerfile` and a parent is present). Non-empty `install` steps append `+` and an
|
|
3042
|
+
* eight-character hash.
|
|
3043
|
+
*/
|
|
3456
3044
|
sandbox: string;
|
|
3457
3045
|
role: CellRole;
|
|
3458
3046
|
/** The variant's name, for variant cells. */
|
|
@@ -3495,9 +3083,9 @@ type EnvironmentRecord = {
|
|
|
3495
3083
|
name: string;
|
|
3496
3084
|
/** The previous `extendEnvironment` layer's name, else null. */
|
|
3497
3085
|
parent: string | null;
|
|
3498
|
-
/** Keys a later layer replaced,
|
|
3086
|
+
/** Keys a later layer replaced, for example `env.KEY`, `skills.typescript`, `mcp.docs`, or `rules`. */
|
|
3499
3087
|
overrides: string[];
|
|
3500
|
-
/** Covers every field but the rules file's path; secret names are included
|
|
3088
|
+
/** Covers every field but the rules file's path; secret names are included; values never are. The image is the sandbox's. */
|
|
3501
3089
|
digest: string;
|
|
3502
3090
|
setup: string[];
|
|
3503
3091
|
skills: Array<{
|
|
@@ -3511,7 +3099,7 @@ type EnvironmentRecord = {
|
|
|
3511
3099
|
/** The plugin the skill came with; unset for a skill from `skills`. */
|
|
3512
3100
|
plugin?: string;
|
|
3513
3101
|
}>;
|
|
3514
|
-
/**
|
|
3102
|
+
/** The names of the MCP servers the agent was given. */
|
|
3515
3103
|
mcp: string[];
|
|
3516
3104
|
/** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
|
|
3517
3105
|
plugins?: PluginRecord[];
|
|
@@ -3520,17 +3108,25 @@ type EnvironmentRecord = {
|
|
|
3520
3108
|
sha256: string;
|
|
3521
3109
|
}>;
|
|
3522
3110
|
vars: string[];
|
|
3111
|
+
/**
|
|
3112
|
+
* Every secret, by the variable it's set as in the sandbox; an MCP header's (`secret.fromEnv`,
|
|
3113
|
+
* `secret.template`, a plugin's `${VAR}`) by each host variable it reads.
|
|
3114
|
+
*/
|
|
3523
3115
|
secrets: string[];
|
|
3524
|
-
/**
|
|
3116
|
+
/**
|
|
3117
|
+
* Secrets the sandbox only had placeholders for, swapped in by the egress gateway, named as in
|
|
3118
|
+
* `secrets`. A variable it also got as its real value somewhere (an `http://` header, an env
|
|
3119
|
+
* secret without `hosts`) isn't listed.
|
|
3120
|
+
*/
|
|
3525
3121
|
brokered: string[];
|
|
3526
3122
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3527
3123
|
unset: string[];
|
|
3528
3124
|
/** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
|
|
3529
3125
|
egress: EgressPolicy;
|
|
3530
3126
|
/**
|
|
3531
|
-
*
|
|
3532
|
-
* project root), the ref and path asked for (null when unset), the commit the
|
|
3533
|
-
* and the hash of the tree that was copied in, which is in the digest.
|
|
3127
|
+
* This records the Git repository the workspace started as: the repo as written (a local one
|
|
3128
|
+
* relative to the project root), the ref and path asked for (null when unset), the commit the
|
|
3129
|
+
* ref resolved to, and the hash of the tree that was copied in, which is in the digest.
|
|
3534
3130
|
*/
|
|
3535
3131
|
workspace?: {
|
|
3536
3132
|
source: string;
|
|
@@ -3547,8 +3143,8 @@ type EnvironmentRecord = {
|
|
|
3547
3143
|
* and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
|
|
3548
3144
|
* can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3549
3145
|
* the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3550
|
-
* harness always tries (its CLI's telemetry, say), kept out of `blocked
|
|
3551
|
-
*
|
|
3146
|
+
* harness always tries (its CLI's telemetry, say), kept out of `blocked`, so `blocked` contains
|
|
3147
|
+
* only unexpected refusals.
|
|
3552
3148
|
*/
|
|
3553
3149
|
type EgressRecord = {
|
|
3554
3150
|
policy: EgressPolicy;
|
|
@@ -3586,7 +3182,7 @@ type AttemptImage = {
|
|
|
3586
3182
|
/** Content-addressed image id (`sha256:…`). */
|
|
3587
3183
|
id: string;
|
|
3588
3184
|
agentCli: string | null;
|
|
3589
|
-
/** The harness image `install`
|
|
3185
|
+
/** The harness image the `install` layers were baked onto: `ref` itself without any. */
|
|
3590
3186
|
base: string;
|
|
3591
3187
|
/** The `docker({ install })` commands, in order. */
|
|
3592
3188
|
install: string[];
|
|
@@ -3611,11 +3207,36 @@ type StartRetry = {
|
|
|
3611
3207
|
/** How long it waited before giving up on that start. */
|
|
3612
3208
|
waitedMs: number;
|
|
3613
3209
|
};
|
|
3614
|
-
/**
|
|
3210
|
+
/**
|
|
3211
|
+
* A process the setup steps left running in a Docker sandbox with a `to: 'checks'` secret without
|
|
3212
|
+
* `hosts` in its environment: it holds the real value while the agent works.
|
|
3213
|
+
*/
|
|
3214
|
+
type SetupProcess = {
|
|
3215
|
+
/** Its command line, cut at 200 characters. */
|
|
3216
|
+
command: string;
|
|
3217
|
+
/** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
|
|
3218
|
+
secrets: string[];
|
|
3219
|
+
};
|
|
3220
|
+
/** One `c.serve` call that listened. */
|
|
3221
|
+
type ServeRecord = {
|
|
3222
|
+
/** The command as the eval passed it. */
|
|
3223
|
+
command: string;
|
|
3224
|
+
/** The port it listens on, which the handle and `c.capture` use. */
|
|
3225
|
+
port: number;
|
|
3226
|
+
/** The `PORT` it was given: the port the eval passed, or the one reeval allocated. */
|
|
3227
|
+
portEnv: number;
|
|
3228
|
+
/** It listened on a port other than `PORT`, the one that started listening after it did. */
|
|
3229
|
+
detected: boolean;
|
|
3230
|
+
/** The address it listens on there, e.g. `127.0.0.1`, `::1` or `0.0.0.0`; empty when the sandbox can't tell. */
|
|
3231
|
+
address: string;
|
|
3232
|
+
/** From the command's start until it listened. */
|
|
3233
|
+
readyMs: number;
|
|
3234
|
+
};
|
|
3235
|
+
/** One `expect` or `expect.require` matcher call an attempt made, or a check another language made. */
|
|
3615
3236
|
type AssertionRecord = {
|
|
3616
3237
|
/**
|
|
3617
|
-
* `
|
|
3618
|
-
*
|
|
3238
|
+
* `label` is the supplied `expect` label. Without one, it is the matcher plus its first
|
|
3239
|
+
* argument or named subject, cut at 60 characters. Repeats get ` #2`, ` #3`, and later suffixes.
|
|
3619
3240
|
*/
|
|
3620
3241
|
label: string;
|
|
3621
3242
|
/** The eval gave the label. */
|
|
@@ -3624,12 +3245,61 @@ type AssertionRecord = {
|
|
|
3624
3245
|
matcher: string;
|
|
3625
3246
|
/** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
|
|
3626
3247
|
status: 'pass' | 'fail' | 'error';
|
|
3248
|
+
/**
|
|
3249
|
+
* A failure didn't end the eval's body: an `expect`. False for `expect.require` and for a check
|
|
3250
|
+
* that threw, such as Python's `assert`, which end the body when they fail.
|
|
3251
|
+
*/
|
|
3627
3252
|
soft: boolean;
|
|
3253
|
+
/** `path:line` of the call in the eval file, relative to the project root, when reeval saw it. */
|
|
3254
|
+
at?: string;
|
|
3628
3255
|
/** Why it failed or couldn't answer, cut at 1 KB. */
|
|
3629
3256
|
message?: string;
|
|
3630
3257
|
/** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
|
|
3631
3258
|
durationMs?: number;
|
|
3632
3259
|
};
|
|
3260
|
+
/**
|
|
3261
|
+
* A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
|
|
3262
|
+
* folder), or why it wasn't.
|
|
3263
|
+
*/
|
|
3264
|
+
type Snapshot = {
|
|
3265
|
+
/** `final` after the checks, or `turn-<n>` after the nth `c.send`. */
|
|
3266
|
+
name: string;
|
|
3267
|
+
/** A plain directory, or a gzipped tarball with `snapshot: { compress: true }`. */
|
|
3268
|
+
form: 'dir' | 'tar.gz';
|
|
3269
|
+
/**
|
|
3270
|
+
* Where it is, relative to the run's directory in `.reeval/runs`:
|
|
3271
|
+
* `<eval>/<cell>/<n>/snapshots/<name>` (with `.tar.gz` when compressed); null when it wasn't
|
|
3272
|
+
* taken.
|
|
3273
|
+
*/
|
|
3274
|
+
path: string | null;
|
|
3275
|
+
/** Its size on disk: the directory's files, or the tarball; 0 when it wasn't taken. */
|
|
3276
|
+
bytes: number;
|
|
3277
|
+
/** Regular files and symbolic links in it. */
|
|
3278
|
+
files: number;
|
|
3279
|
+
/** Directories left out by name, such as `node_modules/` and `.git/`, at most 50. */
|
|
3280
|
+
excluded: string[];
|
|
3281
|
+
/** Text files whose secret-looking values were masked, at most 50. */
|
|
3282
|
+
masked: string[];
|
|
3283
|
+
/** Binary files left out because they held a secret-looking value, at most 50. */
|
|
3284
|
+
dropped: string[];
|
|
3285
|
+
/**
|
|
3286
|
+
* Why it wasn't taken: `size` when the workspace was over its form's cap, `error` when the
|
|
3287
|
+
* sandbox couldn't archive it; null when it was.
|
|
3288
|
+
*/
|
|
3289
|
+
skipped: {
|
|
3290
|
+
cause: 'size' | 'error';
|
|
3291
|
+
message: string;
|
|
3292
|
+
} | null;
|
|
3293
|
+
};
|
|
3294
|
+
/** `snapshot`: the shorthand, or the object that also picks the form. */
|
|
3295
|
+
type SnapshotOption = boolean | 'turns' | SnapshotOptions;
|
|
3296
|
+
/** `snapshot` as an object, merged field by field over the levels above. */
|
|
3297
|
+
type SnapshotOptions = {
|
|
3298
|
+
/** `true` after the checks, `'turns'` after each `c.send` too, `false` never. Default `true`. */
|
|
3299
|
+
when?: boolean | 'turns';
|
|
3300
|
+
/** Write each as `<name>.tar.gz` instead of a directory. Default `false`. */
|
|
3301
|
+
compress?: boolean;
|
|
3302
|
+
};
|
|
3633
3303
|
/** Which environment ran as the control, so the studio can show it. */
|
|
3634
3304
|
type EvalCompareMeta = {
|
|
3635
3305
|
baseline: string;
|
|
@@ -3640,16 +3310,20 @@ type EvalCompareMeta = {
|
|
|
3640
3310
|
from?: 'environment' | 'empty';
|
|
3641
3311
|
};
|
|
3642
3312
|
/**
|
|
3643
|
-
* A sandbox with more privilege than the default `reeval` user:
|
|
3644
|
-
* `docker({ capabilities })`. One per level that set such a sandbox,
|
|
3313
|
+
* A `SandboxWarning` describes a sandbox with more privilege than the default `reeval` user:
|
|
3314
|
+
* `docker({ root: true })` or `docker({ capabilities })`. One per level that set such a sandbox,
|
|
3315
|
+
* however many evaluations inherit it.
|
|
3645
3316
|
*/
|
|
3646
3317
|
type SandboxWarning = {
|
|
3647
3318
|
kind: 'root' | 'capabilities';
|
|
3648
3319
|
/** Which level set the sandbox. */
|
|
3649
3320
|
level: 'config' | 'profile' | 'group' | 'eval';
|
|
3650
|
-
/**
|
|
3321
|
+
/**
|
|
3322
|
+
* The value is the level as reeval names it: the `reeval.config.ts` file, `profiles.pr.defaults`,
|
|
3323
|
+
* `group("net")`, or `evaluate("installs deps")`.
|
|
3324
|
+
*/
|
|
3651
3325
|
where: string;
|
|
3652
|
-
/** The
|
|
3326
|
+
/** The evaluation file that sets it, for a group or evaluation; null for the config or a profile. */
|
|
3653
3327
|
file: string | null;
|
|
3654
3328
|
/** The capabilities added back, for `capabilities`. */
|
|
3655
3329
|
capabilities?: string[];
|
|
@@ -3665,5 +3339,5 @@ type ResolvedGateway = {
|
|
|
3665
3339
|
}>;
|
|
3666
3340
|
};
|
|
3667
3341
|
//#endregion
|
|
3668
|
-
export {
|
|
3669
|
-
//# sourceMappingURL=types-
|
|
3342
|
+
export { FakeScriptStep as $, Verdict as $t, CursorHarnessDescriptor as A, Spread as An, PluginPart as At, EnvironmentHelpers as B, effortLabel as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, DIRECTIONAL_MIN as Cn, OpenCodeHarnessDescriptor as Ct, CompareOptions as D, EffortMetric as Dn, PairVote as Dt, CompareCut as E, EffortContrast as En, PairFactor as Et, Duration as F, cellEffort as Fn, Question as Ft, EvalOptions as G, formatCount as Gn, SetupStep as Gt, EnvironmentSpec as H, effortSample as Hn, Secret as Ht, EgressPolicy as I, cellSamples as In, QuestionInputs as It, FakeAnalysisModel as J, tokenValues as Jn, TranscriptEvent as Jt, ExecResult as K, formatRatio as Kn, SkillSource as Kt, EgressRecord as L, contrastEffort as Ln, QuestionOptions as Lt, DirectGatewayDescriptor as M, VERDICT_TEXT as Mn, Profile as Mt, DirectoryOptions as N, bothUnderATenth as Nn, ProfileDefaults as Nt, ConfigHelpers as O, EffortSample as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, cellContrast as Pn, ProfileLimit as Pt, FakeSandboxDescriptor as Q, VercelGatewayDescriptor as Qt, EnvVars as R, directional as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, CellEffort as Sn, McpServer as St, CompareAnswer as T, EFFORT_MIN_RUNS as Tn, Outcome as Tt, ErrorKind as U, effortSummary as Un, SecretTemplate as Ut, EnvironmentInput as V, effortOf as Vn, SandboxWarning as Vt, EvalFn as W, effortVerdict as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, DeclaredNode as Xn, TurnData as Xt, FakeAnswer as Y, DeclarationTree as Yn, Turn as Yt, FakeJudgeModel as Z, TurnStatus as Zt, CaptureUseScript as _, JudgmentRecord as _n, JudgeInputs as _t, AttemptContext as a, ArtifactRef as an, GrokBuildHarnessDescriptor as at, Cell as b, StackSummary as bn, JudgeState as bt, BlindPair as c, CheckTally as cn, HarnessDescriptor as ct, CaptureCall as d, FamilyLiftSummary as dn, HistoryConfig as dt, VertexGatewayDescriptor as en, FakeVote as et, CaptureInstruction as f, FamilyRate as fn, HostFile as ft, CaptureUseAction as g, HistoryEntry as gn, JudgeFailure as gt, CaptureUse as h, HistoryCell as hn, JudgeCallOptions as ht, AttemptAnswer as i, WebEventType as in, GooseHarnessDescriptor as it, Diff as j, TOKENS_UNREPORTED as jn, PluginSource as jt, ConfigJudgeOptions as k, EffortVerdict as kn, PinnableKey as kt, Budget as l, EvalEntry as ln, HarnessEdit as lt, CaptureOptions as m, FileRef as mn, JudgeArtifactRef as mt, AnalysisConfig as n, Viewport as nn, GatewayDescriptor as nt, BackgroundHandle as o, AttemptRecord as on, GroupOptions as ot, CaptureKey as p, FamilyRecord as pn, JSONValue as pt, ExecRetry as q, formatSeconds as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, Wait as rn, GatewayName as rt, BlindAttempt as s, CellRecord as sn, HarnessData as st, AgentHarnessDescriptor as t, VertexPublisher as tn, FxHarnessDescriptor as tt, Capture as u, FamilyLift as un, HarnessOption as ut, CaptureWait as v, Lift as vn, JudgeModel as vt, CodexHarnessDescriptor as w, EFFORT_ALPHA as wn, OpenRouterGatewayDescriptor as wt, CellRole as x, UpstreamCount as xn, JudgeSubject as xt, CaptureWindow as y, RunRecord as yn, JudgeOptions as yt, Environment as z, effortChangeLabel as zn, ResolvedGateway as zt };
|
|
3343
|
+
//# sourceMappingURL=types-DVvW66ci.d.ts.map
|