reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,3 +1,63 @@
|
|
|
1
|
+
//#region src/declaration.d.ts
|
|
2
|
+
/** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
|
|
3
|
+
type DeclaredMode = 'run' | 'skip' | 'only';
|
|
4
|
+
/** A `group`: options its evals and groups inherit, and its children in declaration order. */
|
|
5
|
+
type DeclaredGroup = {
|
|
6
|
+
type: 'group';
|
|
7
|
+
/** The group's name, as passed to `group`. */
|
|
8
|
+
name: string;
|
|
9
|
+
mode: DeclaredMode;
|
|
10
|
+
/** The group's own options, before the cascade. */
|
|
11
|
+
options: EvalOptionsData;
|
|
12
|
+
children: DeclaredNode[];
|
|
13
|
+
};
|
|
14
|
+
/** An `evaluate`: one eval, planned into a test per attempt. */
|
|
15
|
+
type DeclaredEval = {
|
|
16
|
+
type: 'eval';
|
|
17
|
+
/** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
|
|
18
|
+
name: string;
|
|
19
|
+
mode: DeclaredMode;
|
|
20
|
+
/** The eval's own options, before the cascade. */
|
|
21
|
+
options: EvalOptionsData;
|
|
22
|
+
/** Names the eval's body to whoever runs it. */
|
|
23
|
+
body: string;
|
|
24
|
+
source: DeclaredSource;
|
|
25
|
+
/** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
|
|
26
|
+
family?: DeclaredFamily;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
|
|
30
|
+
* text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
|
|
31
|
+
* whitespace, non-ASCII characters as themselves).
|
|
32
|
+
*/
|
|
33
|
+
type DeclaredSource = {
|
|
34
|
+
/** sha256 of the body function's source text. */
|
|
35
|
+
body: string;
|
|
36
|
+
/** sha256 of the case's data; set on each case of an `evaluate.each`. */
|
|
37
|
+
case?: string;
|
|
38
|
+
};
|
|
39
|
+
/** Where a case of a pooled `evaluate.each` sits in its list. */
|
|
40
|
+
type DeclaredFamily = {
|
|
41
|
+
/** The name template passed to `evaluate.each`. */
|
|
42
|
+
name: string;
|
|
43
|
+
/** The case's index, from 0. */
|
|
44
|
+
case: number;
|
|
45
|
+
/** How many cases the list has. */
|
|
46
|
+
cases: number;
|
|
47
|
+
};
|
|
48
|
+
/** `evaluate.todo` or `group.todo`. */
|
|
49
|
+
type DeclaredTodo = {
|
|
50
|
+
type: 'todo';
|
|
51
|
+
/** The name passed to `.todo`. */
|
|
52
|
+
name: string;
|
|
53
|
+
};
|
|
54
|
+
/** One node of the tree: a group, an eval or a todo. */
|
|
55
|
+
type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
|
|
56
|
+
/** An eval file's top-level groups, evals and todos, in declaration order. */
|
|
57
|
+
type DeclarationTree = {
|
|
58
|
+
children: DeclaredNode[];
|
|
59
|
+
};
|
|
60
|
+
//#endregion
|
|
1
61
|
//#region src/view/effort.d.ts
|
|
2
62
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
3
63
|
declare const EFFORT_ALPHA = 0.05;
|
|
@@ -192,7 +252,7 @@ type JudgeRetry = {
|
|
|
192
252
|
error: string;
|
|
193
253
|
delayMs: number;
|
|
194
254
|
};
|
|
195
|
-
type
|
|
255
|
+
type JudgmentRecord = {
|
|
196
256
|
question: string;
|
|
197
257
|
verdict: 'pass' | 'fail';
|
|
198
258
|
score: number;
|
|
@@ -202,9 +262,15 @@ type JudgementRecord = {
|
|
|
202
262
|
resolvedModel?: string;
|
|
203
263
|
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
204
264
|
gateway?: string;
|
|
265
|
+
/** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
|
|
266
|
+
upstream?: string;
|
|
267
|
+
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
268
|
+
costUsd?: number;
|
|
205
269
|
/**
|
|
206
270
|
* `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
|
|
207
|
-
* verdict can be followed across runs (and an edited question starts a new series).
|
|
271
|
+
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
272
|
+
* is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
|
|
273
|
+
* whichever finishes first.
|
|
208
274
|
*/
|
|
209
275
|
key: string;
|
|
210
276
|
/** Transient provider errors retried before this verdict, with the backoff before each retry. */
|
|
@@ -216,7 +282,7 @@ type AttemptFiles = {
|
|
|
216
282
|
/** Each environment setup step's command, exit code, duration and output. */
|
|
217
283
|
setupLog?: FileRef;
|
|
218
284
|
transcript?: FileRef;
|
|
219
|
-
/**
|
|
285
|
+
/** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
|
|
220
286
|
rawTranscript?: FileRef;
|
|
221
287
|
diff?: FileRef;
|
|
222
288
|
judgeInput?: FileRef;
|
|
@@ -227,6 +293,11 @@ type ContextFile = {
|
|
|
227
293
|
path: string;
|
|
228
294
|
sha256: string;
|
|
229
295
|
};
|
|
296
|
+
/** An upstream provider behind a gateway, and how many of the attempt's calls it served. */
|
|
297
|
+
type UpstreamCount = {
|
|
298
|
+
provider: string;
|
|
299
|
+
calls: number;
|
|
300
|
+
};
|
|
230
301
|
type AttemptRecord = {
|
|
231
302
|
/** `<eval key> # <cell key>#<n>`: unique in the run. */
|
|
232
303
|
key: string;
|
|
@@ -265,6 +336,17 @@ type AttemptRecord = {
|
|
|
265
336
|
* gateway's, so never part of `servedModels`. Absent when there's none.
|
|
266
337
|
*/
|
|
267
338
|
askedModels?: string[];
|
|
339
|
+
/**
|
|
340
|
+
* The upstream providers a gateway said served the harness's own API calls (OpenRouter's
|
|
341
|
+
* `provider`), each with how many it served, in the order first seen; more than one means the
|
|
342
|
+
* gateway spread or failed the calls over between them. Absent when the gateway names none.
|
|
343
|
+
*/
|
|
344
|
+
upstreams?: UpstreamCount[];
|
|
345
|
+
/**
|
|
346
|
+
* What a gateway said the harness's own API calls cost, in US dollars, summed (OpenRouter's
|
|
347
|
+
* `usage.cost`); absent when it names none.
|
|
348
|
+
*/
|
|
349
|
+
reportedCostUsd?: number;
|
|
268
350
|
/**
|
|
269
351
|
* The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
|
|
270
352
|
* null when the factory left it unset. Fast is not this.
|
|
@@ -325,9 +407,18 @@ type AttemptRecord = {
|
|
|
325
407
|
/** Longest tool `durationMs` in the transcript; null when none reported a duration. */
|
|
326
408
|
maxToolMs: number | null;
|
|
327
409
|
};
|
|
328
|
-
judgements:
|
|
410
|
+
judgements: JudgmentRecord[];
|
|
329
411
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
330
412
|
startRetries?: StartRetry[];
|
|
413
|
+
/**
|
|
414
|
+
* Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
|
|
415
|
+
* without `hosts` in their environment, so they hold its real value while the agent works and
|
|
416
|
+
* the agent, as the same user, can read it from them. Found by the secret's variable name, so a
|
|
417
|
+
* copy under another name or in a command's arguments isn't. At most 20, each
|
|
418
|
+
* `{ command, secrets }`: its command line, masked and cut at 200 characters, and those secrets'
|
|
419
|
+
* names. Absent when there were none.
|
|
420
|
+
*/
|
|
421
|
+
setupProcesses?: SetupProcess[];
|
|
331
422
|
/**
|
|
332
423
|
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
333
424
|
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
@@ -369,11 +460,21 @@ type EvalEntry = {
|
|
|
369
460
|
name: string;
|
|
370
461
|
/** Results are comparable only when fingerprints match. */
|
|
371
462
|
fingerprint: string;
|
|
372
|
-
/**
|
|
463
|
+
/**
|
|
464
|
+
* The eval file, by content hash: where the eval lives. Records from before `source` also list
|
|
465
|
+
* the local files it imported, and were fingerprinted over them.
|
|
466
|
+
*/
|
|
373
467
|
sources: Array<{
|
|
374
468
|
path: string;
|
|
375
469
|
sha256: string;
|
|
376
470
|
}>;
|
|
471
|
+
/**
|
|
472
|
+
* Hashes of what the `sources` fingerprint part covers but `revision`: the eval's body, its
|
|
473
|
+
* case data and its options with its groups'. Unset on records from before it.
|
|
474
|
+
*/
|
|
475
|
+
source?: EvalSource;
|
|
476
|
+
/** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
|
|
477
|
+
revision?: Revision;
|
|
377
478
|
tags: string[];
|
|
378
479
|
/** The baseline environment, when the eval uses `compare`. */
|
|
379
480
|
compare?: EvalCompareMeta;
|
|
@@ -535,7 +636,7 @@ type Lift = {
|
|
|
535
636
|
};
|
|
536
637
|
/** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
|
|
537
638
|
type FingerprintParts = {
|
|
538
|
-
/** Over `EvalEntry.sources`. */
|
|
639
|
+
/** Over `EvalEntry.source` and `EvalEntry.revision`; on records from before them, over `EvalEntry.sources`. */
|
|
539
640
|
sources: string;
|
|
540
641
|
/** Over the harness descriptor, less concurrency caps. */
|
|
541
642
|
harness: string;
|
|
@@ -548,13 +649,21 @@ type FingerprintParts = {
|
|
|
548
649
|
context: string;
|
|
549
650
|
/** The gateway its model calls went through; unset on `direct`, each provider's own key. */
|
|
550
651
|
gateway?: string;
|
|
652
|
+
/**
|
|
653
|
+
* The cell's gate (`none`, `passRate 0.5`); unset on the default `all`. Compare reads a change
|
|
654
|
+
* in it as a policy change, not a reason the runs can't be compared.
|
|
655
|
+
*/
|
|
656
|
+
gate?: string;
|
|
551
657
|
};
|
|
552
658
|
/** One harness and environment of an eval, and how its attempts went. */
|
|
553
659
|
type CellRecord = Cell & {
|
|
554
660
|
/** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
|
|
555
661
|
fingerprint: string;
|
|
556
662
|
fingerprintParts: FingerprintParts;
|
|
557
|
-
/**
|
|
663
|
+
/**
|
|
664
|
+
* The environment's digest; null for none. With `environment`, the key into
|
|
665
|
+
* `RunRecord.environments`: environments with the same contents share a digest.
|
|
666
|
+
*/
|
|
558
667
|
environmentDigest: string | null;
|
|
559
668
|
harnessKind: string;
|
|
560
669
|
/**
|
|
@@ -698,6 +807,7 @@ type RunRecord = {
|
|
|
698
807
|
config: {
|
|
699
808
|
sandbox: string;
|
|
700
809
|
environment: string | null;
|
|
810
|
+
/** The config's `runs`; 1 when it sets none, though a compare's cells then run 4 each. */
|
|
701
811
|
runs: number;
|
|
702
812
|
judgeModel: string | null;
|
|
703
813
|
threshold: number;
|
|
@@ -824,8 +934,15 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
|
|
|
824
934
|
};
|
|
825
935
|
/** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
|
|
826
936
|
effort: CellEffort | null;
|
|
827
|
-
/** Of `fail`, the attempts the run's interrupt
|
|
828
|
-
|
|
937
|
+
/** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
|
|
938
|
+
canceled?: number;
|
|
939
|
+
};
|
|
940
|
+
/** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
|
|
941
|
+
type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
|
|
942
|
+
lo: number;
|
|
943
|
+
hi: number;
|
|
944
|
+
/** Unset when the compare has no `minLift`, so the lift can't fail. */
|
|
945
|
+
minLift?: number;
|
|
829
946
|
};
|
|
830
947
|
/**
|
|
831
948
|
* One line of the derived history listing: cached locally by the CLI, and the run header in the
|
|
@@ -860,636 +977,24 @@ type HistoryEntry = {
|
|
|
860
977
|
slug: string;
|
|
861
978
|
fingerprint: string;
|
|
862
979
|
sources: EvalEntry['sources'];
|
|
980
|
+
source?: EvalEntry['source'];
|
|
981
|
+
revision?: EvalEntry['revision'];
|
|
863
982
|
outcome: Outcome;
|
|
864
983
|
passRate: number | null;
|
|
865
984
|
cells: HistoryCell[];
|
|
866
985
|
}>;
|
|
867
986
|
/** The run's pooled families; unset when none ran. */
|
|
868
|
-
families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'
|
|
987
|
+
families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
|
|
988
|
+
/** The eval ids of its cases in this run. */
|
|
989
|
+
caseIds: string[];
|
|
990
|
+
lifts: FamilyLiftSummary[];
|
|
991
|
+
}>;
|
|
869
992
|
/** The run's errors outside any attempt; unset when there were none. */
|
|
870
993
|
errors?: string[];
|
|
871
994
|
};
|
|
872
995
|
//#endregion
|
|
873
|
-
//#region src/view/suite.d.ts
|
|
874
|
-
/**
|
|
875
|
-
* A variant against its baseline: `better` or `worse` when the stored 95% interval on the
|
|
876
|
-
* difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
|
|
877
|
-
* including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
|
|
878
|
-
*/
|
|
879
|
-
type Effect = 'better' | 'worse' | 'same' | 'unclear';
|
|
880
|
-
/**
|
|
881
|
-
* What an eval's cells compare: `change` when it has variants (or several environments),
|
|
882
|
-
* `harness` when it has several harnesses, `both`, or `none`.
|
|
883
|
-
*/
|
|
884
|
-
type Across = 'none' | 'change' | 'harness' | 'both';
|
|
885
|
-
/**
|
|
886
|
-
* One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
|
|
887
|
-
* lists them all in the order the nodes appear.
|
|
888
|
-
*
|
|
889
|
-
* A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
|
|
890
|
-
*
|
|
891
|
-
* - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
|
|
892
|
-
* (`evals/a.eval.ts › fixes $id`);
|
|
893
|
-
* - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
|
|
894
|
-
* - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
|
|
895
|
-
* - `<cell>/checks/<label>`: a check, by its assertion label;
|
|
896
|
-
* - `<cell>/effort`: the cell's effort, only when it was tested;
|
|
897
|
-
* - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
|
|
898
|
-
*
|
|
899
|
-
* A case segment never has a raw `+` and a cell segment always has two, so the two can't be
|
|
900
|
-
* confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
|
|
901
|
-
* fact in every run that has it.
|
|
902
|
-
*/
|
|
903
|
-
type SuiteResult = {
|
|
904
|
-
runId: string;
|
|
905
|
-
status: RunRecord['status'];
|
|
906
|
-
/**
|
|
907
|
-
* The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
|
|
908
|
-
* an attempt, and `skipped` on an interrupted run.
|
|
909
|
-
*/
|
|
910
|
-
outcome: Outcome;
|
|
911
|
-
/**
|
|
912
|
-
* Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
|
|
913
|
-
* awaited, a worker that died. Any one fails the run.
|
|
914
|
-
*/
|
|
915
|
-
errors: string[];
|
|
916
|
-
/**
|
|
917
|
-
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
918
|
-
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
|
|
919
|
-
* `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
|
|
920
|
-
*/
|
|
921
|
-
tally: SuiteTally;
|
|
922
|
-
/** Sorted by path. */
|
|
923
|
-
files: ResultFile[];
|
|
924
|
-
/** Every node's `path`, in the order they appear. */
|
|
925
|
-
paths: string[];
|
|
926
|
-
};
|
|
927
|
-
type SuiteTally = {
|
|
928
|
-
errors: number;
|
|
929
|
-
pass: number;
|
|
930
|
-
notGated: number;
|
|
931
|
-
fail: number;
|
|
932
|
-
infraError: number;
|
|
933
|
-
cancelled: number;
|
|
934
|
-
skipped: number;
|
|
935
|
-
};
|
|
936
|
-
/**
|
|
937
|
-
* An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
|
|
938
|
-
* gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
|
|
939
|
-
*/
|
|
940
|
-
type ResultOutcome = Outcome | 'cancelled';
|
|
941
|
-
type ResultFile = {
|
|
942
|
-
path: string;
|
|
943
|
-
evals: EvalResult[];
|
|
944
|
-
};
|
|
945
|
-
type EvalResult = {
|
|
946
|
-
path: string;
|
|
947
|
-
/** The eval's id, or the family's. */
|
|
948
|
-
id: string;
|
|
949
|
-
group: string[];
|
|
950
|
-
/** The eval's name; on a family, its template. */
|
|
951
|
-
name: string;
|
|
952
|
-
/** A family's name template (`fixes $id`); null on a single eval. */
|
|
953
|
-
template: string | null;
|
|
954
|
-
/** Source order in the file. */
|
|
955
|
-
order: number;
|
|
956
|
-
across: Across;
|
|
957
|
-
/**
|
|
958
|
-
* A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
|
|
959
|
-
* unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
|
|
960
|
-
*/
|
|
961
|
-
family: boolean;
|
|
962
|
-
/** A placeholder (`evaluate.todo`) that never ran. */
|
|
963
|
-
todo: boolean;
|
|
964
|
-
/** On a family: `fail` when a case failed or its pooled gate did. */
|
|
965
|
-
outcome: ResultOutcome;
|
|
966
|
-
cells: CellResult[] | null;
|
|
967
|
-
cases: CaseResult[] | null;
|
|
968
|
-
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
969
|
-
pooled: PooledArm[] | null;
|
|
970
|
-
/**
|
|
971
|
-
* Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
|
|
972
|
-
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
973
|
-
*/
|
|
974
|
-
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
975
|
-
};
|
|
976
|
-
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
977
|
-
type QuestionResult = {
|
|
978
|
-
kind: 'pairs';
|
|
979
|
-
question: string;
|
|
980
|
-
/** Pairs voted, each counted once per order it was asked in. */
|
|
981
|
-
votes: number;
|
|
982
|
-
rates: Array<{
|
|
983
|
-
cut: 'within' | 'across';
|
|
984
|
-
name: string;
|
|
985
|
-
wins: number;
|
|
986
|
-
n: number;
|
|
987
|
-
}>;
|
|
988
|
-
/** Why some or all of its votes are missing: the model call failed. */
|
|
989
|
-
error: string | null;
|
|
990
|
-
};
|
|
991
|
-
/** A question asked of each attempt on its own: what each one answered. Never gates. */
|
|
992
|
-
type AttemptQuestionResult = {
|
|
993
|
-
kind: 'attempts';
|
|
994
|
-
question: string;
|
|
995
|
-
/** Attempts it was asked of, failed calls included. */
|
|
996
|
-
asked: number;
|
|
997
|
-
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
998
|
-
choices: Array<{
|
|
999
|
-
choice: string;
|
|
1000
|
-
n: number;
|
|
1001
|
-
}> | null;
|
|
1002
|
-
/** Each attempt's answer, in attempt order; failed calls left out. */
|
|
1003
|
-
answers: Array<{
|
|
1004
|
-
attempt: string;
|
|
1005
|
-
choice: string | null;
|
|
1006
|
-
text: string;
|
|
1007
|
-
}>;
|
|
1008
|
-
/** Calls that failed, and the first one's error. */
|
|
1009
|
-
failed: number;
|
|
1010
|
-
error: string | null;
|
|
1011
|
-
};
|
|
1012
|
-
type CaseResult = {
|
|
1013
|
-
path: string;
|
|
1014
|
-
/** The case's own eval id. */
|
|
1015
|
-
id: string;
|
|
1016
|
-
/** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
|
|
1017
|
-
label: string;
|
|
1018
|
-
name: string;
|
|
1019
|
-
/** Its place in the case list. */
|
|
1020
|
-
order: number;
|
|
1021
|
-
outcome: ResultOutcome;
|
|
1022
|
-
cells: CellResult[];
|
|
1023
|
-
};
|
|
1024
|
-
type CellResult = {
|
|
1025
|
-
path: string;
|
|
1026
|
-
key: string;
|
|
1027
|
-
harness: string;
|
|
1028
|
-
/** `-` when no environment is set. */
|
|
1029
|
-
environment: string;
|
|
1030
|
-
sandbox: string;
|
|
1031
|
-
/** The variant's name; null on a control. */
|
|
1032
|
-
variant: string | null;
|
|
1033
|
-
/** The control of an eval that has variants: the arm the variants are measured against. */
|
|
1034
|
-
baseline: boolean;
|
|
1035
|
-
/** On a variant, its baseline cell's key; else null. */
|
|
1036
|
-
vs: string | null;
|
|
1037
|
-
/** The cell's place in the eval's cells. */
|
|
1038
|
-
order: number;
|
|
1039
|
-
/** Its failures fail the eval (its gate isn't `'none'`). */
|
|
1040
|
-
gated: boolean;
|
|
1041
|
-
outcome: Outcome;
|
|
1042
|
-
/** Every attempt: `passed + failed + infraError + skipped`. */
|
|
1043
|
-
attempts: number;
|
|
1044
|
-
passed: number;
|
|
1045
|
-
failed: number;
|
|
1046
|
-
/** Infrastructure errors, which are neither passes nor fails. */
|
|
1047
|
-
infraError: number;
|
|
1048
|
-
skipped: number;
|
|
1049
|
-
fingerprint: string;
|
|
1050
|
-
/** The skill sources and files the cell's environment placed, as written. */
|
|
1051
|
-
context: string[];
|
|
1052
|
-
/**
|
|
1053
|
-
* Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
|
|
1054
|
-
* attempt (every one an infra error or skipped) lists each as `not run`.
|
|
1055
|
-
*/
|
|
1056
|
-
checks: CheckResult[];
|
|
1057
|
-
/** On a variant, against its baseline; null on a control. */
|
|
1058
|
-
effect: Effect | null;
|
|
1059
|
-
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1060
|
-
interval: Interval | null;
|
|
1061
|
-
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1062
|
-
minLift: number | null;
|
|
1063
|
-
/** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
|
|
1064
|
-
effort: EffortCounters | null;
|
|
1065
|
-
/** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
|
|
1066
|
-
effortDiffers: boolean;
|
|
1067
|
-
/** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
|
|
1068
|
-
errors: CellError[];
|
|
1069
|
-
/** Web searches and fetches per counted attempt; null where the harness can't report them. */
|
|
1070
|
-
searches: Array<number | null>;
|
|
1071
|
-
fetches: Array<number | null>;
|
|
1072
|
-
};
|
|
1073
|
-
/** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
|
|
1074
|
-
declare const cancelledOf: (cell: CellResult) => number;
|
|
1075
|
-
/** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
|
|
1076
|
-
declare const countedOf: (cell: CellResult) => number;
|
|
1077
|
-
type CheckResult = {
|
|
1078
|
-
path: string;
|
|
1079
|
-
label: string;
|
|
1080
|
-
/** First-seen order across the eval's attempts. */
|
|
1081
|
-
order: number;
|
|
1082
|
-
outcome: CheckOutcome;
|
|
1083
|
-
/**
|
|
1084
|
-
* On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
|
|
1085
|
-
* null on a control or without one.
|
|
1086
|
-
*/
|
|
1087
|
-
effect: Effect | null;
|
|
1088
|
-
/** Set when the check is a judge verdict. */
|
|
1089
|
-
judge: JudgeResult | null;
|
|
1090
|
-
/**
|
|
1091
|
-
* A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
|
|
1092
|
-
* it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
|
|
1093
|
-
* when none did.
|
|
1094
|
-
*/
|
|
1095
|
-
retried?: string;
|
|
1096
|
-
};
|
|
1097
|
-
/**
|
|
1098
|
-
* `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
|
|
1099
|
-
* none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
|
|
1100
|
-
* reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
|
|
1101
|
-
* first failure's message.
|
|
1102
|
-
*/
|
|
1103
|
-
type CheckOutcome = {
|
|
1104
|
-
status: 'not run';
|
|
1105
|
-
} | {
|
|
1106
|
-
status: 'ran';
|
|
1107
|
-
passed: number;
|
|
1108
|
-
failed: number;
|
|
1109
|
-
errored: number;
|
|
1110
|
-
notRun: number;
|
|
1111
|
-
assertion: string | null;
|
|
1112
|
-
};
|
|
1113
|
-
/**
|
|
1114
|
-
* One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
|
|
1115
|
-
* an error), its message's first line, and how often.
|
|
1116
|
-
*/
|
|
1117
|
-
type CellError = {
|
|
1118
|
-
kind: ErrorKind | 'turn';
|
|
1119
|
-
message: string;
|
|
1120
|
-
attempts: number;
|
|
1121
|
-
};
|
|
1122
|
-
type JudgeResult = {
|
|
1123
|
-
question: string;
|
|
1124
|
-
threshold: number;
|
|
1125
|
-
/** One per counted attempt that judged it. */
|
|
1126
|
-
scores: number[];
|
|
1127
|
-
};
|
|
1128
|
-
type Interval = {
|
|
1129
|
-
low: number;
|
|
1130
|
-
high: number;
|
|
1131
|
-
};
|
|
1132
|
-
/** A null counter is one the harness didn't record; a recorded zero stays zero. */
|
|
1133
|
-
type EffortCounters = {
|
|
1134
|
-
tools: number | null;
|
|
1135
|
-
agentMs: number | null;
|
|
1136
|
-
tokens: number | null;
|
|
1137
|
-
};
|
|
1138
|
-
type PooledArm = {
|
|
1139
|
-
path: string;
|
|
1140
|
-
/** The variant cell's key, the same in every case. */
|
|
1141
|
-
key: string;
|
|
1142
|
-
vs: string;
|
|
1143
|
-
effect: Effect;
|
|
1144
|
-
/** Cases with counted attempts on both sides. */
|
|
1145
|
-
pairs: number;
|
|
1146
|
-
interval: Interval;
|
|
1147
|
-
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
1148
|
-
minLift: number | null;
|
|
1149
|
-
outcome: 'pass' | 'fail';
|
|
1150
|
-
};
|
|
1151
|
-
type Side = {
|
|
1152
|
-
pass: number;
|
|
1153
|
-
counted: number;
|
|
1154
|
-
};
|
|
1155
|
-
/**
|
|
1156
|
-
* The effect of `variant` against `control`, from the interval on their difference. A side
|
|
1157
|
-
* with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
|
|
1158
|
-
*/
|
|
1159
|
-
declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
|
|
1160
|
-
/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
|
|
1161
|
-
declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
|
|
1162
|
-
/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
|
|
1163
|
-
declare const factSegment: (s: string) => string;
|
|
1164
|
-
declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
|
|
1165
|
-
/**
|
|
1166
|
-
* An error's one line: its first line with any letter or digit in it, joined with the next one
|
|
1167
|
-
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1168
|
-
*/
|
|
1169
|
-
declare function messageLine(text: string): string;
|
|
1170
|
-
/** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
|
|
1171
|
-
declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
|
|
1172
|
-
/**
|
|
1173
|
-
* A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
|
|
1174
|
-
* gating it, so it reads `not gated`, never `pass`.
|
|
1175
|
-
*/
|
|
1176
|
-
declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
|
|
1177
|
-
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1178
|
-
declare function suiteResult(record: RunRecord): SuiteResult;
|
|
1179
|
-
//#endregion
|
|
1180
|
-
//#region src/view/insights.d.ts
|
|
1181
|
-
/** `<history>/<run>/insights.json`, beside the record. */
|
|
1182
|
-
declare const INSIGHTS_FILE = "insights.json";
|
|
1183
|
-
/** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
|
|
1184
|
-
declare const NO_MODEL = "none";
|
|
1185
|
-
/** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
|
|
1186
|
-
declare const fromModel: (insights: RunInsights) => boolean;
|
|
1187
|
-
/** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
|
|
1188
|
-
type Job = 'choose' | 'finish' | 'other';
|
|
1189
|
-
/**
|
|
1190
|
-
* A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
|
|
1191
|
-
* They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
|
|
1192
|
-
*/
|
|
1193
|
-
type RunInsights = {
|
|
1194
|
-
kind: 'reeval-insights';
|
|
1195
|
-
schemaVersion: 1;
|
|
1196
|
-
runId: string;
|
|
1197
|
-
/**
|
|
1198
|
-
* The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
|
|
1199
|
-
* reeval wrote the file without one, to hold the run's deterministic remarks only.
|
|
1200
|
-
*/
|
|
1201
|
-
model: string;
|
|
1202
|
-
/** The prompts' version: insights from another version were asked differently. */
|
|
1203
|
-
promptVersion: string;
|
|
1204
|
-
/** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
|
|
1205
|
-
factsHash: string;
|
|
1206
|
-
createdAt: string;
|
|
1207
|
-
/**
|
|
1208
|
-
* Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
|
|
1209
|
-
* reports them. `costUsd` is null when reeval doesn't know the model's price.
|
|
1210
|
-
*/
|
|
1211
|
-
usage: {
|
|
1212
|
-
inputTokens: number;
|
|
1213
|
-
outputTokens: number;
|
|
1214
|
-
reasoningTokens?: number;
|
|
1215
|
-
costUsd: number | null;
|
|
1216
|
-
};
|
|
1217
|
-
/**
|
|
1218
|
-
* `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
|
|
1219
|
-
* without insights. Absent when no budget applied.
|
|
1220
|
-
*/
|
|
1221
|
-
budget?: {
|
|
1222
|
-
maxTokens: number;
|
|
1223
|
-
skipped: string[];
|
|
1224
|
-
};
|
|
1225
|
-
/**
|
|
1226
|
-
* Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
|
|
1227
|
-
* timeout), with the last error: they print without insights. Absent when none failed.
|
|
1228
|
-
*/
|
|
1229
|
-
failed?: Array<{
|
|
1230
|
-
evalId: string;
|
|
1231
|
-
error: string;
|
|
1232
|
-
}>;
|
|
1233
|
-
/**
|
|
1234
|
-
* Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
|
|
1235
|
-
* the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
|
|
1236
|
-
* during the headline call, which leaves `headline` null.
|
|
1237
|
-
*/
|
|
1238
|
-
stopped?: {
|
|
1239
|
-
signal: 'SIGINT' | 'SIGTERM';
|
|
1240
|
-
skipped: string[];
|
|
1241
|
-
};
|
|
1242
|
-
/** One sentence for the whole run, citing facts; null when the model's was dropped. */
|
|
1243
|
-
headline: Sentence | null;
|
|
1244
|
-
/** One per eval (a family is one), in the order the output prints them. */
|
|
1245
|
-
evals: EvalInsight[];
|
|
1246
|
-
/**
|
|
1247
|
-
* Remarks about the evals' own quality, ranked: every one this run raised, including those an
|
|
1248
|
-
* earlier run printed first (`firstSeenRun`), which don't print again.
|
|
1249
|
-
*/
|
|
1250
|
-
remarks: Remark[];
|
|
1251
|
-
/**
|
|
1252
|
-
* What the checks dropped, and how many sentences were asked for again, so the output can say
|
|
1253
|
-
* so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
|
|
1254
|
-
* its text and why.
|
|
1255
|
-
*/
|
|
1256
|
-
validation: {
|
|
1257
|
-
dropped: number;
|
|
1258
|
-
regenerated: number;
|
|
1259
|
-
drops?: Array<{
|
|
1260
|
-
slot: string;
|
|
1261
|
-
text: string;
|
|
1262
|
-
reason: string;
|
|
1263
|
-
}>;
|
|
1264
|
-
};
|
|
1265
|
-
/** Set when insights failed; the facts still print. */
|
|
1266
|
-
error?: string;
|
|
1267
|
-
};
|
|
1268
|
-
type EvalInsight = {
|
|
1269
|
-
/** The eval's `SuiteResult` path; for a family, the family's. */
|
|
1270
|
-
evalId: string;
|
|
1271
|
-
intent: {
|
|
1272
|
-
/** What one attempt is testing. Can't be read from the cell shape. */
|
|
1273
|
-
job: Job;
|
|
1274
|
-
/** What the cells compare. Deterministic, copied from the facts. */
|
|
1275
|
-
across: Across;
|
|
1276
|
-
/** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
|
|
1277
|
-
question: string;
|
|
1278
|
-
/** The tool a choose eval asks about, and the change a measure eval makes. */
|
|
1279
|
-
subject: {
|
|
1280
|
-
tool?: string;
|
|
1281
|
-
change?: string;
|
|
1282
|
-
};
|
|
1283
|
-
/** How the eval name relates to what the eval checks. */
|
|
1284
|
-
name: 'states-intent' | 'no-intent' | 'contradicts';
|
|
1285
|
-
/**
|
|
1286
|
-
* sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
|
|
1287
|
-
* the model and the prompt version: the classification's cache key.
|
|
1288
|
-
*/
|
|
1289
|
-
key: string;
|
|
1290
|
-
};
|
|
1291
|
-
/** The answer to `question`, at most two sentences; null when it was dropped. */
|
|
1292
|
-
answer: Sentence | null;
|
|
1293
|
-
/** Up to three supporting points: what moved, what broke, what the agent did. */
|
|
1294
|
-
findings: Sentence[];
|
|
1295
|
-
/**
|
|
1296
|
-
* At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
|
|
1297
|
-
* Only rows the eval's layout opens a slot on are kept.
|
|
1298
|
-
*/
|
|
1299
|
-
rows?: Array<Sentence & {
|
|
1300
|
-
row: string;
|
|
1301
|
-
}>;
|
|
1302
|
-
/**
|
|
1303
|
-
* Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
|
|
1304
|
-
* verbatim in that attempt's final message. Only on `choose` evals.
|
|
1305
|
-
*/
|
|
1306
|
-
namedTools?: Array<{
|
|
1307
|
-
attempt: string;
|
|
1308
|
-
tools: string[];
|
|
1309
|
-
}>;
|
|
1310
|
-
};
|
|
1311
|
-
/**
|
|
1312
|
-
* A plain sentence citing the facts it states: `SuiteResult` paths
|
|
1313
|
-
* (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
|
|
1314
|
-
* (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
|
|
1315
|
-
*/
|
|
1316
|
-
type Sentence = {
|
|
1317
|
-
text: string;
|
|
1318
|
-
cites: string[];
|
|
1319
|
-
};
|
|
1320
|
-
/**
|
|
1321
|
-
* Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
|
|
1322
|
-
* gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
|
|
1323
|
-
* severity is the kind's, never the model's.
|
|
1324
|
-
*/
|
|
1325
|
-
declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
1326
|
-
origin: Remark['origin'];
|
|
1327
|
-
severity: 'concern' | 'suggestion';
|
|
1328
|
-
}>;
|
|
1329
|
-
/** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
|
|
1330
|
-
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
|
|
1331
|
-
/** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
|
|
1332
|
-
type Remark = {
|
|
1333
|
-
/**
|
|
1334
|
-
* sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
|
|
1335
|
-
* per kind (and server) per eval version.
|
|
1336
|
-
*/
|
|
1337
|
-
id: string;
|
|
1338
|
-
evalId: string;
|
|
1339
|
-
/** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
|
|
1340
|
-
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
|
|
1341
|
-
/** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
|
|
1342
|
-
severity: 'concern' | 'suggestion';
|
|
1343
|
-
/** One plain sentence: what's wrong. */
|
|
1344
|
-
text: string;
|
|
1345
|
-
/**
|
|
1346
|
-
* Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
|
|
1347
|
-
* file at the run's source hash. Every model remark has one.
|
|
1348
|
-
*/
|
|
1349
|
-
source?: {
|
|
1350
|
-
at: string;
|
|
1351
|
-
quote: string;
|
|
1352
|
-
};
|
|
1353
|
-
/** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
|
|
1354
|
-
evidence: string[];
|
|
1355
|
-
/**
|
|
1356
|
-
* An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
|
|
1357
|
-
* empty when the model gave none, and its `at` is the quoted line when it named no line.
|
|
1358
|
-
*/
|
|
1359
|
-
fix: {
|
|
1360
|
-
at: string;
|
|
1361
|
-
edit: string;
|
|
1362
|
-
};
|
|
1363
|
-
origin: 'model' | 'deterministic';
|
|
1364
|
-
/**
|
|
1365
|
-
* The first run that printed this `id`, or this run when none has: it prints only in that run,
|
|
1366
|
-
* except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
|
|
1367
|
-
* every run it happens.
|
|
1368
|
-
*/
|
|
1369
|
-
firstSeenRun: string;
|
|
1370
|
-
/**
|
|
1371
|
-
* Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
|
|
1372
|
-
* when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
|
|
1373
|
-
* `firstSeenRun` only.
|
|
1374
|
-
*/
|
|
1375
|
-
printed?: boolean;
|
|
1376
|
-
/** Set on later runs once the cited span changed: the remark was acted on. */
|
|
1377
|
-
actedOn?: {
|
|
1378
|
-
run: string;
|
|
1379
|
-
};
|
|
1380
|
-
};
|
|
1381
|
-
//#endregion
|
|
1382
|
-
//#region src/insights/input.d.ts
|
|
1383
|
-
/** One fact the model may cite, by its path. */
|
|
1384
|
-
type InsightFact = {
|
|
1385
|
-
path: string;
|
|
1386
|
-
kind: string;
|
|
1387
|
-
} & Record<string, unknown>;
|
|
1388
|
-
type InsightAttempt = {
|
|
1389
|
-
/** `<cell path>#<n>`. */
|
|
1390
|
-
id: string;
|
|
1391
|
-
cell: string;
|
|
1392
|
-
outcome: string;
|
|
1393
|
-
/** The agent's final message, clipped. */
|
|
1394
|
-
lastText: string;
|
|
1395
|
-
/** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
|
|
1396
|
-
lines: string[];
|
|
1397
|
-
};
|
|
1398
|
-
/** Where an insight sentence can go, as the eval's layout opens them. */
|
|
1399
|
-
type InsightSlots = {
|
|
1400
|
-
/** Row paths that take one sentence each. */
|
|
1401
|
-
rows: string[];
|
|
1402
|
-
findings: boolean;
|
|
1403
|
-
namedTools: boolean;
|
|
1404
|
-
};
|
|
1405
|
-
/** One eval as the classifier and the insight model see it. */
|
|
1406
|
-
type InsightInput = {
|
|
1407
|
-
call: 'classify' | 'insights';
|
|
1408
|
-
eval: {
|
|
1409
|
-
path: string;
|
|
1410
|
-
/** The eval's name, or a family's template: the author's statement of what it asks. */
|
|
1411
|
-
name: string;
|
|
1412
|
-
group: string[];
|
|
1413
|
-
file: string;
|
|
1414
|
-
across: Across;
|
|
1415
|
-
family: boolean;
|
|
1416
|
-
verdict: string;
|
|
1417
|
-
};
|
|
1418
|
-
/** The first attempt's prompts, in order. */
|
|
1419
|
-
prompts: string[];
|
|
1420
|
-
judgeQuestions: string[];
|
|
1421
|
-
checks: string[];
|
|
1422
|
-
/** Set on the insights call: the classification. */
|
|
1423
|
-
intent?: {
|
|
1424
|
-
job: Job;
|
|
1425
|
-
question: string;
|
|
1426
|
-
subject: {
|
|
1427
|
-
tool?: string;
|
|
1428
|
-
change?: string;
|
|
1429
|
-
};
|
|
1430
|
-
};
|
|
1431
|
-
/** Set on the insights call. */
|
|
1432
|
-
slots?: InsightSlots;
|
|
1433
|
-
facts: InsightFact[];
|
|
1434
|
-
attempts: InsightAttempt[];
|
|
1435
|
-
/**
|
|
1436
|
-
* Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
|
|
1437
|
-
* numbers it, which a remark's `source` and `fix` cite.
|
|
1438
|
-
*/
|
|
1439
|
-
source?: {
|
|
1440
|
-
file: string;
|
|
1441
|
-
lines: string[];
|
|
1442
|
-
};
|
|
1443
|
-
/** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
|
|
1444
|
-
trimmed?: string[];
|
|
1445
|
-
/** On a regeneration: why the earlier sentences were refused. */
|
|
1446
|
-
retry?: string[];
|
|
1447
|
-
};
|
|
1448
|
-
/** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
|
|
1449
|
-
type HeadlineInput = {
|
|
1450
|
-
call: 'headline';
|
|
1451
|
-
/**
|
|
1452
|
-
* Counts results, not evals: each case of a family counts one, as the tally line does, and
|
|
1453
|
-
* `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
|
|
1454
|
-
* results and stay out of it. The eval count is `evals.length`.
|
|
1455
|
-
*/
|
|
1456
|
-
tally: SuiteResult['tally'] & {
|
|
1457
|
-
total: number;
|
|
1458
|
-
unit: 'results';
|
|
1459
|
-
};
|
|
1460
|
-
evals: Array<{
|
|
1461
|
-
path: string;
|
|
1462
|
-
name: string;
|
|
1463
|
-
verdict: string;
|
|
1464
|
-
answer: string | null;
|
|
1465
|
-
}>;
|
|
1466
|
-
retry?: string[];
|
|
1467
|
-
};
|
|
1468
|
-
/** A fact as the model sees it: its short id (`F1`), which is what it cites. */
|
|
1469
|
-
type ModelFact = InsightFact & {
|
|
1470
|
-
id: string;
|
|
1471
|
-
};
|
|
1472
|
-
/** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
|
|
1473
|
-
type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
|
|
1474
|
-
id: string;
|
|
1475
|
-
cell: string;
|
|
1476
|
-
};
|
|
1477
|
-
/** One eval's call as the model sees it: facts, attempts and row slots by short id. */
|
|
1478
|
-
type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
|
|
1479
|
-
facts: ModelFact[];
|
|
1480
|
-
attempts: ModelAttempt[];
|
|
1481
|
-
slots?: Omit<InsightSlots, 'rows'> & {
|
|
1482
|
-
rows: string[];
|
|
1483
|
-
};
|
|
1484
|
-
};
|
|
1485
|
-
/** The headline call as the model sees it: each eval by its short id. */
|
|
1486
|
-
type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
|
|
1487
|
-
evals: Array<HeadlineInput['evals'][number] & {
|
|
1488
|
-
id: string;
|
|
1489
|
-
}>;
|
|
1490
|
-
};
|
|
1491
|
-
//#endregion
|
|
1492
996
|
//#region src/types.d.ts
|
|
997
|
+
/** More than zero and at most 596h: `90s`, `15m`, `1h`, or a number of milliseconds. */
|
|
1493
998
|
type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
|
|
1494
999
|
/** When an event started: wall clock, and ms since its turn started. */
|
|
1495
1000
|
type EventTime = {
|
|
@@ -1500,13 +1005,17 @@ type FileEditEvent = {
|
|
|
1500
1005
|
type: 'file-edit';
|
|
1501
1006
|
toolCallId: string;
|
|
1502
1007
|
path: string;
|
|
1503
|
-
|
|
1008
|
+
/**
|
|
1009
|
+
* `write` is a whole-file write whose result doesn't say whether the file existed before
|
|
1010
|
+
* (Antigravity's, fx's, Goose's, Grok Build's and pi's write tools, for example).
|
|
1011
|
+
*/
|
|
1012
|
+
kind: 'create' | 'modify' | 'write' | 'delete';
|
|
1504
1013
|
/** Line counts, or null when the harness didn't send the content (e.g. deletes). */
|
|
1505
1014
|
added: number | null;
|
|
1506
1015
|
removed: number | null;
|
|
1507
1016
|
};
|
|
1508
1017
|
/**
|
|
1509
|
-
*
|
|
1018
|
+
* Normalized agent events. Each tool call is one event, merged with its result; `durationMs`
|
|
1510
1019
|
* is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
|
|
1511
1020
|
* when the harness never reported the end. Long shell or edit calls set `stalled: true`.
|
|
1512
1021
|
*/
|
|
@@ -1683,6 +1192,7 @@ type Capture = {
|
|
|
1683
1192
|
* default `name` is derived from the path. `wait` and `instructions` are call-only.
|
|
1684
1193
|
*/
|
|
1685
1194
|
type CaptureCall = {
|
|
1195
|
+
/** The `artifacts/<name>.png` file name: 1 to 100 of `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. Unique within the attempt. */
|
|
1686
1196
|
name?: string;
|
|
1687
1197
|
path?: string;
|
|
1688
1198
|
viewport?: Viewport;
|
|
@@ -1884,6 +1394,10 @@ type Verdict = {
|
|
|
1884
1394
|
resolvedModel?: string;
|
|
1885
1395
|
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
1886
1396
|
gateway?: GatewayName;
|
|
1397
|
+
/** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
|
|
1398
|
+
upstream?: string;
|
|
1399
|
+
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
1400
|
+
costUsd?: number;
|
|
1887
1401
|
/** Captures this call sent the judge as file parts, when any. */
|
|
1888
1402
|
artifacts?: JudgeArtifactRef[];
|
|
1889
1403
|
};
|
|
@@ -1891,7 +1405,7 @@ type Verdict = {
|
|
|
1891
1405
|
* What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
|
|
1892
1406
|
* so the judges and checks after it still run.
|
|
1893
1407
|
*/
|
|
1894
|
-
type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
|
|
1408
|
+
type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel' | 'upstream' | 'costUsd'> & {
|
|
1895
1409
|
verdict: 'error';
|
|
1896
1410
|
score: null;
|
|
1897
1411
|
error: string;
|
|
@@ -1966,7 +1480,7 @@ type FakeHarnessDescriptor = {
|
|
|
1966
1480
|
files?: Record<string, string>;
|
|
1967
1481
|
/** The fake agent's final message on every turn; default `Done.`. */
|
|
1968
1482
|
text?: string;
|
|
1969
|
-
/** Simulated turn length,
|
|
1483
|
+
/** Simulated turn length, honors the abort signal. */
|
|
1970
1484
|
delayMs?: number;
|
|
1971
1485
|
/** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
|
|
1972
1486
|
ignoreAbort?: boolean;
|
|
@@ -2243,7 +1757,7 @@ type CodexHarnessDescriptor = {
|
|
|
2243
1757
|
};
|
|
2244
1758
|
/**
|
|
2245
1759
|
* fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
|
|
2246
|
-
* OpenAI's API by the model id, and reports each turn's tokens.
|
|
1760
|
+
* OpenAI's API by the model id, or on OpenRouter or Vertex AI, and reports each turn's tokens.
|
|
2247
1761
|
*/
|
|
2248
1762
|
type FxHarnessDescriptor = {
|
|
2249
1763
|
kind: 'fx';
|
|
@@ -2255,10 +1769,18 @@ type FxHarnessDescriptor = {
|
|
|
2255
1769
|
/**
|
|
2256
1770
|
* A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
|
|
2257
1771
|
* `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
|
|
2258
|
-
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
|
|
1772
|
+
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`. On `openrouter()`, either
|
|
1773
|
+
* runs on OpenRouter's Chat Completions API with `OPENROUTER_API_KEY`, a bare id as `openai/<id>`.
|
|
1774
|
+
* On `vertex()`, a Gemini model id, `google/<id>` or a bare `gemini-…` id, run on Vertex AI's
|
|
1775
|
+
* Chat Completions endpoint, e.g. `google/gemini-3.8-flash`.
|
|
2259
1776
|
*/
|
|
2260
1777
|
model: string;
|
|
2261
|
-
/**
|
|
1778
|
+
/**
|
|
1779
|
+
* Reasoning effort, sent as `reasoning_effort` on each model call; only on `vertex()`, since fx
|
|
1780
|
+
* sends none itself. Values are what the model takes there, e.g. `low`, `medium` or `high`.
|
|
1781
|
+
*/
|
|
1782
|
+
effort?: string;
|
|
1783
|
+
/** Cell id; defaults to `fx:<model>`, or `fx:<model>[effort=<value>]` when `effort` is set. */
|
|
2262
1784
|
id?: string;
|
|
2263
1785
|
/** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
|
|
2264
1786
|
maxConcurrent?: number;
|
|
@@ -2298,12 +1820,21 @@ type GrokBuildHarnessDescriptor = {
|
|
|
2298
1820
|
};
|
|
2299
1821
|
/**
|
|
2300
1822
|
* A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
|
|
2301
|
-
* `vercel` is Vercel AI Gateway
|
|
1823
|
+
* `vercel` is Vercel AI Gateway; `openrouter` is OpenRouter; `vertex` is
|
|
1824
|
+
* Google Cloud's Vertex AI.
|
|
1825
|
+
*/
|
|
1826
|
+
type GatewayName = 'direct' | 'vercel' | 'openrouter' | 'vertex';
|
|
1827
|
+
/**
|
|
1828
|
+
* Each provider's own key: made by `direct()` in `reeval/gateway`. Every `gateway` list ends with
|
|
1829
|
+
* it unless it's listed elsewhere, where it's tried in that place.
|
|
2302
1830
|
*/
|
|
2303
|
-
type GatewayName = 'direct' | 'vercel';
|
|
2304
|
-
/** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
|
|
2305
1831
|
type DirectGatewayDescriptor = {
|
|
2306
1832
|
kind: 'direct';
|
|
1833
|
+
/**
|
|
1834
|
+
* `false` turns off the fallback to each provider's own key, so a cell no other gateway in the
|
|
1835
|
+
* list serves is refused at plan time. Default `true`.
|
|
1836
|
+
*/
|
|
1837
|
+
enabled?: boolean;
|
|
2307
1838
|
};
|
|
2308
1839
|
/**
|
|
2309
1840
|
* Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
|
|
@@ -2312,8 +1843,37 @@ type DirectGatewayDescriptor = {
|
|
|
2312
1843
|
type VercelGatewayDescriptor = {
|
|
2313
1844
|
kind: 'vercel';
|
|
2314
1845
|
};
|
|
1846
|
+
/**
|
|
1847
|
+
* OpenRouter, with `OPENROUTER_API_KEY`: made by `openrouter()` in `reeval/gateway`. It serves
|
|
1848
|
+
* Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi.
|
|
1849
|
+
*/
|
|
1850
|
+
type OpenRouterGatewayDescriptor = {
|
|
1851
|
+
kind: 'openrouter';
|
|
1852
|
+
/**
|
|
1853
|
+
* Let OpenRouter fail over to another upstream provider when the one it picked can't serve a
|
|
1854
|
+
* call. Off by default, so reeval's own calls (the judge, analysis, compare questions and
|
|
1855
|
+
* captures) send `allow_fallbacks: false`. The harnesses' calls take OpenRouter's own routing
|
|
1856
|
+
* either way.
|
|
1857
|
+
*/
|
|
1858
|
+
fallbacks?: boolean;
|
|
1859
|
+
};
|
|
1860
|
+
/** A model publisher on Vertex AI, which may serve its models in a region of its own. */
|
|
1861
|
+
type VertexPublisher = 'anthropic' | 'google' | 'xai';
|
|
1862
|
+
/**
|
|
1863
|
+
* Google Cloud's Vertex AI, on Application Default Credentials: made by `vertex()` in
|
|
1864
|
+
* `reeval/gateway`.
|
|
1865
|
+
*/
|
|
1866
|
+
type VertexGatewayDescriptor = {
|
|
1867
|
+
kind: 'vertex';
|
|
1868
|
+
/** The Google Cloud project the calls are billed to; defaults to `GOOGLE_CLOUD_PROJECT`. */
|
|
1869
|
+
project?: string;
|
|
1870
|
+
/** The region the models are called in, e.g. `us-east5`; defaults to `global`. */
|
|
1871
|
+
location?: string;
|
|
1872
|
+
/** A region per publisher, where it differs from `location`, e.g. `{ anthropic: 'us-east5' }`. */
|
|
1873
|
+
regions?: Partial<Record<VertexPublisher, string>>;
|
|
1874
|
+
};
|
|
2315
1875
|
/** One entry of the config's `gateway` list. */
|
|
2316
|
-
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
|
|
1876
|
+
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor | OpenRouterGatewayDescriptor | VertexGatewayDescriptor;
|
|
2317
1877
|
/** A real coding agent, driven through its AI SDK harness adapter. */
|
|
2318
1878
|
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
|
|
2319
1879
|
/** The agent an attempt runs: a real harness, or the fake one. */
|
|
@@ -2892,7 +2452,7 @@ type QuestionOptions = {
|
|
|
2892
2452
|
type PairFactor = 'environment' | 'harness';
|
|
2893
2453
|
/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
|
|
2894
2454
|
type CompareCut = 'within' | 'across';
|
|
2895
|
-
/** One
|
|
2455
|
+
/** One judgment of one pair of attempts, in one order. */
|
|
2896
2456
|
type PairVote = {
|
|
2897
2457
|
question: string;
|
|
2898
2458
|
cut: CompareCut;
|
|
@@ -2986,7 +2546,11 @@ type EvalOptionFields = {
|
|
|
2986
2546
|
* Not an environment: it does not invent skills, files or a name.
|
|
2987
2547
|
*/
|
|
2988
2548
|
sandbox?: SandboxDescriptor;
|
|
2989
|
-
/**
|
|
2549
|
+
/**
|
|
2550
|
+
* Attempts per cell. Unset at every level, a compare with variants runs 4 a side, the fewest
|
|
2551
|
+
* whose lift can reach a verdict, and anything else runs once, a pooled `evaluate.each` case
|
|
2552
|
+
* included.
|
|
2553
|
+
*/
|
|
2990
2554
|
runs?: number;
|
|
2991
2555
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2992
2556
|
gate?: GatePolicy;
|
|
@@ -3014,9 +2578,25 @@ type EvalOptionFields = {
|
|
|
3014
2578
|
timeout?: Duration;
|
|
3015
2579
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
3016
2580
|
tags?: string[];
|
|
2581
|
+
/**
|
|
2582
|
+
* The eval's version, part of its fingerprint: bump it when a change the eval file doesn't show
|
|
2583
|
+
* (a helper, a fixture, data it reads) changes what it measures. A group's applies to the evals
|
|
2584
|
+
* inside it that set none.
|
|
2585
|
+
*/
|
|
2586
|
+
revision?: Revision;
|
|
3017
2587
|
/** Keys a profile's `limit` may not narrow for this group or eval. */
|
|
3018
2588
|
pin?: PinnableKey[];
|
|
3019
2589
|
};
|
|
2590
|
+
/** An eval's `revision`: a whole number or a non-empty string, compared exactly. */
|
|
2591
|
+
type Revision = number | string;
|
|
2592
|
+
/**
|
|
2593
|
+
* An eval's own source as hashes: its body and case data as declared, and its options and its
|
|
2594
|
+
* enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
|
|
2595
|
+
*/
|
|
2596
|
+
type EvalSource = DeclaredSource & {
|
|
2597
|
+
/** sha256 of the canonical options. */
|
|
2598
|
+
options: string;
|
|
2599
|
+
};
|
|
3020
2600
|
/**
|
|
3021
2601
|
* `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
|
|
3022
2602
|
* A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
|
|
@@ -3065,7 +2645,7 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
|
3065
2645
|
type HistoryConfig = {
|
|
3066
2646
|
/** Committed history folder (default: `.history` in the directory the evals live in). */
|
|
3067
2647
|
dir?: string;
|
|
3068
|
-
/** Artifacts recorded next to each run: every
|
|
2648
|
+
/** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
|
|
3069
2649
|
artifacts?: ArtifactLevel;
|
|
3070
2650
|
};
|
|
3071
2651
|
/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
|
|
@@ -3148,74 +2728,6 @@ type FakeAnalysisModel = {
|
|
|
3148
2728
|
answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
|
|
3149
2729
|
error: string;
|
|
3150
2730
|
};
|
|
3151
|
-
/**
|
|
3152
|
-
* What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
|
|
3153
|
-
* function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
|
|
3154
|
-
* or `headline` once; it may return a promise), or `{ error }` to fail every call. It counts a
|
|
3155
|
-
* token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
|
|
3156
|
-
*/
|
|
3157
|
-
insights?: FakeInsights | ((input: FakeInsightsInput) => FakeInsights | Promise<FakeInsights>) | {
|
|
3158
|
-
error: string;
|
|
3159
|
-
};
|
|
3160
|
-
};
|
|
3161
|
-
/** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
|
|
3162
|
-
type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
|
|
3163
|
-
/**
|
|
3164
|
-
* A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
|
|
3165
|
-
* `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
|
|
3166
|
-
* left out is empty, and the intent defaults to `job: 'other'`.
|
|
3167
|
-
*/
|
|
3168
|
-
type FakeInsights = {
|
|
3169
|
-
intent?: {
|
|
3170
|
-
job: 'choose' | 'finish' | 'other';
|
|
3171
|
-
question?: string;
|
|
3172
|
-
subject?: {
|
|
3173
|
-
tool?: string;
|
|
3174
|
-
change?: string;
|
|
3175
|
-
};
|
|
3176
|
-
name?: 'states-intent' | 'no-intent' | 'contradicts';
|
|
3177
|
-
/** `low` falls back to `job: 'other'`, as a real classification does. */
|
|
3178
|
-
confidence?: 'high' | 'medium' | 'low';
|
|
3179
|
-
};
|
|
3180
|
-
answer?: {
|
|
3181
|
-
text: string;
|
|
3182
|
-
cites: string[];
|
|
3183
|
-
} | null;
|
|
3184
|
-
findings?: Array<{
|
|
3185
|
-
text: string;
|
|
3186
|
-
cites: string[];
|
|
3187
|
-
}>;
|
|
3188
|
-
rows?: Array<{
|
|
3189
|
-
row: string;
|
|
3190
|
-
text: string;
|
|
3191
|
-
cites: string[];
|
|
3192
|
-
}>;
|
|
3193
|
-
namedTools?: Array<{
|
|
3194
|
-
attempt: string;
|
|
3195
|
-
tools: string[];
|
|
3196
|
-
}>;
|
|
3197
|
-
/**
|
|
3198
|
-
* The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
|
|
3199
|
-
* `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
|
|
3200
|
-
* They go through the same gates as a model's, so one missing a part is dropped.
|
|
3201
|
-
*/
|
|
3202
|
-
remarks?: Array<{
|
|
3203
|
-
kind: string;
|
|
3204
|
-
text: string;
|
|
3205
|
-
source?: {
|
|
3206
|
-
at: string;
|
|
3207
|
-
quote: string;
|
|
3208
|
-
} | null;
|
|
3209
|
-
evidence?: string[];
|
|
3210
|
-
fix?: {
|
|
3211
|
-
at: string;
|
|
3212
|
-
edit: string;
|
|
3213
|
-
} | null;
|
|
3214
|
-
}>;
|
|
3215
|
-
headline?: {
|
|
3216
|
-
text: string;
|
|
3217
|
-
cites: string[];
|
|
3218
|
-
} | null;
|
|
3219
2731
|
};
|
|
3220
2732
|
/** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
|
|
3221
2733
|
type FakeVote = 'A' | 'B' | 'tie';
|
|
@@ -3269,7 +2781,7 @@ type BlindSide = BlindTurn & {
|
|
|
3269
2781
|
command: string;
|
|
3270
2782
|
notRun: true;
|
|
3271
2783
|
}>;
|
|
3272
|
-
/** Its captures, in the order sent,
|
|
2784
|
+
/** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
|
|
3273
2785
|
artifacts?: Array<{
|
|
3274
2786
|
label: string;
|
|
3275
2787
|
mediaType: 'image/png';
|
|
@@ -3279,24 +2791,13 @@ type BlindSide = BlindTurn & {
|
|
|
3279
2791
|
missing: true;
|
|
3280
2792
|
}>;
|
|
3281
2793
|
};
|
|
3282
|
-
/**
|
|
2794
|
+
/** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
|
|
3283
2795
|
type AnalysisConfig = {
|
|
3284
2796
|
/**
|
|
3285
|
-
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`.
|
|
3286
|
-
*
|
|
2797
|
+
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Unset, `reeval analyze` and questions
|
|
2798
|
+
* fall back to the judge's when it's a language model.
|
|
3287
2799
|
*/
|
|
3288
2800
|
model?: string | FakeAnalysisModel;
|
|
3289
|
-
/**
|
|
3290
|
-
* `reeval run` classifies each eval and explains its result with `model` once the attempts
|
|
3291
|
-
* finish (default true when `model` is set; `--no-insights` turns it off for one run).
|
|
3292
|
-
*/
|
|
3293
|
-
insights?: boolean;
|
|
3294
|
-
/**
|
|
3295
|
-
* Tokens (in and out) that one run's insights may use, over every classification, insight and
|
|
3296
|
-
* headline call (default 250000). Past it, the evals left print without insights; the run's
|
|
3297
|
-
* outcome and exit code never change.
|
|
3298
|
-
*/
|
|
3299
|
-
maxTokens?: number;
|
|
3300
2801
|
/** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
|
|
3301
2802
|
instructions?: string;
|
|
3302
2803
|
/** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
|
|
@@ -3316,7 +2817,7 @@ type ReevalConfig = {
|
|
|
3316
2817
|
sandbox?: SandboxDescriptor;
|
|
3317
2818
|
/** What every eval runs in when it names no environment, and a baseline-less compare's control. */
|
|
3318
2819
|
environment?: Environment;
|
|
3319
|
-
/** Attempts per cell;
|
|
2820
|
+
/** Attempts per cell; unset, a compare with variants runs 4 a side and anything else once. */
|
|
3320
2821
|
runs?: number;
|
|
3321
2822
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
3322
2823
|
gate?: GatePolicy;
|
|
@@ -3350,7 +2851,7 @@ type ReevalConfig = {
|
|
|
3350
2851
|
ci?: string;
|
|
3351
2852
|
local?: string;
|
|
3352
2853
|
};
|
|
3353
|
-
/** `unbrokered: 'error'` refuses Docker secrets without `hosts
|
|
2854
|
+
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
|
|
3354
2855
|
secrets?: {
|
|
3355
2856
|
unbrokered?: 'warn' | 'error';
|
|
3356
2857
|
};
|
|
@@ -3362,11 +2863,12 @@ type ReevalConfig = {
|
|
|
3362
2863
|
keep?: number;
|
|
3363
2864
|
traceBufferMb?: number;
|
|
3364
2865
|
};
|
|
3365
|
-
/**
|
|
2866
|
+
/** `reeval analyze`, and questions without a model of their own. */
|
|
3366
2867
|
analysis?: AnalysisConfig;
|
|
3367
2868
|
/**
|
|
3368
2869
|
* The gateways model calls may go through, in the order you prefer them: each harness takes
|
|
3369
|
-
* the first that can serve it.
|
|
2870
|
+
* the first that can serve it. `direct()`, each provider's own key, ends the list unless it's listed
|
|
2871
|
+
* in another place or turned off with `direct({ enabled: false })`. Default `[direct()]`.
|
|
3370
2872
|
*/
|
|
3371
2873
|
gateway?: GatewayDescriptor[];
|
|
3372
2874
|
};
|
|
@@ -3520,8 +3022,16 @@ type EnvironmentRecord = {
|
|
|
3520
3022
|
sha256: string;
|
|
3521
3023
|
}>;
|
|
3522
3024
|
vars: string[];
|
|
3025
|
+
/**
|
|
3026
|
+
* Every secret, by the variable it's set as in the sandbox; an MCP header's (`secret.fromEnv`,
|
|
3027
|
+
* `secret.template`, a plugin's `${VAR}`) by each host variable it reads.
|
|
3028
|
+
*/
|
|
3523
3029
|
secrets: string[];
|
|
3524
|
-
/**
|
|
3030
|
+
/**
|
|
3031
|
+
* Secrets the sandbox only had placeholders for, swapped in by the egress gateway, named as in
|
|
3032
|
+
* `secrets`. A variable it also got as its real value somewhere (an `http://` header, an env
|
|
3033
|
+
* secret without `hosts`) isn't listed.
|
|
3034
|
+
*/
|
|
3525
3035
|
brokered: string[];
|
|
3526
3036
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3527
3037
|
unset: string[];
|
|
@@ -3611,6 +3121,16 @@ type StartRetry = {
|
|
|
3611
3121
|
/** How long it waited before giving up on that start. */
|
|
3612
3122
|
waitedMs: number;
|
|
3613
3123
|
};
|
|
3124
|
+
/**
|
|
3125
|
+
* A process the setup steps left running in a Docker sandbox with a `to: 'checks'` secret without
|
|
3126
|
+
* `hosts` in its environment: it holds the real value while the agent works.
|
|
3127
|
+
*/
|
|
3128
|
+
type SetupProcess = {
|
|
3129
|
+
/** Its command line, cut at 200 characters. */
|
|
3130
|
+
command: string;
|
|
3131
|
+
/** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
|
|
3132
|
+
secrets: string[];
|
|
3133
|
+
};
|
|
3614
3134
|
/** One `expect` or `expect.soft` matcher call an attempt made. */
|
|
3615
3135
|
type AssertionRecord = {
|
|
3616
3136
|
/**
|
|
@@ -3665,5 +3185,5 @@ type ResolvedGateway = {
|
|
|
3665
3185
|
}>;
|
|
3666
3186
|
};
|
|
3667
3187
|
//#endregion
|
|
3668
|
-
export {
|
|
3669
|
-
//# sourceMappingURL=types-
|
|
3188
|
+
export { FakeVote as $, VertexPublisher as $t, Diff as A, VERDICT_TEXT as An, Profile as At, EnvironmentInput as B, effortSample as Bn, Secret as Bt, CodexHarnessDescriptor as C, EFFORT_MIN_RUNS as Cn, Outcome as Ct, ConfigHelpers as D, EffortVerdict as Dn, PinnableKey as Dt, CompareOptions as E, EffortSample as En, PiHarnessDescriptor as Et, EgressPolicy as F, contrastEffort as Fn, QuestionStateOptions as Ft, ExecResult as G, formatSeconds as Gn, TokenUsage as Gt, ErrorKind as H, effortVerdict as Hn, ServeHandle as Ht, EgressRecord as I, directional as In, ReevalConfig as It, FakeAnswer as J, DeclaredNode as Jn, TurnData as Jt, ExecRetry as K, tokenValues as Kn, TranscriptEvent as Kt, EnvVars as L, effortChangeLabel as Ln, ResolvedGateway as Lt, DirectoryOptions as M, cellContrast as Mn, ProfileLimit as Mt, DockerSandboxDescriptor as N, cellEffort as Nn, Question as Nt, ConfigJudgeOptions as O, Spread as On, PluginPart as Ot, Duration as P, cellSamples as Pn, QuestionOptions as Pt, FakeScriptStep as Q, VertexGatewayDescriptor as Qt, Environment as R, effortLabel as Rn, SandboxDescriptor as Rt, ClaudeCodeHarnessDescriptor as S, EFFORT_ALPHA as Sn, OpenRouterGatewayDescriptor as St, CompareCut as T, EffortMetric as Tn, PairVote as Tt, EvalFn as U, formatCount as Un, SetupStep as Ut, EnvironmentSpec as V, effortSummary as Vn, SecretTemplate as Vt, EvalOptions as W, formatRatio as Wn, SkillSource as Wt, FakeJudgeModel as X, VercelGatewayDescriptor as Xt, FakeHarnessDescriptor as Y, TurnStatus as Yt, FakeSandboxDescriptor as Z, Verdict as Zt, CaptureWait as _, RunRecord as _n, JudgeOptions as _t, AttemptContext as a, CellRecord as an, GroupOptions as at, CellRole as b, CellEffort as bn, McpServer as bt, BlindPair as c, FamilyLift as cn, HarnessEdit as ct, CaptureCall as d, FamilyRecord as dn, HostFile as dt, Viewport as en, FxHarnessDescriptor as et, CaptureInstruction as f, FileRef as fn, JSONValue as ft, CaptureUseScript as g, Lift as gn, JudgeModel as gt, CaptureUseAction as h, JudgmentRecord as hn, JudgeFailure as ht, AttemptAnswer as i, AttemptRecord as in, GrokBuildHarnessDescriptor as it, DirectGatewayDescriptor as j, bothUnderATenth as jn, ProfileDefaults as jt, CursorHarnessDescriptor as k, TOKENS_UNREPORTED as kn, PluginSource as kt, Budget as l, FamilyLiftSummary as ln, HarnessOption as lt, CaptureUse as m, HistoryEntry as mn, JudgeCallOptions as mt, AnalysisConfig as n, WebEventType as nn, GatewayName as nt, BackgroundHandle as o, CheckTally as on, HarnessData as ot, CaptureOptions as p, HistoryCell as pn, JudgeArtifactRef as pt, FakeAnalysisModel as q, DeclarationTree as qn, Turn as qt, AntigravityHarnessDescriptor as r, ArtifactRef as rn, GooseHarnessDescriptor as rt, BlindAttempt as s, EvalEntry as sn, HarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, GatewayDescriptor as tt, Capture as u, FamilyRate as un, HistoryConfig as ut, CaptureWindow as v, StackSummary as vn, JudgeState as vt, CompareAnswer as w, EffortContrast as wn, PairFactor as wt, ClaudeCodeEffort as x, DIRECTIONAL_MIN as xn, OpenCodeHarnessDescriptor as xt, Cell as y, UpstreamCount as yn, JudgeStateOptions as yt, EnvironmentHelpers as z, effortOf as zn, SandboxWarning as zt };
|
|
3189
|
+
//# sourceMappingURL=types-coMWHaU2.d.ts.map
|