reeval 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/dist/{adapter-IKKo4sZj.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-m-_t-IuW.js} +53 -25
- package/dist/{agent-ClE-g9VL.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-BDEpelpi.js} +92 -35
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-CVRwRTnb.js} +572 -36
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-C_4gaJ50.js} +132 -140
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C6vwmjS7.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C6vwmjS7.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CANun0m0.js → command-BXLUfw4h.js} +12 -12
- package/dist/{command-CANun0m0.js.map → command-BXLUfw4h.js.map} +1 -1
- package/dist/{command-CBaHukyl.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CBaHukyl.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-D_kEnYcX.js → command-Bxdy62dW.js} +14 -13
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-DJUuFoWQ.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-BnRlTpZA.js} +2 -2
- package/dist/{docker-DuHcQidD.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CQxucRPe.js → docker-D01slDF3.js} +26 -10
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-DQ2p0euG.js} +135 -36
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-CO0P0soF.js} +8 -7
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BUmsN7yG.js} +66 -27
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-C9-ritA5.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-Cfx1H70y.js → facts-DzOUTreB.js} +12 -4
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-CN1QfenC.js → files-B-CJeeEh.js} +13 -8
- package/dist/{files-CN1QfenC.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-Bs6H3HiG.js → files-C_gQNVB-.js} +3 -3
- package/dist/{files-Bs6H3HiG.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-D27nUk0K.js → files-kUCrswI6.js} +2 -2
- package/dist/{files-D27nUk0K.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-BnrvBAzF.js → generate-DZ7vp69F.js} +18 -17
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-Bk8DSwHy.js} +2 -2
- package/dist/{git-rPfl7AD1.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-1OkumOAm.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-BVzbRnNv.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-BVzbRnNv.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-m784ueGK.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-D_ePcwyL.js → invocations-DzyKSJBa.js} +2 -2
- package/dist/{invocations-D_ePcwyL.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-1ZLr9Dcv.js} +40 -26
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-UgHDMhI1.js → model-CgnOpss4.js} +57 -318
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/{model-B8H3gL69.js → model-Ch8kX87u.js} +1 -1
- package/dist/{node-CGZYh4Fj.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-snfeWd6Z.js → opencode-providers-klkqgzvb.js} +23 -52
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-uMq9rCl_.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-CZYgXnZh.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-Dkp_H3pB.js} +4 -4
- package/dist/{plugins-P-uh54VF.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-D3HvKeAx.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-Dx7dOnJB.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-CfZpFpAh.js} +2 -2
- package/dist/{provenance-C_-8y-x3.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-DXthKmq0.js → query-CemclveB.js} +2 -2
- package/dist/{query-DXthKmq0.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-KENpWnfV.js → questions-bAudIc04.js} +308 -45
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{remarks-Ml6bcSdV.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-Ml6bcSdV.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-Dkeghk8k.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-CBoKzQMP.js} +8 -9
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-D-naai_Q.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-8-nnW8iO.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-8-nnW8iO.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{rules-CcV3zLDR.js → rules-VhftPOOT.js} +53 -5
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-Bel7Vde4.js → run-CwIEK16q.js} +9 -9
- package/dist/{run-Bel7Vde4.js.map → run-CwIEK16q.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-De8oH3bt.js → server-TGW4WyAz.js} +25 -26
- package/dist/{server-De8oH3bt.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-C686eZpY.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-C686eZpY.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-CIpKL45U.js → src-8zoX_qrb.js} +36 -26
- package/dist/{src-CIpKL45U.js.map → src-8zoX_qrb.js.map} +1 -1
- package/dist/{src-CzQrAcF8.js → src-CHIwOy0o.js} +84 -52
- package/dist/src-CHIwOy0o.js.map +1 -0
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-djNNYIdP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-djNNYIdP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/{src-DrJSnoMR.js → src-CyaDNG9r.js} +1 -1
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-C7W6kg81.js → src-DKPVX2Us.js} +34 -16
- package/dist/{src-C7W6kg81.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-DmUgeTlE.js} +4 -4
- package/dist/{src-DeY52Gyf.js.map → src-DmUgeTlE.js.map} +1 -1
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-Xj_BZKbq.js → src-NSdQ9zI6.js} +31 -16
- package/dist/{src-Xj_BZKbq.js.map → src-NSdQ9zI6.js.map} +1 -1
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-C0i_ujQZ.js → state-B1t143wG.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-WgT7m6Vs.js} +76 -11
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-DXBbP9Mk.d.ts} +298 -70
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-DWjkshl1.js → validate-BhwqrVsm.js} +44 -12
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +133 -86
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-Dk8MCQKr.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-CqrzGPF4.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-CLeOybkr.js → workspace-CDI68wtX.js} +4 -4
- package/dist/{workspace-CLeOybkr.js.map → workspace-CDI68wtX.js.map} +1 -1
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +6 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -200,6 +200,8 @@ type JudgementRecord = {
|
|
|
200
200
|
model: string;
|
|
201
201
|
/** The model id the provider says served the call, when it reports one. */
|
|
202
202
|
resolvedModel?: string;
|
|
203
|
+
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
204
|
+
gateway?: string;
|
|
203
205
|
/**
|
|
204
206
|
* `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
|
|
205
207
|
* verdict can be followed across runs (and an edited question starts a new series).
|
|
@@ -257,6 +259,12 @@ type AttemptRecord = {
|
|
|
257
259
|
* recorded them.
|
|
258
260
|
*/
|
|
259
261
|
servedModels: string[] | null;
|
|
262
|
+
/**
|
|
263
|
+
* The models the harness's successful API calls asked for, when nothing named the one that
|
|
264
|
+
* served them (fx on Vercel AI Gateway with a Gemini model): the agent's word, not the
|
|
265
|
+
* gateway's, so never part of `servedModels`. Absent when there's none.
|
|
266
|
+
*/
|
|
267
|
+
askedModels?: string[];
|
|
260
268
|
/**
|
|
261
269
|
* The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
|
|
262
270
|
* null when the factory left it unset. Fast is not this.
|
|
@@ -320,11 +328,19 @@ type AttemptRecord = {
|
|
|
320
328
|
judgements: JudgementRecord[];
|
|
321
329
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
322
330
|
startRetries?: StartRetry[];
|
|
331
|
+
/**
|
|
332
|
+
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
333
|
+
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
334
|
+
*/
|
|
323
335
|
checks: Array<{
|
|
324
336
|
command: string;
|
|
325
337
|
exitCode: number;
|
|
326
338
|
durationMs: number;
|
|
327
339
|
log?: FileRef;
|
|
340
|
+
/** The `retries` it was given, when more than 0: it could run `retries + 1` times. */
|
|
341
|
+
retries?: number;
|
|
342
|
+
/** Every try, in order, when it was given `retries`; its log has each try's output. */
|
|
343
|
+
tries?: ExecTry[];
|
|
328
344
|
}>;
|
|
329
345
|
/** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
|
|
330
346
|
assertions?: AssertionRecord[];
|
|
@@ -333,6 +349,13 @@ type AttemptRecord = {
|
|
|
333
349
|
count?: number;
|
|
334
350
|
}>;
|
|
335
351
|
files: AttemptFiles;
|
|
352
|
+
/**
|
|
353
|
+
* The eval's questions asked of this attempt on its own (those without `across`), in question
|
|
354
|
+
* order; unset without any. They annotate: `outcome` never depends on them. Asked once the
|
|
355
|
+
* attempt ends, when it ran to a pass or a fail. Each answer's `input` is `questions/<n>.json`
|
|
356
|
+
* in the attempt's directory, `<n>` the question's place in the eval's `questions`.
|
|
357
|
+
*/
|
|
358
|
+
answers?: AttemptAnswer[];
|
|
336
359
|
};
|
|
337
360
|
type EvalEntry = {
|
|
338
361
|
/** `file › group › … › name`, as shown. */
|
|
@@ -367,8 +390,8 @@ type EvalEntry = {
|
|
|
367
390
|
cells: CellRecord[];
|
|
368
391
|
attempts: AttemptRecord[];
|
|
369
392
|
/**
|
|
370
|
-
* The
|
|
371
|
-
* unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
393
|
+
* The eval's pairwise questions (those with `across`), answered after its attempts by votes,
|
|
394
|
+
* one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
372
395
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
373
396
|
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
374
397
|
* `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
|
|
@@ -523,6 +546,8 @@ type FingerprintParts = {
|
|
|
523
546
|
judge: string;
|
|
524
547
|
/** Over the attempt's context files. */
|
|
525
548
|
context: string;
|
|
549
|
+
/** The gateway its model calls went through; unset on `direct`, each provider's own key. */
|
|
550
|
+
gateway?: string;
|
|
526
551
|
};
|
|
527
552
|
/** One harness and environment of an eval, and how its attempts went. */
|
|
528
553
|
type CellRecord = Cell & {
|
|
@@ -532,6 +557,11 @@ type CellRecord = Cell & {
|
|
|
532
557
|
/** The environment's digest: the join key into `RunRecord.environments`; null for none. */
|
|
533
558
|
environmentDigest: string | null;
|
|
534
559
|
harnessKind: string;
|
|
560
|
+
/**
|
|
561
|
+
* The gateway in the config's `gateway` list its harness resolved to (`direct` is each
|
|
562
|
+
* provider's own key); unset when no config, profile or pin named one.
|
|
563
|
+
*/
|
|
564
|
+
gateway?: string;
|
|
535
565
|
/** The configured harness model, when it has one. */
|
|
536
566
|
model: string | null;
|
|
537
567
|
/**
|
|
@@ -885,7 +915,8 @@ type SuiteResult = {
|
|
|
885
915
|
errors: string[];
|
|
886
916
|
/**
|
|
887
917
|
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
888
|
-
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors
|
|
918
|
+
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
|
|
919
|
+
* `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
|
|
889
920
|
*/
|
|
890
921
|
tally: SuiteTally;
|
|
891
922
|
/** Sorted by path. */
|
|
@@ -896,6 +927,7 @@ type SuiteResult = {
|
|
|
896
927
|
type SuiteTally = {
|
|
897
928
|
errors: number;
|
|
898
929
|
pass: number;
|
|
930
|
+
notGated: number;
|
|
899
931
|
fail: number;
|
|
900
932
|
infraError: number;
|
|
901
933
|
cancelled: number;
|
|
@@ -936,13 +968,14 @@ type EvalResult = {
|
|
|
936
968
|
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
937
969
|
pooled: PooledArm[] | null;
|
|
938
970
|
/**
|
|
939
|
-
* Its
|
|
940
|
-
*
|
|
971
|
+
* Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
|
|
972
|
+
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
941
973
|
*/
|
|
942
|
-
questions: QuestionResult
|
|
974
|
+
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
943
975
|
};
|
|
944
|
-
/** A
|
|
976
|
+
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
945
977
|
type QuestionResult = {
|
|
978
|
+
kind: 'pairs';
|
|
946
979
|
question: string;
|
|
947
980
|
/** Pairs voted, each counted once per order it was asked in. */
|
|
948
981
|
votes: number;
|
|
@@ -955,6 +988,27 @@ type QuestionResult = {
|
|
|
955
988
|
/** Why some or all of its votes are missing: the model call failed. */
|
|
956
989
|
error: string | null;
|
|
957
990
|
};
|
|
991
|
+
/** A question asked of each attempt on its own: what each one answered. Never gates. */
|
|
992
|
+
type AttemptQuestionResult = {
|
|
993
|
+
kind: 'attempts';
|
|
994
|
+
question: string;
|
|
995
|
+
/** Attempts it was asked of, failed calls included. */
|
|
996
|
+
asked: number;
|
|
997
|
+
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
998
|
+
choices: Array<{
|
|
999
|
+
choice: string;
|
|
1000
|
+
n: number;
|
|
1001
|
+
}> | null;
|
|
1002
|
+
/** Each attempt's answer, in attempt order; failed calls left out. */
|
|
1003
|
+
answers: Array<{
|
|
1004
|
+
attempt: string;
|
|
1005
|
+
choice: string | null;
|
|
1006
|
+
text: string;
|
|
1007
|
+
}>;
|
|
1008
|
+
/** Calls that failed, and the first one's error. */
|
|
1009
|
+
failed: number;
|
|
1010
|
+
error: string | null;
|
|
1011
|
+
};
|
|
958
1012
|
type CaseResult = {
|
|
959
1013
|
path: string;
|
|
960
1014
|
/** The case's own eval id. */
|
|
@@ -1033,6 +1087,12 @@ type CheckResult = {
|
|
|
1033
1087
|
effect: Effect | null;
|
|
1034
1088
|
/** Set when the check is a judge verdict. */
|
|
1035
1089
|
judge: JudgeResult | null;
|
|
1090
|
+
/**
|
|
1091
|
+
* A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
|
|
1092
|
+
* it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
|
|
1093
|
+
* when none did.
|
|
1094
|
+
*/
|
|
1095
|
+
retried?: string;
|
|
1036
1096
|
};
|
|
1037
1097
|
/**
|
|
1038
1098
|
* `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
|
|
@@ -1107,6 +1167,13 @@ declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'enviro
|
|
|
1107
1167
|
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1108
1168
|
*/
|
|
1109
1169
|
declare function messageLine(text: string): string;
|
|
1170
|
+
/** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
|
|
1171
|
+
declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
|
|
1172
|
+
/**
|
|
1173
|
+
* A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
|
|
1174
|
+
* gating it, so it reads `not gated`, never `pass`.
|
|
1175
|
+
*/
|
|
1176
|
+
declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
|
|
1110
1177
|
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1111
1178
|
declare function suiteResult(record: RunRecord): SuiteResult;
|
|
1112
1179
|
//#endregion
|
|
@@ -1260,7 +1327,7 @@ declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
|
1260
1327
|
severity: 'concern' | 'suggestion';
|
|
1261
1328
|
}>;
|
|
1262
1329
|
/** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
|
|
1263
|
-
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "slow-mcp-start" | "too-few-tries")[];
|
|
1330
|
+
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
|
|
1264
1331
|
/** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
|
|
1265
1332
|
type Remark = {
|
|
1266
1333
|
/**
|
|
@@ -1270,7 +1337,7 @@ type Remark = {
|
|
|
1270
1337
|
id: string;
|
|
1271
1338
|
evalId: string;
|
|
1272
1339
|
/** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
|
|
1273
|
-
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start';
|
|
1340
|
+
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
|
|
1274
1341
|
/** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
|
|
1275
1342
|
severity: 'concern' | 'suggestion';
|
|
1276
1343
|
/** One plain sentence: what's wrong. */
|
|
@@ -1496,6 +1563,18 @@ type ExecResult = {
|
|
|
1496
1563
|
stdout: string;
|
|
1497
1564
|
stderr: string;
|
|
1498
1565
|
};
|
|
1566
|
+
/** How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient. */
|
|
1567
|
+
type ExecRetry = {
|
|
1568
|
+
/** Extra tries after a non-zero exit, a whole number from 0. Default 0. */
|
|
1569
|
+
retries?: number;
|
|
1570
|
+
/** Milliseconds to wait before each extra try. Default 1000. */
|
|
1571
|
+
interval?: number;
|
|
1572
|
+
};
|
|
1573
|
+
/** One run of a `c.exec` command that had `retries`. */
|
|
1574
|
+
type ExecTry = {
|
|
1575
|
+
exitCode: number;
|
|
1576
|
+
durationMs: number;
|
|
1577
|
+
};
|
|
1499
1578
|
/** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
|
|
1500
1579
|
type CaptureWindow = {
|
|
1501
1580
|
/** In CSS pixels. */
|
|
@@ -1569,8 +1648,8 @@ type CaptureOptions = {
|
|
|
1569
1648
|
/** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
|
|
1570
1649
|
browser?: boolean;
|
|
1571
1650
|
/**
|
|
1572
|
-
* Vision model for `instructions`. Cascades like `question`. Default is
|
|
1573
|
-
*
|
|
1651
|
+
* Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else
|
|
1652
|
+
* the attempt's judge model, whichever is first a language model; never Jev.
|
|
1574
1653
|
*/
|
|
1575
1654
|
model?: string;
|
|
1576
1655
|
};
|
|
@@ -1665,7 +1744,7 @@ type TurnData = {
|
|
|
1665
1744
|
};
|
|
1666
1745
|
type Turn = TurnData & {
|
|
1667
1746
|
/** Runs in the eval's sandbox. Same as `c.exec`. */
|
|
1668
|
-
exec(command: string): Promise<ExecResult>;
|
|
1747
|
+
exec(command: string, options?: ExecRetry): Promise<ExecResult>;
|
|
1669
1748
|
readFile(path: string): Promise<string | null>;
|
|
1670
1749
|
};
|
|
1671
1750
|
/** Any JSON value. */
|
|
@@ -1730,6 +1809,11 @@ type JudgeOptions = {
|
|
|
1730
1809
|
* attempt's state. Merged key by key down the cascade.
|
|
1731
1810
|
*/
|
|
1732
1811
|
state?: JudgeStateOptions;
|
|
1812
|
+
/**
|
|
1813
|
+
* The gateway the judge's calls go through, by name or by its helper. It skips the config's
|
|
1814
|
+
* `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
|
|
1815
|
+
*/
|
|
1816
|
+
gateway?: GatewayName | GatewayDescriptor;
|
|
1733
1817
|
};
|
|
1734
1818
|
/**
|
|
1735
1819
|
* What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
|
|
@@ -1798,6 +1882,8 @@ type Verdict = {
|
|
|
1798
1882
|
model: string;
|
|
1799
1883
|
/** The model id the provider says served the call, when it reports one. */
|
|
1800
1884
|
resolvedModel?: string;
|
|
1885
|
+
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
1886
|
+
gateway?: GatewayName;
|
|
1801
1887
|
/** Captures this call sent the judge as file parts, when any. */
|
|
1802
1888
|
artifacts?: JudgeArtifactRef[];
|
|
1803
1889
|
};
|
|
@@ -1822,11 +1908,17 @@ type AttemptContext = {
|
|
|
1822
1908
|
* `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
|
|
1823
1909
|
*/
|
|
1824
1910
|
judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1825
|
-
|
|
1911
|
+
/**
|
|
1912
|
+
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1913
|
+
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
1914
|
+
*/
|
|
1915
|
+
exec(command: string, options?: ExecRetry & {
|
|
1916
|
+
background?: false;
|
|
1917
|
+
}): Promise<ExecResult>;
|
|
1826
1918
|
exec(command: string, options: {
|
|
1827
1919
|
background: true;
|
|
1828
1920
|
}): Promise<BackgroundHandle>;
|
|
1829
|
-
exec(command: string, options?: {
|
|
1921
|
+
exec(command: string, options?: ExecRetry & {
|
|
1830
1922
|
background?: boolean;
|
|
1831
1923
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1832
1924
|
/**
|
|
@@ -1863,6 +1955,11 @@ type EvalFn = (c: AttemptContext) => Promise<void>;
|
|
|
1863
1955
|
/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
|
|
1864
1956
|
type FakeHarnessDescriptor = {
|
|
1865
1957
|
kind: 'fake';
|
|
1958
|
+
/**
|
|
1959
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1960
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1961
|
+
*/
|
|
1962
|
+
gateway?: GatewayName;
|
|
1866
1963
|
/** Cell id; defaults to `fake`. */
|
|
1867
1964
|
id?: string;
|
|
1868
1965
|
/** Files the fake agent writes into the sandbox on every turn. */
|
|
@@ -1949,6 +2046,11 @@ type FakeScriptStep = {
|
|
|
1949
2046
|
*/
|
|
1950
2047
|
type CursorHarnessDescriptor = {
|
|
1951
2048
|
kind: 'cursor';
|
|
2049
|
+
/**
|
|
2050
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2051
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2052
|
+
*/
|
|
2053
|
+
gateway?: GatewayName;
|
|
1952
2054
|
/** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
|
|
1953
2055
|
model: string;
|
|
1954
2056
|
/**
|
|
@@ -1976,6 +2078,11 @@ type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
|
1976
2078
|
/** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
|
|
1977
2079
|
type ClaudeCodeHarnessDescriptor = {
|
|
1978
2080
|
kind: 'claude-code';
|
|
2081
|
+
/**
|
|
2082
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2083
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2084
|
+
*/
|
|
2085
|
+
gateway?: GatewayName;
|
|
1979
2086
|
/** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
|
|
1980
2087
|
model: string;
|
|
1981
2088
|
/**
|
|
@@ -1996,6 +2103,11 @@ type ClaudeCodeHarnessDescriptor = {
|
|
|
1996
2103
|
/** The Antigravity CLI through `@reeval/harness-antigravity`, made by `antigravity()`. */
|
|
1997
2104
|
type AntigravityHarnessDescriptor = {
|
|
1998
2105
|
kind: 'antigravity';
|
|
2106
|
+
/**
|
|
2107
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2108
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2109
|
+
*/
|
|
2110
|
+
gateway?: GatewayName;
|
|
1999
2111
|
/** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
|
|
2000
2112
|
model: string;
|
|
2001
2113
|
/**
|
|
@@ -2020,6 +2132,11 @@ type AntigravityHarnessDescriptor = {
|
|
|
2020
2132
|
*/
|
|
2021
2133
|
type OpenCodeHarnessDescriptor = {
|
|
2022
2134
|
kind: 'opencode';
|
|
2135
|
+
/**
|
|
2136
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2137
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2138
|
+
*/
|
|
2139
|
+
gateway?: GatewayName;
|
|
2023
2140
|
/**
|
|
2024
2141
|
* OpenCode's model id, `<provider>/<model>`, e.g. `google/gemini-3.8-flash` or
|
|
2025
2142
|
* `anthropic/claude-haiku-4-5`; the provider picks the key.
|
|
@@ -2046,6 +2163,11 @@ type OpenCodeHarnessDescriptor = {
|
|
|
2046
2163
|
*/
|
|
2047
2164
|
type GooseHarnessDescriptor = {
|
|
2048
2165
|
kind: 'goose';
|
|
2166
|
+
/**
|
|
2167
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2168
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2169
|
+
*/
|
|
2170
|
+
gateway?: GatewayName;
|
|
2049
2171
|
/** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
|
|
2050
2172
|
model: string;
|
|
2051
2173
|
/**
|
|
@@ -2069,6 +2191,11 @@ type GooseHarnessDescriptor = {
|
|
|
2069
2191
|
*/
|
|
2070
2192
|
type PiHarnessDescriptor = {
|
|
2071
2193
|
kind: 'pi';
|
|
2194
|
+
/**
|
|
2195
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2196
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2197
|
+
*/
|
|
2198
|
+
gateway?: GatewayName;
|
|
2072
2199
|
/** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
|
|
2073
2200
|
model: string;
|
|
2074
2201
|
/**
|
|
@@ -2092,6 +2219,11 @@ type PiHarnessDescriptor = {
|
|
|
2092
2219
|
*/
|
|
2093
2220
|
type CodexHarnessDescriptor = {
|
|
2094
2221
|
kind: 'codex';
|
|
2222
|
+
/**
|
|
2223
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2224
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2225
|
+
*/
|
|
2226
|
+
gateway?: GatewayName;
|
|
2095
2227
|
/** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
|
|
2096
2228
|
model: string;
|
|
2097
2229
|
/**
|
|
@@ -2115,10 +2247,15 @@ type CodexHarnessDescriptor = {
|
|
|
2115
2247
|
*/
|
|
2116
2248
|
type FxHarnessDescriptor = {
|
|
2117
2249
|
kind: 'fx';
|
|
2250
|
+
/**
|
|
2251
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2252
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2253
|
+
*/
|
|
2254
|
+
gateway?: GatewayName;
|
|
2118
2255
|
/**
|
|
2119
2256
|
* A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
|
|
2120
2257
|
* `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
|
|
2121
|
-
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-
|
|
2258
|
+
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
|
|
2122
2259
|
*/
|
|
2123
2260
|
model: string;
|
|
2124
2261
|
/** Cell id; defaults to `fx:<model>`. */
|
|
@@ -2137,6 +2274,11 @@ type FxHarnessDescriptor = {
|
|
|
2137
2274
|
*/
|
|
2138
2275
|
type GrokBuildHarnessDescriptor = {
|
|
2139
2276
|
kind: 'grok-build';
|
|
2277
|
+
/**
|
|
2278
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2279
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2280
|
+
*/
|
|
2281
|
+
gateway?: GatewayName;
|
|
2140
2282
|
/** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
|
|
2141
2283
|
model: string;
|
|
2142
2284
|
/**
|
|
@@ -2154,6 +2296,24 @@ type GrokBuildHarnessDescriptor = {
|
|
|
2154
2296
|
*/
|
|
2155
2297
|
startTimeout?: Duration;
|
|
2156
2298
|
};
|
|
2299
|
+
/**
|
|
2300
|
+
* A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
|
|
2301
|
+
* `vercel` is Vercel AI Gateway.
|
|
2302
|
+
*/
|
|
2303
|
+
type GatewayName = 'direct' | 'vercel';
|
|
2304
|
+
/** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
|
|
2305
|
+
type DirectGatewayDescriptor = {
|
|
2306
|
+
kind: 'direct';
|
|
2307
|
+
};
|
|
2308
|
+
/**
|
|
2309
|
+
* Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
|
|
2310
|
+
* `reeval/gateway`. It serves Claude Code, Codex, Goose, pi, OpenCode, fx and Grok Build.
|
|
2311
|
+
*/
|
|
2312
|
+
type VercelGatewayDescriptor = {
|
|
2313
|
+
kind: 'vercel';
|
|
2314
|
+
};
|
|
2315
|
+
/** One entry of the config's `gateway` list. */
|
|
2316
|
+
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
|
|
2157
2317
|
/** A real coding agent, driven through its AI SDK harness adapter. */
|
|
2158
2318
|
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
|
|
2159
2319
|
/** The agent an attempt runs: a real harness, or the fake one. */
|
|
@@ -2359,12 +2519,13 @@ type SetupStep = string | {
|
|
|
2359
2519
|
cwd?: string;
|
|
2360
2520
|
};
|
|
2361
2521
|
/**
|
|
2362
|
-
* Where the sandbox may connect:
|
|
2363
|
-
*
|
|
2364
|
-
*
|
|
2365
|
-
*
|
|
2522
|
+
* Where the sandbox may connect: any host over HTTP(S) through the egress gateway (`'proxy'`, the
|
|
2523
|
+
* default), anywhere including raw TCP and UDP around the gateway (`'allow-all'`), nowhere
|
|
2524
|
+
* (`'deny-all'`), or only the listed hosts (`'*.example.com'` matches subdomains). Under a
|
|
2525
|
+
* restricted policy the harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are
|
|
2526
|
+
* allowed too; any other request gets a refusal, and the attempt goes on.
|
|
2366
2527
|
*/
|
|
2367
|
-
type EgressPolicy = 'allow-all' | 'deny-all' | {
|
|
2528
|
+
type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
|
|
2368
2529
|
allow: readonly string[];
|
|
2369
2530
|
};
|
|
2370
2531
|
/**
|
|
@@ -2651,45 +2812,40 @@ type CompareOptions<E = Environment> = {
|
|
|
2651
2812
|
minLift?: number;
|
|
2652
2813
|
/** Harnesses (ids or descriptors) to compare on; the others run the control only. */
|
|
2653
2814
|
harness?: Array<string | HarnessDescriptor>;
|
|
2654
|
-
/**
|
|
2655
|
-
* Asked of pairs of attempts after the run, always a list. They annotate the compare with
|
|
2656
|
-
* win rates and never change pass, fail or the exit code.
|
|
2657
|
-
*/
|
|
2658
|
-
questions?: CompareQuestion[];
|
|
2659
|
-
/**
|
|
2660
|
-
* The factor every question varies when pairing, when a question does not name its own
|
|
2661
|
-
* `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
|
|
2662
|
-
* both, or omitted, asks both pair kinds.
|
|
2663
|
-
*/
|
|
2664
|
-
across?: PairFactor | PairFactor[];
|
|
2665
2815
|
};
|
|
2666
2816
|
/**
|
|
2667
|
-
* A question
|
|
2668
|
-
*
|
|
2669
|
-
*
|
|
2670
|
-
* `
|
|
2671
|
-
* `
|
|
2817
|
+
* A question a language model answers after the attempts: its text, or the text and its options.
|
|
2818
|
+
* With `across`, it picks between two attempts that differ only in that factor, in both orders;
|
|
2819
|
+
* without it, it's asked of each attempt on its own. Without a model, the cascaded
|
|
2820
|
+
* `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
|
|
2821
|
+
* `state` and `artifacts` choose what it's shown of each attempt.
|
|
2672
2822
|
*/
|
|
2673
|
-
type
|
|
2823
|
+
type Question = string | {
|
|
2674
2824
|
question: string;
|
|
2675
2825
|
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
2676
2826
|
model?: string;
|
|
2677
|
-
/**
|
|
2678
|
-
|
|
2679
|
-
|
|
2827
|
+
/**
|
|
2828
|
+
* The factor it varies when pairing, or both: `'environment'` holds the harness,
|
|
2829
|
+
* `'harness'` holds the environment. Unset, it's asked of each attempt on its own.
|
|
2830
|
+
*/
|
|
2831
|
+
across?: PairFactor | PairFactor[];
|
|
2832
|
+
/** The answers a question asked of each attempt picks from, with a short reason. */
|
|
2833
|
+
choices?: string[];
|
|
2834
|
+
/** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
|
|
2680
2835
|
state?: QuestionStateOptions;
|
|
2681
|
-
/** Each
|
|
2836
|
+
/** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
|
|
2682
2837
|
artifacts?: true | string[];
|
|
2683
2838
|
};
|
|
2684
2839
|
/**
|
|
2685
|
-
* What
|
|
2686
|
-
* attempts' workspaces are gone when questions are asked). An unset key keeps its default
|
|
2687
|
-
*
|
|
2840
|
+
* What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
|
|
2841
|
+
* attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
|
|
2842
|
+
* pairwise question shows the task and the diff; one asked of each attempt also shows the final
|
|
2843
|
+
* message and the transcript.
|
|
2688
2844
|
*/
|
|
2689
2845
|
type QuestionStateOptions = {
|
|
2690
2846
|
/** The prompts the attempt was sent, once for both sides when they match. Default true. */
|
|
2691
2847
|
task?: boolean;
|
|
2692
|
-
/** The final message: the agent's own account of its work. Default false. */
|
|
2848
|
+
/** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
|
|
2693
2849
|
output?: boolean;
|
|
2694
2850
|
/** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
|
|
2695
2851
|
files?: boolean;
|
|
@@ -2701,7 +2857,8 @@ type QuestionStateOptions = {
|
|
|
2701
2857
|
/**
|
|
2702
2858
|
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
2703
2859
|
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2704
|
-
* harness's tools, so a pair of two harnesses isn't blind. Default false
|
|
2860
|
+
* harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
|
|
2861
|
+
* attempt.
|
|
2705
2862
|
*/
|
|
2706
2863
|
transcript?: boolean | 'tools' | 'full';
|
|
2707
2864
|
/**
|
|
@@ -2722,16 +2879,16 @@ type ResolvedQuestionStateOptions = {
|
|
|
2722
2879
|
turns: 'last' | 'all';
|
|
2723
2880
|
checks: boolean | string[];
|
|
2724
2881
|
};
|
|
2725
|
-
/** Defaults for every
|
|
2882
|
+
/** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
|
|
2726
2883
|
type QuestionOptions = {
|
|
2727
2884
|
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2728
2885
|
model?: string;
|
|
2729
|
-
/** What each
|
|
2886
|
+
/** What it's shown of each attempt. */
|
|
2730
2887
|
state?: QuestionStateOptions;
|
|
2731
|
-
/** Each
|
|
2888
|
+
/** Each attempt's own captures, by name; `true` sends all of them. Default none. */
|
|
2732
2889
|
artifacts?: true | string[];
|
|
2733
2890
|
};
|
|
2734
|
-
/** A matrix factor a
|
|
2891
|
+
/** A matrix factor a pairwise question varies; the others are held. */
|
|
2735
2892
|
type PairFactor = 'environment' | 'harness';
|
|
2736
2893
|
/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
|
|
2737
2894
|
type CompareCut = 'within' | 'across';
|
|
@@ -2760,7 +2917,7 @@ type PairVote = {
|
|
|
2760
2917
|
b: string[];
|
|
2761
2918
|
};
|
|
2762
2919
|
};
|
|
2763
|
-
/** A
|
|
2920
|
+
/** A pairwise question's answer: every vote, and each side's wins over decided votes per cut. */
|
|
2764
2921
|
type CompareAnswer = {
|
|
2765
2922
|
question: string;
|
|
2766
2923
|
/** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
|
|
@@ -2784,6 +2941,28 @@ type CompareAnswer = {
|
|
|
2784
2941
|
/** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
|
|
2785
2942
|
input?: FileRef;
|
|
2786
2943
|
};
|
|
2944
|
+
/** A question asked of one attempt on its own, and the model's answer. Never gates. */
|
|
2945
|
+
type AttemptAnswer = {
|
|
2946
|
+
question: string;
|
|
2947
|
+
/** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
|
|
2948
|
+
model: string;
|
|
2949
|
+
/** The model id the provider says served the call, when it reports one. */
|
|
2950
|
+
resolvedModel?: string;
|
|
2951
|
+
/** What the attempt showed, every key resolved. */
|
|
2952
|
+
stateOptions: ResolvedQuestionStateOptions;
|
|
2953
|
+
/** The captures it sent, as asked: `true` for all, or their names. */
|
|
2954
|
+
artifacts?: true | string[];
|
|
2955
|
+
/** The answers it picked from; unset on a free-text question. */
|
|
2956
|
+
choices?: string[];
|
|
2957
|
+
/** One of `choices`; unset on a free-text question and when the call failed. */
|
|
2958
|
+
choice?: string;
|
|
2959
|
+
/** The model's answer, or its reason for `choice`; capped, and empty when the call failed. */
|
|
2960
|
+
text: string;
|
|
2961
|
+
/** Why there's no answer: the model call failed. Outcomes never change. */
|
|
2962
|
+
error?: string;
|
|
2963
|
+
/** `questions/<n>.json` under the attempt's directory: what the model was sent. */
|
|
2964
|
+
input?: FileRef;
|
|
2965
|
+
};
|
|
2787
2966
|
/** An option a profile's `limit` can narrow, and `pin` can protect. */
|
|
2788
2967
|
type PinnableKey = 'harness' | 'runs' | 'compare';
|
|
2789
2968
|
/**
|
|
@@ -2822,8 +3001,15 @@ type EvalOptionFields = {
|
|
|
2822
3001
|
judge?: JudgeOptions;
|
|
2823
3002
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
2824
3003
|
capture?: CaptureOptions;
|
|
2825
|
-
/** Defaults for every
|
|
3004
|
+
/** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
|
|
2826
3005
|
question?: QuestionOptions;
|
|
3006
|
+
/**
|
|
3007
|
+
* What a language model is asked about the attempts, always a list; they never change pass,
|
|
3008
|
+
* fail or the exit code. A question with `across` picks between pairs of attempts once they all
|
|
3009
|
+
* end; one without is asked of each attempt as it ends. Replaces the inherited list, so `[]`
|
|
3010
|
+
* asks none.
|
|
3011
|
+
*/
|
|
3012
|
+
questions?: Question[];
|
|
2827
3013
|
/** Per attempt. */
|
|
2828
3014
|
timeout?: Duration;
|
|
2829
3015
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
@@ -2911,7 +3097,7 @@ type ProfileDefaults = {
|
|
|
2911
3097
|
timeout?: Duration;
|
|
2912
3098
|
/** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
|
|
2913
3099
|
capture?: CaptureOptions;
|
|
2914
|
-
/** Defaults for every
|
|
3100
|
+
/** Defaults for every question, merged over the config's. */
|
|
2915
3101
|
question?: QuestionOptions;
|
|
2916
3102
|
};
|
|
2917
3103
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
@@ -2939,6 +3125,8 @@ type Profile = {
|
|
|
2939
3125
|
record?: RecordMode;
|
|
2940
3126
|
/** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
|
|
2941
3127
|
tags?: string[];
|
|
3128
|
+
/** Replaces the config's `gateway` list for this profile's runs. */
|
|
3129
|
+
gateway?: GatewayDescriptor[];
|
|
2942
3130
|
};
|
|
2943
3131
|
/** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
|
|
2944
3132
|
type FakeAnalysisModel = {
|
|
@@ -2946,12 +3134,20 @@ type FakeAnalysisModel = {
|
|
|
2946
3134
|
/** The model's structured answer, or a function of the digest it was given. */
|
|
2947
3135
|
output?: unknown | ((digest: unknown) => unknown);
|
|
2948
3136
|
/**
|
|
2949
|
-
* How it answers
|
|
3137
|
+
* How it answers pairwise questions: a fixed pick, a function of the blinded pair, or
|
|
2950
3138
|
* `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
|
|
2951
3139
|
*/
|
|
2952
3140
|
vote?: FakePick | ((pair: BlindPair) => FakePick) | {
|
|
2953
3141
|
error: string;
|
|
2954
3142
|
};
|
|
3143
|
+
/**
|
|
3144
|
+
* How it answers questions asked of each attempt: a fixed answer, a function of the attempt as
|
|
3145
|
+
* the model sees it, or `{ error }` to fail every call. An answer may be `{ choice?, text? }`.
|
|
3146
|
+
* Default the first of the question's `choices`, with the text `fake answer`.
|
|
3147
|
+
*/
|
|
3148
|
+
answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
|
|
3149
|
+
error: string;
|
|
3150
|
+
};
|
|
2955
3151
|
/**
|
|
2956
3152
|
* What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
|
|
2957
3153
|
* function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
|
|
@@ -3023,18 +3219,30 @@ type FakeInsights = {
|
|
|
3023
3219
|
};
|
|
3024
3220
|
/** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
|
|
3025
3221
|
type FakeVote = 'A' | 'B' | 'tie';
|
|
3026
|
-
/** A fake
|
|
3222
|
+
/** A fake pairwise pick, optionally with a reason. */
|
|
3027
3223
|
type FakePick = FakeVote | {
|
|
3028
3224
|
winner: FakeVote;
|
|
3029
3225
|
reason?: string;
|
|
3030
3226
|
};
|
|
3031
|
-
/**
|
|
3227
|
+
/** A fake per-attempt answer: its text, or a choice and its reason. */
|
|
3228
|
+
type FakeAnswer = string | {
|
|
3229
|
+
choice?: string;
|
|
3230
|
+
text?: string;
|
|
3231
|
+
};
|
|
3232
|
+
/** What a pairwise question's model is shown: the prompt, and the two sides in slot order. */
|
|
3032
3233
|
type BlindPair = {
|
|
3033
3234
|
question: string;
|
|
3034
3235
|
prompt: string;
|
|
3035
3236
|
a: BlindSide;
|
|
3036
3237
|
b: BlindSide;
|
|
3037
3238
|
};
|
|
3239
|
+
/** What a per-attempt question's model is shown: the prompt, its choices, and the attempt. */
|
|
3240
|
+
type BlindAttempt = {
|
|
3241
|
+
question: string;
|
|
3242
|
+
prompt: string;
|
|
3243
|
+
choices?: string[];
|
|
3244
|
+
attempt: BlindSide;
|
|
3245
|
+
};
|
|
3038
3246
|
/** A turn of one side, with the keys its question's `state` turned on. */
|
|
3039
3247
|
type BlindTurn = {
|
|
3040
3248
|
task?: string;
|
|
@@ -3042,9 +3250,9 @@ type BlindTurn = {
|
|
|
3042
3250
|
transcript?: string;
|
|
3043
3251
|
};
|
|
3044
3252
|
/**
|
|
3045
|
-
* One side of a blinded pair
|
|
3046
|
-
* `turns: 'all'`, `task`, `output` and `transcript` are
|
|
3047
|
-
* the pair's prompt's, not either side's.
|
|
3253
|
+
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
|
|
3254
|
+
* its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
|
|
3255
|
+
* under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
|
|
3048
3256
|
*/
|
|
3049
3257
|
type BlindSide = BlindTurn & {
|
|
3050
3258
|
turns?: BlindTurn[];
|
|
@@ -3075,7 +3283,7 @@ type BlindSide = BlindTurn & {
|
|
|
3075
3283
|
type AnalysisConfig = {
|
|
3076
3284
|
/**
|
|
3077
3285
|
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
|
|
3078
|
-
* and
|
|
3286
|
+
* and questions fall back to the judge's when it's a language model.
|
|
3079
3287
|
*/
|
|
3080
3288
|
model?: string | FakeAnalysisModel;
|
|
3081
3289
|
/**
|
|
@@ -3114,7 +3322,7 @@ type ReevalConfig = {
|
|
|
3114
3322
|
gate?: GatePolicy;
|
|
3115
3323
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
3116
3324
|
env?: EnvVars;
|
|
3117
|
-
/** Where the sandbox may connect; default `'
|
|
3325
|
+
/** Where the sandbox may connect; default `'proxy'`. */
|
|
3118
3326
|
egress?: EgressPolicy;
|
|
3119
3327
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3120
3328
|
judge?: ConfigJudgeOptions;
|
|
@@ -3125,7 +3333,7 @@ type ReevalConfig = {
|
|
|
3125
3333
|
* binary when `c.capture()` runs; it does not change this object.
|
|
3126
3334
|
*/
|
|
3127
3335
|
capture?: CaptureOptions;
|
|
3128
|
-
/** Defaults for every
|
|
3336
|
+
/** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
|
|
3129
3337
|
question?: QuestionOptions;
|
|
3130
3338
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
3131
3339
|
concurrency?: number;
|
|
@@ -3154,8 +3362,13 @@ type ReevalConfig = {
|
|
|
3154
3362
|
keep?: number;
|
|
3155
3363
|
traceBufferMb?: number;
|
|
3156
3364
|
};
|
|
3157
|
-
/** Insights during `reeval run`, `reeval analyze`, and
|
|
3365
|
+
/** Insights during `reeval run`, `reeval analyze`, and questions without a model. */
|
|
3158
3366
|
analysis?: AnalysisConfig;
|
|
3367
|
+
/**
|
|
3368
|
+
* The gateways model calls may go through, in the order you prefer them: each harness takes
|
|
3369
|
+
* the first that can serve it. Default `[direct()]`, each provider's own key.
|
|
3370
|
+
*/
|
|
3371
|
+
gateway?: GatewayDescriptor[];
|
|
3159
3372
|
};
|
|
3160
3373
|
/** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
|
|
3161
3374
|
type ConfigHelpers = EnvironmentHelpers & {
|
|
@@ -3312,7 +3525,7 @@ type EnvironmentRecord = {
|
|
|
3312
3525
|
brokered: string[];
|
|
3313
3526
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3314
3527
|
unset: string[];
|
|
3315
|
-
/** The policy the attempt ran under, from the options and the environment; in the digest unless `
|
|
3528
|
+
/** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
|
|
3316
3529
|
egress: EgressPolicy;
|
|
3317
3530
|
/**
|
|
3318
3531
|
* The git repository the workspace started as: the repo as written (a local one relative to the
|
|
@@ -3331,15 +3544,22 @@ type EnvironmentRecord = {
|
|
|
3331
3544
|
};
|
|
3332
3545
|
/**
|
|
3333
3546
|
* An attempt's network traffic. `allowed` is every host a restricted policy let through, its own
|
|
3334
|
-
* and the automatic ones; null under `allow-all`. `recorded` is false when the sandbox
|
|
3335
|
-
* the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3336
|
-
* sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3337
|
-
* always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
|
|
3547
|
+
* and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
|
|
3548
|
+
* can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3549
|
+
* the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3550
|
+
* harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
|
|
3551
|
+
* out.
|
|
3338
3552
|
*/
|
|
3339
3553
|
type EgressRecord = {
|
|
3340
3554
|
policy: EgressPolicy;
|
|
3341
3555
|
allowed: string[] | null;
|
|
3342
3556
|
recorded: boolean;
|
|
3557
|
+
/**
|
|
3558
|
+
* True when traffic could leave around the gateway (`allow-all` in a container that kept its
|
|
3559
|
+
* network): `contacted` lists only what went through the gateway, and anything that ignored the
|
|
3560
|
+
* proxy (raw TCP and UDP, SSH, database drivers) went out unrecorded. Unset otherwise.
|
|
3561
|
+
*/
|
|
3562
|
+
partial?: true;
|
|
3343
3563
|
contacted: Array<{
|
|
3344
3564
|
host: string;
|
|
3345
3565
|
port: number;
|
|
@@ -3436,6 +3656,14 @@ type SandboxWarning = {
|
|
|
3436
3656
|
/** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
|
|
3437
3657
|
message: string;
|
|
3438
3658
|
};
|
|
3659
|
+
/** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
|
|
3660
|
+
type ResolvedGateway = {
|
|
3661
|
+
name: GatewayName;
|
|
3662
|
+
skipped: Array<{
|
|
3663
|
+
gateway: GatewayName;
|
|
3664
|
+
why: string;
|
|
3665
|
+
}>;
|
|
3666
|
+
};
|
|
3439
3667
|
//#endregion
|
|
3440
|
-
export {
|
|
3441
|
-
//# sourceMappingURL=types-
|
|
3668
|
+
export { FakeJudgeModel as $, RunRecord as $n, Verdict as $t, CursorHarnessDescriptor as A, SuiteTally as An, PluginPart as At, EnvironmentHelpers as B, suiteResult as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, Interval as Cn, effortVerdict as Cr, McpServer as Ct, CompareOptions as D, ResultFile as Dn, tokenValues as Dr, PairVote as Dt, CompareCut as E, QuestionResult as En, formatSeconds as Er, PairFactor as Et, Duration as F, evalAcross as Fn, Question as Ft, EvalOptionsData as G, FamilyLift as Gn, SetupStep as Gt, EnvironmentSpec as H, AttemptRecord as Hn, Secret as Ht, EgressPolicy as I, factSegment as In, QuestionOptions as It, FakeAnalysisModel as J, FileRef as Jn, TranscriptEvent as Jt, ExecResult as K, FamilyRate as Kn, SkillSource as Kt, EgressRecord as L, isNotGatedResult as Ln, QuestionStateOptions as Lt, DirectGatewayDescriptor as M, checkEffect as Mn, Profile as Mt, DirectoryOptions as N, countedOf as Nn, ProfileDefaults as Nt, ConfigHelpers as O, ResultOutcome as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, effectOf as Pn, ProfileLimit as Pt, FakeInsightsInput as Q, Lift as Qn, VercelGatewayDescriptor as Qt, EnvVars as R, isStudy as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, EvalResult as Sn, effortSummary as Sr, JudgeStateOptions as St, CompareAnswer as T, PooledArm as Tn, formatRatio as Tr, Outcome as Tt, EvalFn as U, CellRecord as Un, SecretTemplate as Ut, EnvironmentInput as V, ArtifactRef as Vn, SandboxWarning as Vt, EvalOptions as W, EvalEntry as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, HistoryEntry as Xn, TurnData as Xt, FakeAnswer as Y, HistoryCell as Yn, Turn as Yt, FakeInsights as Z, JudgementRecord as Zn, TurnStatus as Zt, CaptureUseScript as _, CellResult as _n, directional as _r, JudgeCallOptions as _t, AttemptAnswer as a, Job as an, EffortContrast as ar, GatewayName as at, Cell as b, Effect as bn, effortOf as br, JudgeOptions as bt, BlindAttempt as c, REMARK_KINDS as cn, EffortVerdict as cr, GroupOptions as ct, Capture as d, Sentence as dn, VERDICT_TEXT as dr, HarnessEdit as dt, Viewport as en, StackSummary as er, FakeSandboxDescriptor as et, CaptureCall as f, fromModel as fn, bothUnderATenth as fr, HarnessOption as ft, CaptureUseAction as g, CellError as gn, contrastEffort as gr, JudgeArtifactRef as gt, CaptureUse as h, CaseResult as hn, cellSamples as hr, JSONValue as ht, AssertionRecord as i, INSIGHTS_FILE as in, EFFORT_MIN_RUNS as ir, GatewayDescriptor as it, Diff as j, cancelledOf as jn, PluginSource as jt, ConfigJudgeOptions as k, SuiteResult as kn, PinnableKey as kt, BlindPair as l, Remark as ln, Spread as lr, HarnessData as lt, CaptureOptions as m, AttemptQuestionResult as mn, cellEffort as mr, HostFile as mt, AnalysisConfig as n, WebEventType as nn, DIRECTIONAL_MIN as nr, FakeVote as nt, AttemptContext as o, MODEL_REMARK_KINDS as on, EffortMetric as or, GooseHarnessDescriptor as ot, CaptureInstruction as p, Across as pn, cellContrast as pr, HistoryConfig as pt, ExecRetry as q, FamilyRecord as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, EvalInsight as rn, EFFORT_ALPHA as rr, FxHarnessDescriptor as rt, BackgroundHandle as s, NO_MODEL as sn, EffortSample as sr, GrokBuildHarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, CellEffort as tr, FakeScriptStep as tt, Budget as u, RunInsights as un, TOKENS_UNREPORTED as ur, HarnessDescriptor as ut, CaptureWait as v, CheckOutcome as vn, effortChangeLabel as vr, JudgeFailure as vt, CodexHarnessDescriptor as w, JudgeResult as wn, formatCount as wr, OpenCodeHarnessDescriptor as wt, CellRole as x, EffortCounters as xn, effortSample as xr, JudgeState as xt, CaptureWindow as y, CheckResult as yn, effortLabel as yr, JudgeModel as yt, Environment as z, messageLine as zn, ResolvedGateway as zt };
|
|
3669
|
+
//# sourceMappingURL=types-DXBbP9Mk.d.ts.map
|