reeval 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -259
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +41 -30
- package/dist/{adapter-DigeVFWC.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-CatUEo5h.js → agent-m-_t-IuW.js} +54 -26
- package/dist/{agent-CatUEo5h.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-Bxub9LMT.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-Bxub9LMT.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-CF063ms6.js → assemble-BDEpelpi.js} +95 -38
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{attempt-CGPV3BZY.js → attempt-jI0OWxlz.js} +2 -2
- package/dist/{attempt-CGPV3BZY.js.map → attempt-jI0OWxlz.js.map} +1 -1
- package/dist/{capture-BiD7lVaM.js → capture-CVRwRTnb.js} +574 -38
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-B5zQCR2r.js → cli-C_4gaJ50.js} +145 -142
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C94CQNh8.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C94CQNh8.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CaZgCTIw.js → command-BXLUfw4h.js} +14 -14
- package/dist/command-BXLUfw4h.js.map +1 -0
- package/dist/{command-CVtgyqlM.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CVtgyqlM.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-BMHXHqMT.js → command-Bxdy62dW.js} +15 -14
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-DTgBZR0k.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-CjpGBUWr.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BXzIDB3X.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-CcoMLC_y.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-BeqRiWLl.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-CMjp3qqF.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-CMjp3qqF.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-BbTlMmYi.js → docker-BnRlTpZA.js} +3 -3
- package/dist/{docker-BbTlMmYi.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CCXdbREk.js → docker-D01slDF3.js} +29 -28
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-DC69dKxB.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-CqttrZcs.js → doctor-DQ2p0euG.js} +136 -38
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-BBz1Yes9.js → environment-CO0P0soF.js} +31 -11
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-CXQ1o-pd.js → evaluate-BUmsN7yG.js} +68 -29
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-yjZKUexW.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-DtENlf9s.js → facts-DzOUTreB.js} +13 -5
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-B-eUpCZ1.js → files-B-CJeeEh.js} +14 -9
- package/dist/{files-B-eUpCZ1.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-BsQbYz2i.js → files-C_gQNVB-.js} +4 -4
- package/dist/{files-BsQbYz2i.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-COHQLEHE.js → files-kUCrswI6.js} +3 -3
- package/dist/{files-COHQLEHE.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-Cs2TMH_k.js → generate-DZ7vp69F.js} +18 -18
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-DhykwgKF.js → git-Bk8DSwHy.js} +3 -3
- package/dist/{git-DhykwgKF.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-BAjw1fcr.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-kSmWatcO.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-kSmWatcO.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-BmJRuTpm.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-BmJRuTpm.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-Bwc9byQr.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-CQTbP092.js → invocations-DzyKSJBa.js} +3 -3
- package/dist/{invocations-CQTbP092.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-B-mel-Gv.js → model-1ZLr9Dcv.js} +59 -35
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-CWlAqmbU.js → model-CgnOpss4.js} +58 -316
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/model-Ch8kX87u.js +1 -0
- package/dist/{node-D0mtjRAO.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-J9colP0-.js → opencode-providers-klkqgzvb.js} +72 -62
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-D4oluWzH.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-D4oluWzH.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-OkwUcJQ9.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-CUWMnSNb.js → plugins-Dkp_H3pB.js} +5 -5
- package/dist/{plugins-CUWMnSNb.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-B6pY89bF.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-Bjm7_ylE.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-Bjm7_ylE.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DbdgtZKe.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-CQmK3SBn.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-CQmK3SBn.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-B3Y17oCC.js → provenance-CfZpFpAh.js} +3 -3
- package/dist/{provenance-B3Y17oCC.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CXN2RFz-.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-eoniqK6w.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-eoniqK6w.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-CLRZmWBT.js → query-CemclveB.js} +2 -2
- package/dist/{query-CLRZmWBT.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-1XOulevV.js → questions-bAudIc04.js} +309 -47
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{redact-DJ9JCShu.js → redact-CRGdfV2s.js} +4 -2
- package/dist/redact-CRGdfV2s.js.map +1 -0
- package/dist/{remarks-DlJCqWRs.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-DlJCqWRs.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-cE2VWfkJ.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-C4zeccCH.js → resolve-CBoKzQMP.js} +9 -10
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-Drtt0uFl.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-Drtt0uFl.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-BEFazUfy.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-BEFazUfy.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DICDHCtp.js → root-Dlff-zp8.js} +5 -5
- package/dist/root-Dlff-zp8.js.map +1 -0
- package/dist/{rules-BFzQqPIm.js → rules-VhftPOOT.js} +53 -4
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-BeMy92Us.js → run-CwIEK16q.js} +12 -12
- package/dist/run-CwIEK16q.js.map +1 -0
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-CQaJj7n9.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-CQaJj7n9.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-xK0les4f.js → server-TGW4WyAz.js} +26 -27
- package/dist/{server-xK0les4f.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-Q1hEKbCs.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-Q1hEKbCs.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-LekQrr_S.js → src-8zoX_qrb.js} +614 -171
- package/dist/src-8zoX_qrb.js.map +1 -0
- package/dist/{src-C0CeKiLU.js → src-BYLM3hEz.js} +4 -3
- package/dist/src-BYLM3hEz.js.map +1 -0
- package/dist/{src-CFp86GtV.js → src-CHIwOy0o.js} +87 -55
- package/dist/{src-CFp86GtV.js.map → src-CHIwOy0o.js.map} +1 -1
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-C7_1SqDP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-C7_1SqDP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/src-CyaDNG9r.js +2 -0
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-BhmaorOU.js → src-DKPVX2Us.js} +43 -17
- package/dist/{src-BhmaorOU.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DiLMkIr9.js → src-DmUgeTlE.js} +47 -17
- package/dist/src-DmUgeTlE.js.map +1 -0
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-qGoFkuJF.js → src-NSdQ9zI6.js} +88 -23
- package/dist/src-NSdQ9zI6.js.map +1 -0
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-BmZ8QVZ8.js → state-B1t143wG.js} +22 -20
- package/dist/{state-BmZ8QVZ8.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-C8sw2pox.js → suite-WgT7m6Vs.js} +76 -13
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-0oQNGskv.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-GraZtfHN.d.ts → types-DXBbP9Mk.d.ts} +302 -72
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-BdoXUuL_.js → validate-BhwqrVsm.js} +48 -16
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +143 -91
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-BoUoX4_A.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-D0-GErmU.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-D0-GErmU.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-tjZFkjxC.js → workspace-CDI68wtX.js} +45 -11
- package/dist/workspace-CDI68wtX.js.map +1 -0
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +8 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-DigeVFWC.js.map +0 -1
- package/dist/assemble-CF063ms6.js.map +0 -1
- package/dist/assemble-CJoD_Rzr.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-BiD7lVaM.js.map +0 -1
- package/dist/capture-Dp3q7K7Y.js +0 -1
- package/dist/cli-B5zQCR2r.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-BMHXHqMT.js.map +0 -1
- package/dist/command-CaZgCTIw.js.map +0 -1
- package/dist/command-DTgBZR0k.js.map +0 -1
- package/dist/commands-CjpGBUWr.js.map +0 -1
- package/dist/config-Dv5MG7Rp.js +0 -753
- package/dist/config-Dv5MG7Rp.js.map +0 -1
- package/dist/contrasts-CcoMLC_y.js.map +0 -1
- package/dist/db-BeqRiWLl.js.map +0 -1
- package/dist/db-CJDRVQal.js +0 -2
- package/dist/docker-CCXdbREk.js.map +0 -1
- package/dist/doctor-CqttrZcs.js.map +0 -1
- package/dist/egress-BfK_O9ce.js +0 -1241
- package/dist/egress-BfK_O9ce.js.map +0 -1
- package/dist/environment-BBz1Yes9.js.map +0 -1
- package/dist/evaluate-CXQ1o-pd.js.map +0 -1
- package/dist/facts-DtENlf9s.js.map +0 -1
- package/dist/files-DqCBW7i8.js +0 -1
- package/dist/files-Ha-jsQXf.js +0 -1
- package/dist/generate-Cs2TMH_k.js.map +0 -1
- package/dist/harness-BAjw1fcr.js.map +0 -1
- package/dist/init-Bwc9byQr.js.map +0 -1
- package/dist/init-D7Y1ny1z.js +0 -1
- package/dist/judge-Dt6KW6_O.js +0 -153
- package/dist/judge-Dt6KW6_O.js.map +0 -1
- package/dist/model-B-mel-Gv.js.map +0 -1
- package/dist/model-BGKYVpoj.js +0 -1
- package/dist/model-C7tSwT7f.js +0 -44
- package/dist/model-C7tSwT7f.js.map +0 -1
- package/dist/model-CWlAqmbU.js.map +0 -1
- package/dist/model-DHY-dfoO.js +0 -1
- package/dist/node-D0mtjRAO.js.map +0 -1
- package/dist/opencode-providers-J9colP0-.js.map +0 -1
- package/dist/options-CPUW97_7.js +0 -1116
- package/dist/options-CPUW97_7.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-OkwUcJQ9.js.map +0 -1
- package/dist/query-CmBDYt0_.js +0 -1
- package/dist/questions-1XOulevV.js.map +0 -1
- package/dist/redact-DJ9JCShu.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-cE2VWfkJ.js.map +0 -1
- package/dist/resolve-C4zeccCH.js.map +0 -1
- package/dist/resolve-D30yZYmB.js +0 -1
- package/dist/root-DICDHCtp.js.map +0 -1
- package/dist/rules-BFzQqPIm.js.map +0 -1
- package/dist/run-BeMy92Us.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-V4-2w0l8.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-BDow_oW8.js +0 -1
- package/dist/src-B9L9yGKI.js +0 -1
- package/dist/src-Bcah6ZkT.js +0 -3
- package/dist/src-C0CeKiLU.js.map +0 -1
- package/dist/src-C0xowZ3C.js +0 -2
- package/dist/src-CHIfIKUu.js +0 -2
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-DLtyz4w5.js +0 -2
- package/dist/src-DPIjmReZ.js +0 -2
- package/dist/src-DXNh5mTr.js +0 -2
- package/dist/src-DiLMkIr9.js.map +0 -1
- package/dist/src-LekQrr_S.js.map +0 -1
- package/dist/src-bCnsA6Vp.js +0 -2
- package/dist/src-qGoFkuJF.js.map +0 -1
- package/dist/src-ynBVosfX.js +0 -2
- package/dist/state-BQR_BCm4.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-CnYJijyI.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-BaHPmAGy.js +0 -1
- package/dist/suite-C8sw2pox.js.map +0 -1
- package/dist/summary-0oQNGskv.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-BdoXUuL_.js.map +0 -1
- package/dist/views-BoUoX4_A.js.map +0 -1
- package/dist/views-noVQNCUP.js +0 -2
- package/dist/workspace-B90YDU_a.js +0 -1
- package/dist/workspace-tjZFkjxC.js.map +0 -1
|
@@ -200,6 +200,8 @@ type JudgementRecord = {
|
|
|
200
200
|
model: string;
|
|
201
201
|
/** The model id the provider says served the call, when it reports one. */
|
|
202
202
|
resolvedModel?: string;
|
|
203
|
+
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
204
|
+
gateway?: string;
|
|
203
205
|
/**
|
|
204
206
|
* `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
|
|
205
207
|
* verdict can be followed across runs (and an edited question starts a new series).
|
|
@@ -257,6 +259,12 @@ type AttemptRecord = {
|
|
|
257
259
|
* recorded them.
|
|
258
260
|
*/
|
|
259
261
|
servedModels: string[] | null;
|
|
262
|
+
/**
|
|
263
|
+
* The models the harness's successful API calls asked for, when nothing named the one that
|
|
264
|
+
* served them (fx on Vercel AI Gateway with a Gemini model): the agent's word, not the
|
|
265
|
+
* gateway's, so never part of `servedModels`. Absent when there's none.
|
|
266
|
+
*/
|
|
267
|
+
askedModels?: string[];
|
|
260
268
|
/**
|
|
261
269
|
* The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
|
|
262
270
|
* null when the factory left it unset. Fast is not this.
|
|
@@ -320,11 +328,19 @@ type AttemptRecord = {
|
|
|
320
328
|
judgements: JudgementRecord[];
|
|
321
329
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
322
330
|
startRetries?: StartRetry[];
|
|
331
|
+
/**
|
|
332
|
+
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
333
|
+
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
334
|
+
*/
|
|
323
335
|
checks: Array<{
|
|
324
336
|
command: string;
|
|
325
337
|
exitCode: number;
|
|
326
338
|
durationMs: number;
|
|
327
339
|
log?: FileRef;
|
|
340
|
+
/** The `retries` it was given, when more than 0: it could run `retries + 1` times. */
|
|
341
|
+
retries?: number;
|
|
342
|
+
/** Every try, in order, when it was given `retries`; its log has each try's output. */
|
|
343
|
+
tries?: ExecTry[];
|
|
328
344
|
}>;
|
|
329
345
|
/** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
|
|
330
346
|
assertions?: AssertionRecord[];
|
|
@@ -333,6 +349,13 @@ type AttemptRecord = {
|
|
|
333
349
|
count?: number;
|
|
334
350
|
}>;
|
|
335
351
|
files: AttemptFiles;
|
|
352
|
+
/**
|
|
353
|
+
* The eval's questions asked of this attempt on its own (those without `across`), in question
|
|
354
|
+
* order; unset without any. They annotate: `outcome` never depends on them. Asked once the
|
|
355
|
+
* attempt ends, when it ran to a pass or a fail. Each answer's `input` is `questions/<n>.json`
|
|
356
|
+
* in the attempt's directory, `<n>` the question's place in the eval's `questions`.
|
|
357
|
+
*/
|
|
358
|
+
answers?: AttemptAnswer[];
|
|
336
359
|
};
|
|
337
360
|
type EvalEntry = {
|
|
338
361
|
/** `file › group › … › name`, as shown. */
|
|
@@ -367,8 +390,8 @@ type EvalEntry = {
|
|
|
367
390
|
cells: CellRecord[];
|
|
368
391
|
attempts: AttemptRecord[];
|
|
369
392
|
/**
|
|
370
|
-
* The
|
|
371
|
-
* unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
393
|
+
* The eval's pairwise questions (those with `across`), answered after its attempts by votes,
|
|
394
|
+
* one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
372
395
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
373
396
|
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
374
397
|
* `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
|
|
@@ -523,6 +546,8 @@ type FingerprintParts = {
|
|
|
523
546
|
judge: string;
|
|
524
547
|
/** Over the attempt's context files. */
|
|
525
548
|
context: string;
|
|
549
|
+
/** The gateway its model calls went through; unset on `direct`, each provider's own key. */
|
|
550
|
+
gateway?: string;
|
|
526
551
|
};
|
|
527
552
|
/** One harness and environment of an eval, and how its attempts went. */
|
|
528
553
|
type CellRecord = Cell & {
|
|
@@ -532,6 +557,11 @@ type CellRecord = Cell & {
|
|
|
532
557
|
/** The environment's digest: the join key into `RunRecord.environments`; null for none. */
|
|
533
558
|
environmentDigest: string | null;
|
|
534
559
|
harnessKind: string;
|
|
560
|
+
/**
|
|
561
|
+
* The gateway in the config's `gateway` list its harness resolved to (`direct` is each
|
|
562
|
+
* provider's own key); unset when no config, profile or pin named one.
|
|
563
|
+
*/
|
|
564
|
+
gateway?: string;
|
|
535
565
|
/** The configured harness model, when it has one. */
|
|
536
566
|
model: string | null;
|
|
537
567
|
/**
|
|
@@ -661,6 +691,10 @@ type RunRecord = {
|
|
|
661
691
|
attempts?: number;
|
|
662
692
|
duration?: string | number;
|
|
663
693
|
} | null;
|
|
694
|
+
/**
|
|
695
|
+
* The run's settings. `sandbox` is the id of each sandbox the planned attempts ran on, after the
|
|
696
|
+
* profile's defaults, joined with `, `.
|
|
697
|
+
*/
|
|
664
698
|
config: {
|
|
665
699
|
sandbox: string;
|
|
666
700
|
environment: string | null;
|
|
@@ -848,8 +882,6 @@ type Effect = 'better' | 'worse' | 'same' | 'unclear';
|
|
|
848
882
|
* `harness` when it has several harnesses, `both`, or `none`.
|
|
849
883
|
*/
|
|
850
884
|
type Across = 'none' | 'change' | 'harness' | 'both';
|
|
851
|
-
/** The fewest counted attempts a side for an effect to be anything but `unclear`. */
|
|
852
|
-
declare const EFFECT_MIN_COUNTED = 4;
|
|
853
885
|
/**
|
|
854
886
|
* One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
|
|
855
887
|
* lists them all in the order the nodes appear.
|
|
@@ -883,7 +915,8 @@ type SuiteResult = {
|
|
|
883
915
|
errors: string[];
|
|
884
916
|
/**
|
|
885
917
|
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
886
|
-
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors
|
|
918
|
+
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
|
|
919
|
+
* `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
|
|
887
920
|
*/
|
|
888
921
|
tally: SuiteTally;
|
|
889
922
|
/** Sorted by path. */
|
|
@@ -894,6 +927,7 @@ type SuiteResult = {
|
|
|
894
927
|
type SuiteTally = {
|
|
895
928
|
errors: number;
|
|
896
929
|
pass: number;
|
|
930
|
+
notGated: number;
|
|
897
931
|
fail: number;
|
|
898
932
|
infraError: number;
|
|
899
933
|
cancelled: number;
|
|
@@ -934,13 +968,14 @@ type EvalResult = {
|
|
|
934
968
|
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
935
969
|
pooled: PooledArm[] | null;
|
|
936
970
|
/**
|
|
937
|
-
* Its
|
|
938
|
-
*
|
|
971
|
+
* Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
|
|
972
|
+
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
939
973
|
*/
|
|
940
|
-
questions: QuestionResult
|
|
974
|
+
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
941
975
|
};
|
|
942
|
-
/** A
|
|
976
|
+
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
943
977
|
type QuestionResult = {
|
|
978
|
+
kind: 'pairs';
|
|
944
979
|
question: string;
|
|
945
980
|
/** Pairs voted, each counted once per order it was asked in. */
|
|
946
981
|
votes: number;
|
|
@@ -953,6 +988,27 @@ type QuestionResult = {
|
|
|
953
988
|
/** Why some or all of its votes are missing: the model call failed. */
|
|
954
989
|
error: string | null;
|
|
955
990
|
};
|
|
991
|
+
/** A question asked of each attempt on its own: what each one answered. Never gates. */
|
|
992
|
+
type AttemptQuestionResult = {
|
|
993
|
+
kind: 'attempts';
|
|
994
|
+
question: string;
|
|
995
|
+
/** Attempts it was asked of, failed calls included. */
|
|
996
|
+
asked: number;
|
|
997
|
+
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
998
|
+
choices: Array<{
|
|
999
|
+
choice: string;
|
|
1000
|
+
n: number;
|
|
1001
|
+
}> | null;
|
|
1002
|
+
/** Each attempt's answer, in attempt order; failed calls left out. */
|
|
1003
|
+
answers: Array<{
|
|
1004
|
+
attempt: string;
|
|
1005
|
+
choice: string | null;
|
|
1006
|
+
text: string;
|
|
1007
|
+
}>;
|
|
1008
|
+
/** Calls that failed, and the first one's error. */
|
|
1009
|
+
failed: number;
|
|
1010
|
+
error: string | null;
|
|
1011
|
+
};
|
|
956
1012
|
type CaseResult = {
|
|
957
1013
|
path: string;
|
|
958
1014
|
/** The case's own eval id. */
|
|
@@ -1031,6 +1087,12 @@ type CheckResult = {
|
|
|
1031
1087
|
effect: Effect | null;
|
|
1032
1088
|
/** Set when the check is a judge verdict. */
|
|
1033
1089
|
judge: JudgeResult | null;
|
|
1090
|
+
/**
|
|
1091
|
+
* A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
|
|
1092
|
+
* it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
|
|
1093
|
+
* when none did.
|
|
1094
|
+
*/
|
|
1095
|
+
retried?: string;
|
|
1034
1096
|
};
|
|
1035
1097
|
/**
|
|
1036
1098
|
* `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
|
|
@@ -1105,6 +1167,13 @@ declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'enviro
|
|
|
1105
1167
|
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1106
1168
|
*/
|
|
1107
1169
|
declare function messageLine(text: string): string;
|
|
1170
|
+
/** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
|
|
1171
|
+
declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
|
|
1172
|
+
/**
|
|
1173
|
+
* A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
|
|
1174
|
+
* gating it, so it reads `not gated`, never `pass`.
|
|
1175
|
+
*/
|
|
1176
|
+
declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
|
|
1108
1177
|
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1109
1178
|
declare function suiteResult(record: RunRecord): SuiteResult;
|
|
1110
1179
|
//#endregion
|
|
@@ -1258,7 +1327,7 @@ declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
|
1258
1327
|
severity: 'concern' | 'suggestion';
|
|
1259
1328
|
}>;
|
|
1260
1329
|
/** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
|
|
1261
|
-
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "slow-mcp-start" | "too-few-tries")[];
|
|
1330
|
+
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "proxy-only-egress" | "slow-mcp-start" | "too-few-tries")[];
|
|
1262
1331
|
/** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
|
|
1263
1332
|
type Remark = {
|
|
1264
1333
|
/**
|
|
@@ -1268,7 +1337,7 @@ type Remark = {
|
|
|
1268
1337
|
id: string;
|
|
1269
1338
|
evalId: string;
|
|
1270
1339
|
/** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
|
|
1271
|
-
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start';
|
|
1340
|
+
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start' | 'proxy-only-egress';
|
|
1272
1341
|
/** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
|
|
1273
1342
|
severity: 'concern' | 'suggestion';
|
|
1274
1343
|
/** One plain sentence: what's wrong. */
|
|
@@ -1494,6 +1563,18 @@ type ExecResult = {
|
|
|
1494
1563
|
stdout: string;
|
|
1495
1564
|
stderr: string;
|
|
1496
1565
|
};
|
|
1566
|
+
/** How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient. */
|
|
1567
|
+
type ExecRetry = {
|
|
1568
|
+
/** Extra tries after a non-zero exit, a whole number from 0. Default 0. */
|
|
1569
|
+
retries?: number;
|
|
1570
|
+
/** Milliseconds to wait before each extra try. Default 1000. */
|
|
1571
|
+
interval?: number;
|
|
1572
|
+
};
|
|
1573
|
+
/** One run of a `c.exec` command that had `retries`. */
|
|
1574
|
+
type ExecTry = {
|
|
1575
|
+
exitCode: number;
|
|
1576
|
+
durationMs: number;
|
|
1577
|
+
};
|
|
1497
1578
|
/** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
|
|
1498
1579
|
type CaptureWindow = {
|
|
1499
1580
|
/** In CSS pixels. */
|
|
@@ -1567,8 +1648,8 @@ type CaptureOptions = {
|
|
|
1567
1648
|
/** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
|
|
1568
1649
|
browser?: boolean;
|
|
1569
1650
|
/**
|
|
1570
|
-
* Vision model for `instructions`. Cascades like `question`. Default is
|
|
1571
|
-
*
|
|
1651
|
+
* Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else
|
|
1652
|
+
* the attempt's judge model, whichever is first a language model; never Jev.
|
|
1572
1653
|
*/
|
|
1573
1654
|
model?: string;
|
|
1574
1655
|
};
|
|
@@ -1663,7 +1744,7 @@ type TurnData = {
|
|
|
1663
1744
|
};
|
|
1664
1745
|
type Turn = TurnData & {
|
|
1665
1746
|
/** Runs in the eval's sandbox. Same as `c.exec`. */
|
|
1666
|
-
exec(command: string): Promise<ExecResult>;
|
|
1747
|
+
exec(command: string, options?: ExecRetry): Promise<ExecResult>;
|
|
1667
1748
|
readFile(path: string): Promise<string | null>;
|
|
1668
1749
|
};
|
|
1669
1750
|
/** Any JSON value. */
|
|
@@ -1728,6 +1809,11 @@ type JudgeOptions = {
|
|
|
1728
1809
|
* attempt's state. Merged key by key down the cascade.
|
|
1729
1810
|
*/
|
|
1730
1811
|
state?: JudgeStateOptions;
|
|
1812
|
+
/**
|
|
1813
|
+
* The gateway the judge's calls go through, by name or by its helper. It skips the config's
|
|
1814
|
+
* `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
|
|
1815
|
+
*/
|
|
1816
|
+
gateway?: GatewayName | GatewayDescriptor;
|
|
1731
1817
|
};
|
|
1732
1818
|
/**
|
|
1733
1819
|
* What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
|
|
@@ -1796,6 +1882,8 @@ type Verdict = {
|
|
|
1796
1882
|
model: string;
|
|
1797
1883
|
/** The model id the provider says served the call, when it reports one. */
|
|
1798
1884
|
resolvedModel?: string;
|
|
1885
|
+
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
1886
|
+
gateway?: GatewayName;
|
|
1799
1887
|
/** Captures this call sent the judge as file parts, when any. */
|
|
1800
1888
|
artifacts?: JudgeArtifactRef[];
|
|
1801
1889
|
};
|
|
@@ -1820,11 +1908,17 @@ type AttemptContext = {
|
|
|
1820
1908
|
* `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
|
|
1821
1909
|
*/
|
|
1822
1910
|
judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1823
|
-
|
|
1911
|
+
/**
|
|
1912
|
+
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1913
|
+
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
1914
|
+
*/
|
|
1915
|
+
exec(command: string, options?: ExecRetry & {
|
|
1916
|
+
background?: false;
|
|
1917
|
+
}): Promise<ExecResult>;
|
|
1824
1918
|
exec(command: string, options: {
|
|
1825
1919
|
background: true;
|
|
1826
1920
|
}): Promise<BackgroundHandle>;
|
|
1827
|
-
exec(command: string, options?: {
|
|
1921
|
+
exec(command: string, options?: ExecRetry & {
|
|
1828
1922
|
background?: boolean;
|
|
1829
1923
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1830
1924
|
/**
|
|
@@ -1861,6 +1955,11 @@ type EvalFn = (c: AttemptContext) => Promise<void>;
|
|
|
1861
1955
|
/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
|
|
1862
1956
|
type FakeHarnessDescriptor = {
|
|
1863
1957
|
kind: 'fake';
|
|
1958
|
+
/**
|
|
1959
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1960
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1961
|
+
*/
|
|
1962
|
+
gateway?: GatewayName;
|
|
1864
1963
|
/** Cell id; defaults to `fake`. */
|
|
1865
1964
|
id?: string;
|
|
1866
1965
|
/** Files the fake agent writes into the sandbox on every turn. */
|
|
@@ -1947,6 +2046,11 @@ type FakeScriptStep = {
|
|
|
1947
2046
|
*/
|
|
1948
2047
|
type CursorHarnessDescriptor = {
|
|
1949
2048
|
kind: 'cursor';
|
|
2049
|
+
/**
|
|
2050
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2051
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2052
|
+
*/
|
|
2053
|
+
gateway?: GatewayName;
|
|
1950
2054
|
/** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
|
|
1951
2055
|
model: string;
|
|
1952
2056
|
/**
|
|
@@ -1974,6 +2078,11 @@ type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
|
1974
2078
|
/** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
|
|
1975
2079
|
type ClaudeCodeHarnessDescriptor = {
|
|
1976
2080
|
kind: 'claude-code';
|
|
2081
|
+
/**
|
|
2082
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2083
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2084
|
+
*/
|
|
2085
|
+
gateway?: GatewayName;
|
|
1977
2086
|
/** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
|
|
1978
2087
|
model: string;
|
|
1979
2088
|
/**
|
|
@@ -1994,6 +2103,11 @@ type ClaudeCodeHarnessDescriptor = {
|
|
|
1994
2103
|
/** The Antigravity CLI through `@reeval/harness-antigravity`, made by `antigravity()`. */
|
|
1995
2104
|
type AntigravityHarnessDescriptor = {
|
|
1996
2105
|
kind: 'antigravity';
|
|
2106
|
+
/**
|
|
2107
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2108
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2109
|
+
*/
|
|
2110
|
+
gateway?: GatewayName;
|
|
1997
2111
|
/** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
|
|
1998
2112
|
model: string;
|
|
1999
2113
|
/**
|
|
@@ -2018,6 +2132,11 @@ type AntigravityHarnessDescriptor = {
|
|
|
2018
2132
|
*/
|
|
2019
2133
|
type OpenCodeHarnessDescriptor = {
|
|
2020
2134
|
kind: 'opencode';
|
|
2135
|
+
/**
|
|
2136
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2137
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2138
|
+
*/
|
|
2139
|
+
gateway?: GatewayName;
|
|
2021
2140
|
/**
|
|
2022
2141
|
* OpenCode's model id, `<provider>/<model>`, e.g. `google/gemini-3.8-flash` or
|
|
2023
2142
|
* `anthropic/claude-haiku-4-5`; the provider picks the key.
|
|
@@ -2044,6 +2163,11 @@ type OpenCodeHarnessDescriptor = {
|
|
|
2044
2163
|
*/
|
|
2045
2164
|
type GooseHarnessDescriptor = {
|
|
2046
2165
|
kind: 'goose';
|
|
2166
|
+
/**
|
|
2167
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2168
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2169
|
+
*/
|
|
2170
|
+
gateway?: GatewayName;
|
|
2047
2171
|
/** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
|
|
2048
2172
|
model: string;
|
|
2049
2173
|
/**
|
|
@@ -2067,6 +2191,11 @@ type GooseHarnessDescriptor = {
|
|
|
2067
2191
|
*/
|
|
2068
2192
|
type PiHarnessDescriptor = {
|
|
2069
2193
|
kind: 'pi';
|
|
2194
|
+
/**
|
|
2195
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2196
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2197
|
+
*/
|
|
2198
|
+
gateway?: GatewayName;
|
|
2070
2199
|
/** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
|
|
2071
2200
|
model: string;
|
|
2072
2201
|
/**
|
|
@@ -2090,6 +2219,11 @@ type PiHarnessDescriptor = {
|
|
|
2090
2219
|
*/
|
|
2091
2220
|
type CodexHarnessDescriptor = {
|
|
2092
2221
|
kind: 'codex';
|
|
2222
|
+
/**
|
|
2223
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2224
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2225
|
+
*/
|
|
2226
|
+
gateway?: GatewayName;
|
|
2093
2227
|
/** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
|
|
2094
2228
|
model: string;
|
|
2095
2229
|
/**
|
|
@@ -2113,10 +2247,15 @@ type CodexHarnessDescriptor = {
|
|
|
2113
2247
|
*/
|
|
2114
2248
|
type FxHarnessDescriptor = {
|
|
2115
2249
|
kind: 'fx';
|
|
2250
|
+
/**
|
|
2251
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2252
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2253
|
+
*/
|
|
2254
|
+
gateway?: GatewayName;
|
|
2116
2255
|
/**
|
|
2117
2256
|
* A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
|
|
2118
2257
|
* `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
|
|
2119
|
-
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-
|
|
2258
|
+
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`.
|
|
2120
2259
|
*/
|
|
2121
2260
|
model: string;
|
|
2122
2261
|
/** Cell id; defaults to `fx:<model>`. */
|
|
@@ -2135,6 +2274,11 @@ type FxHarnessDescriptor = {
|
|
|
2135
2274
|
*/
|
|
2136
2275
|
type GrokBuildHarnessDescriptor = {
|
|
2137
2276
|
kind: 'grok-build';
|
|
2277
|
+
/**
|
|
2278
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
2279
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2280
|
+
*/
|
|
2281
|
+
gateway?: GatewayName;
|
|
2138
2282
|
/** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
|
|
2139
2283
|
model: string;
|
|
2140
2284
|
/**
|
|
@@ -2152,6 +2296,24 @@ type GrokBuildHarnessDescriptor = {
|
|
|
2152
2296
|
*/
|
|
2153
2297
|
startTimeout?: Duration;
|
|
2154
2298
|
};
|
|
2299
|
+
/**
|
|
2300
|
+
* A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
|
|
2301
|
+
* `vercel` is Vercel AI Gateway.
|
|
2302
|
+
*/
|
|
2303
|
+
type GatewayName = 'direct' | 'vercel';
|
|
2304
|
+
/** Each provider's own key, as without `gateway`: made by `direct()` in `reeval/gateway`. */
|
|
2305
|
+
type DirectGatewayDescriptor = {
|
|
2306
|
+
kind: 'direct';
|
|
2307
|
+
};
|
|
2308
|
+
/**
|
|
2309
|
+
* Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
|
|
2310
|
+
* `reeval/gateway`. It serves Claude Code, Codex, Goose, pi, OpenCode, fx and Grok Build.
|
|
2311
|
+
*/
|
|
2312
|
+
type VercelGatewayDescriptor = {
|
|
2313
|
+
kind: 'vercel';
|
|
2314
|
+
};
|
|
2315
|
+
/** One entry of the config's `gateway` list. */
|
|
2316
|
+
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor;
|
|
2155
2317
|
/** A real coding agent, driven through its AI SDK harness adapter. */
|
|
2156
2318
|
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
|
|
2157
2319
|
/** The agent an attempt runs: a real harness, or the fake one. */
|
|
@@ -2357,12 +2519,13 @@ type SetupStep = string | {
|
|
|
2357
2519
|
cwd?: string;
|
|
2358
2520
|
};
|
|
2359
2521
|
/**
|
|
2360
|
-
* Where the sandbox may connect:
|
|
2361
|
-
*
|
|
2362
|
-
*
|
|
2363
|
-
*
|
|
2522
|
+
* Where the sandbox may connect: any host over HTTP(S) through the egress gateway (`'proxy'`, the
|
|
2523
|
+
* default), anywhere including raw TCP and UDP around the gateway (`'allow-all'`), nowhere
|
|
2524
|
+
* (`'deny-all'`), or only the listed hosts (`'*.example.com'` matches subdomains). Under a
|
|
2525
|
+
* restricted policy the harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are
|
|
2526
|
+
* allowed too; any other request gets a refusal, and the attempt goes on.
|
|
2364
2527
|
*/
|
|
2365
|
-
type EgressPolicy = 'allow-all' | 'deny-all' | {
|
|
2528
|
+
type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
|
|
2366
2529
|
allow: readonly string[];
|
|
2367
2530
|
};
|
|
2368
2531
|
/**
|
|
@@ -2649,45 +2812,40 @@ type CompareOptions<E = Environment> = {
|
|
|
2649
2812
|
minLift?: number;
|
|
2650
2813
|
/** Harnesses (ids or descriptors) to compare on; the others run the control only. */
|
|
2651
2814
|
harness?: Array<string | HarnessDescriptor>;
|
|
2652
|
-
/**
|
|
2653
|
-
* Asked of pairs of attempts after the run, always a list. They annotate the compare with
|
|
2654
|
-
* win rates and never change pass, fail or the exit code.
|
|
2655
|
-
*/
|
|
2656
|
-
questions?: CompareQuestion[];
|
|
2657
|
-
/**
|
|
2658
|
-
* The factor every question varies when pairing, when a question does not name its own
|
|
2659
|
-
* `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
|
|
2660
|
-
* both, or omitted, asks both pair kinds.
|
|
2661
|
-
*/
|
|
2662
|
-
across?: PairFactor | PairFactor[];
|
|
2663
2815
|
};
|
|
2664
2816
|
/**
|
|
2665
|
-
* A question
|
|
2666
|
-
*
|
|
2667
|
-
*
|
|
2668
|
-
* `
|
|
2669
|
-
* `
|
|
2817
|
+
* A question a language model answers after the attempts: its text, or the text and its options.
|
|
2818
|
+
* With `across`, it picks between two attempts that differ only in that factor, in both orders;
|
|
2819
|
+
* without it, it's asked of each attempt on its own. Without a model, the cascaded
|
|
2820
|
+
* `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
|
|
2821
|
+
* `state` and `artifacts` choose what it's shown of each attempt.
|
|
2670
2822
|
*/
|
|
2671
|
-
type
|
|
2823
|
+
type Question = string | {
|
|
2672
2824
|
question: string;
|
|
2673
2825
|
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
2674
2826
|
model?: string;
|
|
2675
|
-
/**
|
|
2676
|
-
|
|
2677
|
-
|
|
2827
|
+
/**
|
|
2828
|
+
* The factor it varies when pairing, or both: `'environment'` holds the harness,
|
|
2829
|
+
* `'harness'` holds the environment. Unset, it's asked of each attempt on its own.
|
|
2830
|
+
*/
|
|
2831
|
+
across?: PairFactor | PairFactor[];
|
|
2832
|
+
/** The answers a question asked of each attempt picks from, with a short reason. */
|
|
2833
|
+
choices?: string[];
|
|
2834
|
+
/** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
|
|
2678
2835
|
state?: QuestionStateOptions;
|
|
2679
|
-
/** Each
|
|
2836
|
+
/** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
|
|
2680
2837
|
artifacts?: true | string[];
|
|
2681
2838
|
};
|
|
2682
2839
|
/**
|
|
2683
|
-
* What
|
|
2684
|
-
* attempts' workspaces are gone when questions are asked). An unset key keeps its default
|
|
2685
|
-
*
|
|
2840
|
+
* What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
|
|
2841
|
+
* attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
|
|
2842
|
+
* pairwise question shows the task and the diff; one asked of each attempt also shows the final
|
|
2843
|
+
* message and the transcript.
|
|
2686
2844
|
*/
|
|
2687
2845
|
type QuestionStateOptions = {
|
|
2688
2846
|
/** The prompts the attempt was sent, once for both sides when they match. Default true. */
|
|
2689
2847
|
task?: boolean;
|
|
2690
|
-
/** The final message: the agent's own account of its work. Default false. */
|
|
2848
|
+
/** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
|
|
2691
2849
|
output?: boolean;
|
|
2692
2850
|
/** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
|
|
2693
2851
|
files?: boolean;
|
|
@@ -2699,7 +2857,8 @@ type QuestionStateOptions = {
|
|
|
2699
2857
|
/**
|
|
2700
2858
|
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
2701
2859
|
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2702
|
-
* harness's tools, so a pair of two harnesses isn't blind. Default false
|
|
2860
|
+
* harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
|
|
2861
|
+
* attempt.
|
|
2703
2862
|
*/
|
|
2704
2863
|
transcript?: boolean | 'tools' | 'full';
|
|
2705
2864
|
/**
|
|
@@ -2720,16 +2879,16 @@ type ResolvedQuestionStateOptions = {
|
|
|
2720
2879
|
turns: 'last' | 'all';
|
|
2721
2880
|
checks: boolean | string[];
|
|
2722
2881
|
};
|
|
2723
|
-
/** Defaults for every
|
|
2882
|
+
/** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
|
|
2724
2883
|
type QuestionOptions = {
|
|
2725
2884
|
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2726
2885
|
model?: string;
|
|
2727
|
-
/** What each
|
|
2886
|
+
/** What it's shown of each attempt. */
|
|
2728
2887
|
state?: QuestionStateOptions;
|
|
2729
|
-
/** Each
|
|
2888
|
+
/** Each attempt's own captures, by name; `true` sends all of them. Default none. */
|
|
2730
2889
|
artifacts?: true | string[];
|
|
2731
2890
|
};
|
|
2732
|
-
/** A matrix factor a
|
|
2891
|
+
/** A matrix factor a pairwise question varies; the others are held. */
|
|
2733
2892
|
type PairFactor = 'environment' | 'harness';
|
|
2734
2893
|
/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
|
|
2735
2894
|
type CompareCut = 'within' | 'across';
|
|
@@ -2758,7 +2917,7 @@ type PairVote = {
|
|
|
2758
2917
|
b: string[];
|
|
2759
2918
|
};
|
|
2760
2919
|
};
|
|
2761
|
-
/** A
|
|
2920
|
+
/** A pairwise question's answer: every vote, and each side's wins over decided votes per cut. */
|
|
2762
2921
|
type CompareAnswer = {
|
|
2763
2922
|
question: string;
|
|
2764
2923
|
/** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
|
|
@@ -2782,6 +2941,28 @@ type CompareAnswer = {
|
|
|
2782
2941
|
/** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
|
|
2783
2942
|
input?: FileRef;
|
|
2784
2943
|
};
|
|
2944
|
+
/** A question asked of one attempt on its own, and the model's answer. Never gates. */
|
|
2945
|
+
type AttemptAnswer = {
|
|
2946
|
+
question: string;
|
|
2947
|
+
/** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
|
|
2948
|
+
model: string;
|
|
2949
|
+
/** The model id the provider says served the call, when it reports one. */
|
|
2950
|
+
resolvedModel?: string;
|
|
2951
|
+
/** What the attempt showed, every key resolved. */
|
|
2952
|
+
stateOptions: ResolvedQuestionStateOptions;
|
|
2953
|
+
/** The captures it sent, as asked: `true` for all, or their names. */
|
|
2954
|
+
artifacts?: true | string[];
|
|
2955
|
+
/** The answers it picked from; unset on a free-text question. */
|
|
2956
|
+
choices?: string[];
|
|
2957
|
+
/** One of `choices`; unset on a free-text question and when the call failed. */
|
|
2958
|
+
choice?: string;
|
|
2959
|
+
/** The model's answer, or its reason for `choice`; capped, and empty when the call failed. */
|
|
2960
|
+
text: string;
|
|
2961
|
+
/** Why there's no answer: the model call failed. Outcomes never change. */
|
|
2962
|
+
error?: string;
|
|
2963
|
+
/** `questions/<n>.json` under the attempt's directory: what the model was sent. */
|
|
2964
|
+
input?: FileRef;
|
|
2965
|
+
};
|
|
2785
2966
|
/** An option a profile's `limit` can narrow, and `pin` can protect. */
|
|
2786
2967
|
type PinnableKey = 'harness' | 'runs' | 'compare';
|
|
2787
2968
|
/**
|
|
@@ -2820,8 +3001,15 @@ type EvalOptionFields = {
|
|
|
2820
3001
|
judge?: JudgeOptions;
|
|
2821
3002
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
2822
3003
|
capture?: CaptureOptions;
|
|
2823
|
-
/** Defaults for every
|
|
3004
|
+
/** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
|
|
2824
3005
|
question?: QuestionOptions;
|
|
3006
|
+
/**
|
|
3007
|
+
* What a language model is asked about the attempts, always a list; they never change pass,
|
|
3008
|
+
* fail or the exit code. A question with `across` picks between pairs of attempts once they all
|
|
3009
|
+
* end; one without is asked of each attempt as it ends. Replaces the inherited list, so `[]`
|
|
3010
|
+
* asks none.
|
|
3011
|
+
*/
|
|
3012
|
+
questions?: Question[];
|
|
2825
3013
|
/** Per attempt. */
|
|
2826
3014
|
timeout?: Duration;
|
|
2827
3015
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
@@ -2909,7 +3097,7 @@ type ProfileDefaults = {
|
|
|
2909
3097
|
timeout?: Duration;
|
|
2910
3098
|
/** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
|
|
2911
3099
|
capture?: CaptureOptions;
|
|
2912
|
-
/** Defaults for every
|
|
3100
|
+
/** Defaults for every question, merged over the config's. */
|
|
2913
3101
|
question?: QuestionOptions;
|
|
2914
3102
|
};
|
|
2915
3103
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
@@ -2937,6 +3125,8 @@ type Profile = {
|
|
|
2937
3125
|
record?: RecordMode;
|
|
2938
3126
|
/** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
|
|
2939
3127
|
tags?: string[];
|
|
3128
|
+
/** Replaces the config's `gateway` list for this profile's runs. */
|
|
3129
|
+
gateway?: GatewayDescriptor[];
|
|
2940
3130
|
};
|
|
2941
3131
|
/** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
|
|
2942
3132
|
type FakeAnalysisModel = {
|
|
@@ -2944,12 +3134,20 @@ type FakeAnalysisModel = {
|
|
|
2944
3134
|
/** The model's structured answer, or a function of the digest it was given. */
|
|
2945
3135
|
output?: unknown | ((digest: unknown) => unknown);
|
|
2946
3136
|
/**
|
|
2947
|
-
* How it answers
|
|
3137
|
+
* How it answers pairwise questions: a fixed pick, a function of the blinded pair, or
|
|
2948
3138
|
* `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
|
|
2949
3139
|
*/
|
|
2950
3140
|
vote?: FakePick | ((pair: BlindPair) => FakePick) | {
|
|
2951
3141
|
error: string;
|
|
2952
3142
|
};
|
|
3143
|
+
/**
|
|
3144
|
+
* How it answers questions asked of each attempt: a fixed answer, a function of the attempt as
|
|
3145
|
+
* the model sees it, or `{ error }` to fail every call. An answer may be `{ choice?, text? }`.
|
|
3146
|
+
* Default the first of the question's `choices`, with the text `fake answer`.
|
|
3147
|
+
*/
|
|
3148
|
+
answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
|
|
3149
|
+
error: string;
|
|
3150
|
+
};
|
|
2953
3151
|
/**
|
|
2954
3152
|
* What insights get back, in `reeval run` and `reeval analyze --insights`: fixed insights, a
|
|
2955
3153
|
* function of each call's input (`input.call` is `classify` or `insights`, once each per eval,
|
|
@@ -3021,18 +3219,30 @@ type FakeInsights = {
|
|
|
3021
3219
|
};
|
|
3022
3220
|
/** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
|
|
3023
3221
|
type FakeVote = 'A' | 'B' | 'tie';
|
|
3024
|
-
/** A fake
|
|
3222
|
+
/** A fake pairwise pick, optionally with a reason. */
|
|
3025
3223
|
type FakePick = FakeVote | {
|
|
3026
3224
|
winner: FakeVote;
|
|
3027
3225
|
reason?: string;
|
|
3028
3226
|
};
|
|
3029
|
-
/**
|
|
3227
|
+
/** A fake per-attempt answer: its text, or a choice and its reason. */
|
|
3228
|
+
type FakeAnswer = string | {
|
|
3229
|
+
choice?: string;
|
|
3230
|
+
text?: string;
|
|
3231
|
+
};
|
|
3232
|
+
/** What a pairwise question's model is shown: the prompt, and the two sides in slot order. */
|
|
3030
3233
|
type BlindPair = {
|
|
3031
3234
|
question: string;
|
|
3032
3235
|
prompt: string;
|
|
3033
3236
|
a: BlindSide;
|
|
3034
3237
|
b: BlindSide;
|
|
3035
3238
|
};
|
|
3239
|
+
/** What a per-attempt question's model is shown: the prompt, its choices, and the attempt. */
|
|
3240
|
+
type BlindAttempt = {
|
|
3241
|
+
question: string;
|
|
3242
|
+
prompt: string;
|
|
3243
|
+
choices?: string[];
|
|
3244
|
+
attempt: BlindSide;
|
|
3245
|
+
};
|
|
3036
3246
|
/** A turn of one side, with the keys its question's `state` turned on. */
|
|
3037
3247
|
type BlindTurn = {
|
|
3038
3248
|
task?: string;
|
|
@@ -3040,9 +3250,9 @@ type BlindTurn = {
|
|
|
3040
3250
|
transcript?: string;
|
|
3041
3251
|
};
|
|
3042
3252
|
/**
|
|
3043
|
-
* One side of a blinded pair
|
|
3044
|
-
* `turns: 'all'`, `task`, `output` and `transcript` are
|
|
3045
|
-
* the pair's prompt's, not either side's.
|
|
3253
|
+
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
|
|
3254
|
+
* its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
|
|
3255
|
+
* under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
|
|
3046
3256
|
*/
|
|
3047
3257
|
type BlindSide = BlindTurn & {
|
|
3048
3258
|
turns?: BlindTurn[];
|
|
@@ -3073,7 +3283,7 @@ type BlindSide = BlindTurn & {
|
|
|
3073
3283
|
type AnalysisConfig = {
|
|
3074
3284
|
/**
|
|
3075
3285
|
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Insights use it only; `reeval analyze`
|
|
3076
|
-
* and
|
|
3286
|
+
* and questions fall back to the judge's when it's a language model.
|
|
3077
3287
|
*/
|
|
3078
3288
|
model?: string | FakeAnalysisModel;
|
|
3079
3289
|
/**
|
|
@@ -3112,7 +3322,7 @@ type ReevalConfig = {
|
|
|
3112
3322
|
gate?: GatePolicy;
|
|
3113
3323
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
3114
3324
|
env?: EnvVars;
|
|
3115
|
-
/** Where the sandbox may connect; default `'
|
|
3325
|
+
/** Where the sandbox may connect; default `'proxy'`. */
|
|
3116
3326
|
egress?: EgressPolicy;
|
|
3117
3327
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3118
3328
|
judge?: ConfigJudgeOptions;
|
|
@@ -3123,7 +3333,7 @@ type ReevalConfig = {
|
|
|
3123
3333
|
* binary when `c.capture()` runs; it does not change this object.
|
|
3124
3334
|
*/
|
|
3125
3335
|
capture?: CaptureOptions;
|
|
3126
|
-
/** Defaults for every
|
|
3336
|
+
/** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
|
|
3127
3337
|
question?: QuestionOptions;
|
|
3128
3338
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
3129
3339
|
concurrency?: number;
|
|
@@ -3152,8 +3362,13 @@ type ReevalConfig = {
|
|
|
3152
3362
|
keep?: number;
|
|
3153
3363
|
traceBufferMb?: number;
|
|
3154
3364
|
};
|
|
3155
|
-
/** Insights during `reeval run`, `reeval analyze`, and
|
|
3365
|
+
/** Insights during `reeval run`, `reeval analyze`, and questions without a model. */
|
|
3156
3366
|
analysis?: AnalysisConfig;
|
|
3367
|
+
/**
|
|
3368
|
+
* The gateways model calls may go through, in the order you prefer them: each harness takes
|
|
3369
|
+
* the first that can serve it. Default `[direct()]`, each provider's own key.
|
|
3370
|
+
*/
|
|
3371
|
+
gateway?: GatewayDescriptor[];
|
|
3157
3372
|
};
|
|
3158
3373
|
/** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
|
|
3159
3374
|
type ConfigHelpers = EnvironmentHelpers & {
|
|
@@ -3310,7 +3525,7 @@ type EnvironmentRecord = {
|
|
|
3310
3525
|
brokered: string[];
|
|
3311
3526
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3312
3527
|
unset: string[];
|
|
3313
|
-
/** The policy the attempt ran under, from the options and the environment; in the digest unless `
|
|
3528
|
+
/** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
|
|
3314
3529
|
egress: EgressPolicy;
|
|
3315
3530
|
/**
|
|
3316
3531
|
* The git repository the workspace started as: the repo as written (a local one relative to the
|
|
@@ -3329,15 +3544,22 @@ type EnvironmentRecord = {
|
|
|
3329
3544
|
};
|
|
3330
3545
|
/**
|
|
3331
3546
|
* An attempt's network traffic. `allowed` is every host a restricted policy let through, its own
|
|
3332
|
-
* and the automatic ones; null under `allow-all`. `recorded` is false when the sandbox
|
|
3333
|
-
* the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3334
|
-
* sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3335
|
-
* always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
|
|
3547
|
+
* and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
|
|
3548
|
+
* can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3549
|
+
* the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3550
|
+
* harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
|
|
3551
|
+
* out.
|
|
3336
3552
|
*/
|
|
3337
3553
|
type EgressRecord = {
|
|
3338
3554
|
policy: EgressPolicy;
|
|
3339
3555
|
allowed: string[] | null;
|
|
3340
3556
|
recorded: boolean;
|
|
3557
|
+
/**
|
|
3558
|
+
* True when traffic could leave around the gateway (`allow-all` in a container that kept its
|
|
3559
|
+
* network): `contacted` lists only what went through the gateway, and anything that ignored the
|
|
3560
|
+
* proxy (raw TCP and UDP, SSH, database drivers) went out unrecorded. Unset otherwise.
|
|
3561
|
+
*/
|
|
3562
|
+
partial?: true;
|
|
3341
3563
|
contacted: Array<{
|
|
3342
3564
|
host: string;
|
|
3343
3565
|
port: number;
|
|
@@ -3434,6 +3656,14 @@ type SandboxWarning = {
|
|
|
3434
3656
|
/** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
|
|
3435
3657
|
message: string;
|
|
3436
3658
|
};
|
|
3659
|
+
/** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
|
|
3660
|
+
type ResolvedGateway = {
|
|
3661
|
+
name: GatewayName;
|
|
3662
|
+
skipped: Array<{
|
|
3663
|
+
gateway: GatewayName;
|
|
3664
|
+
why: string;
|
|
3665
|
+
}>;
|
|
3666
|
+
};
|
|
3437
3667
|
//#endregion
|
|
3438
|
-
export {
|
|
3439
|
-
//# sourceMappingURL=types-
|
|
3668
|
+
export { FakeJudgeModel as $, RunRecord as $n, Verdict as $t, CursorHarnessDescriptor as A, SuiteTally as An, PluginPart as At, EnvironmentHelpers as B, suiteResult as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, Interval as Cn, effortVerdict as Cr, McpServer as Ct, CompareOptions as D, ResultFile as Dn, tokenValues as Dr, PairVote as Dt, CompareCut as E, QuestionResult as En, formatSeconds as Er, PairFactor as Et, Duration as F, evalAcross as Fn, Question as Ft, EvalOptionsData as G, FamilyLift as Gn, SetupStep as Gt, EnvironmentSpec as H, AttemptRecord as Hn, Secret as Ht, EgressPolicy as I, factSegment as In, QuestionOptions as It, FakeAnalysisModel as J, FileRef as Jn, TranscriptEvent as Jt, ExecResult as K, FamilyRate as Kn, SkillSource as Kt, EgressRecord as L, isNotGatedResult as Ln, QuestionStateOptions as Lt, DirectGatewayDescriptor as M, checkEffect as Mn, Profile as Mt, DirectoryOptions as N, countedOf as Nn, ProfileDefaults as Nt, ConfigHelpers as O, ResultOutcome as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, effectOf as Pn, ProfileLimit as Pt, FakeInsightsInput as Q, Lift as Qn, VercelGatewayDescriptor as Qt, EnvVars as R, isStudy as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, EvalResult as Sn, effortSummary as Sr, JudgeStateOptions as St, CompareAnswer as T, PooledArm as Tn, formatRatio as Tr, Outcome as Tt, EvalFn as U, CellRecord as Un, SecretTemplate as Ut, EnvironmentInput as V, ArtifactRef as Vn, SandboxWarning as Vt, EvalOptions as W, EvalEntry as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, HistoryEntry as Xn, TurnData as Xt, FakeAnswer as Y, HistoryCell as Yn, Turn as Yt, FakeInsights as Z, JudgementRecord as Zn, TurnStatus as Zt, CaptureUseScript as _, CellResult as _n, directional as _r, JudgeCallOptions as _t, AttemptAnswer as a, Job as an, EffortContrast as ar, GatewayName as at, Cell as b, Effect as bn, effortOf as br, JudgeOptions as bt, BlindAttempt as c, REMARK_KINDS as cn, EffortVerdict as cr, GroupOptions as ct, Capture as d, Sentence as dn, VERDICT_TEXT as dr, HarnessEdit as dt, Viewport as en, StackSummary as er, FakeSandboxDescriptor as et, CaptureCall as f, fromModel as fn, bothUnderATenth as fr, HarnessOption as ft, CaptureUseAction as g, CellError as gn, contrastEffort as gr, JudgeArtifactRef as gt, CaptureUse as h, CaseResult as hn, cellSamples as hr, JSONValue as ht, AssertionRecord as i, INSIGHTS_FILE as in, EFFORT_MIN_RUNS as ir, GatewayDescriptor as it, Diff as j, cancelledOf as jn, PluginSource as jt, ConfigJudgeOptions as k, SuiteResult as kn, PinnableKey as kt, BlindPair as l, Remark as ln, Spread as lr, HarnessData as lt, CaptureOptions as m, AttemptQuestionResult as mn, cellEffort as mr, HostFile as mt, AnalysisConfig as n, WebEventType as nn, DIRECTIONAL_MIN as nr, FakeVote as nt, AttemptContext as o, MODEL_REMARK_KINDS as on, EffortMetric as or, GooseHarnessDescriptor as ot, CaptureInstruction as p, Across as pn, cellContrast as pr, HistoryConfig as pt, ExecRetry as q, FamilyRecord as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, EvalInsight as rn, EFFORT_ALPHA as rr, FxHarnessDescriptor as rt, BackgroundHandle as s, NO_MODEL as sn, EffortSample as sr, GrokBuildHarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, CellEffort as tr, FakeScriptStep as tt, Budget as u, RunInsights as un, TOKENS_UNREPORTED as ur, HarnessDescriptor as ut, CaptureWait as v, CheckOutcome as vn, effortChangeLabel as vr, JudgeFailure as vt, CodexHarnessDescriptor as w, JudgeResult as wn, formatCount as wr, OpenCodeHarnessDescriptor as wt, CellRole as x, EffortCounters as xn, effortSample as xr, JudgeState as xt, CaptureWindow as y, CheckResult as yn, effortLabel as yr, JudgeModel as yt, Environment as z, messageLine as zn, ResolvedGateway as zt };
|
|
3669
|
+
//# sourceMappingURL=types-DXBbP9Mk.d.ts.map
|