reeval 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/dist/{adapter-IKKo4sZj.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-m-_t-IuW.js} +53 -25
- package/dist/{agent-ClE-g9VL.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-BDEpelpi.js} +92 -35
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-CVRwRTnb.js} +572 -36
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-C_4gaJ50.js} +132 -140
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C6vwmjS7.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C6vwmjS7.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CANun0m0.js → command-BXLUfw4h.js} +12 -12
- package/dist/{command-CANun0m0.js.map → command-BXLUfw4h.js.map} +1 -1
- package/dist/{command-CBaHukyl.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CBaHukyl.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-D_kEnYcX.js → command-Bxdy62dW.js} +14 -13
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-DJUuFoWQ.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-BnRlTpZA.js} +2 -2
- package/dist/{docker-DuHcQidD.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CQxucRPe.js → docker-D01slDF3.js} +26 -10
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-DQ2p0euG.js} +135 -36
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-CO0P0soF.js} +8 -7
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BUmsN7yG.js} +66 -27
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-C9-ritA5.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-Cfx1H70y.js → facts-DzOUTreB.js} +12 -4
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-CN1QfenC.js → files-B-CJeeEh.js} +13 -8
- package/dist/{files-CN1QfenC.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-Bs6H3HiG.js → files-C_gQNVB-.js} +3 -3
- package/dist/{files-Bs6H3HiG.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-D27nUk0K.js → files-kUCrswI6.js} +2 -2
- package/dist/{files-D27nUk0K.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-BnrvBAzF.js → generate-DZ7vp69F.js} +18 -17
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-Bk8DSwHy.js} +2 -2
- package/dist/{git-rPfl7AD1.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-1OkumOAm.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-BVzbRnNv.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-BVzbRnNv.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-m784ueGK.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-D_ePcwyL.js → invocations-DzyKSJBa.js} +2 -2
- package/dist/{invocations-D_ePcwyL.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-1ZLr9Dcv.js} +40 -26
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-UgHDMhI1.js → model-CgnOpss4.js} +57 -318
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/{model-B8H3gL69.js → model-Ch8kX87u.js} +1 -1
- package/dist/{node-CGZYh4Fj.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-snfeWd6Z.js → opencode-providers-klkqgzvb.js} +23 -52
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-uMq9rCl_.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-CZYgXnZh.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-Dkp_H3pB.js} +4 -4
- package/dist/{plugins-P-uh54VF.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-D3HvKeAx.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-Dx7dOnJB.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-CfZpFpAh.js} +2 -2
- package/dist/{provenance-C_-8y-x3.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-DXthKmq0.js → query-CemclveB.js} +2 -2
- package/dist/{query-DXthKmq0.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-KENpWnfV.js → questions-bAudIc04.js} +308 -45
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{remarks-Ml6bcSdV.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-Ml6bcSdV.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-Dkeghk8k.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-CBoKzQMP.js} +8 -9
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-D-naai_Q.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-8-nnW8iO.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-8-nnW8iO.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{rules-CcV3zLDR.js → rules-VhftPOOT.js} +53 -5
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-Bel7Vde4.js → run-CwIEK16q.js} +9 -9
- package/dist/{run-Bel7Vde4.js.map → run-CwIEK16q.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-De8oH3bt.js → server-TGW4WyAz.js} +25 -26
- package/dist/{server-De8oH3bt.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-C686eZpY.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-C686eZpY.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-CIpKL45U.js → src-8zoX_qrb.js} +36 -26
- package/dist/{src-CIpKL45U.js.map → src-8zoX_qrb.js.map} +1 -1
- package/dist/{src-CzQrAcF8.js → src-CHIwOy0o.js} +84 -52
- package/dist/src-CHIwOy0o.js.map +1 -0
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-djNNYIdP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-djNNYIdP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/{src-DrJSnoMR.js → src-CyaDNG9r.js} +1 -1
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-C7W6kg81.js → src-DKPVX2Us.js} +34 -16
- package/dist/{src-C7W6kg81.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-DmUgeTlE.js} +4 -4
- package/dist/{src-DeY52Gyf.js.map → src-DmUgeTlE.js.map} +1 -1
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-Xj_BZKbq.js → src-NSdQ9zI6.js} +31 -16
- package/dist/{src-Xj_BZKbq.js.map → src-NSdQ9zI6.js.map} +1 -1
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-C0i_ujQZ.js → state-B1t143wG.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-WgT7m6Vs.js} +76 -11
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-DXBbP9Mk.d.ts} +298 -70
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-DWjkshl1.js → validate-BhwqrVsm.js} +44 -12
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +133 -86
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-Dk8MCQKr.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-CqrzGPF4.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-CLeOybkr.js → workspace-CDI68wtX.js} +4 -4
- package/dist/{workspace-CLeOybkr.js.map → workspace-CDI68wtX.js.map} +1 -1
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +6 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
|
-
import {
|
|
2
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
+
import { c as jsonValueSchema, i as durationSchema, n as dataProblem, p as parseDuration } from "./shared-BDUzStad.js";
|
|
4
|
+
import { v as gatewayNameSchema } from "./harness-COVXSwJ6.js";
|
|
3
5
|
import { l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
6
|
+
import { Ct as judgeStateSchema, bt as judgeFields, v as gatewayScope, vt as captureWindowSchema, xt as judgeModelNameSchema, y as resolveModelGateway, yt as fakeJudgeDataSchema } from "./config-BrdHA9bJ.js";
|
|
7
|
+
import { l as judgeRoute } from "./judge-yLvjC5A3.js";
|
|
8
|
+
import { ft as shellQuote } from "./src-8zoX_qrb.js";
|
|
9
|
+
import { s as cacheDir } from "./root-Dlff-zp8.js";
|
|
7
10
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
8
|
-
import { n as getRunContext, o as SUBJECT } from "./runtime-PX_oXiCS.js";
|
|
9
|
-
import { i as debugTrace, r as debugLog } from "./attempt-jI0OWxlz.js";
|
|
10
11
|
import { i as formatDuration, n as clip } from "./text-CrVy5ioL.js";
|
|
11
|
-
import { o as
|
|
12
|
-
import {
|
|
12
|
+
import { n as getRunContext, o as SUBJECT } from "./runtime-BgQRyCLl.js";
|
|
13
|
+
import { i as debugTrace, r as debugLog } from "./attempt-jI0OWxlz.js";
|
|
14
|
+
import { o as providerModel } from "./model-1ZLr9Dcv.js";
|
|
15
|
+
import { t as addUsage } from "./usage-BNY1ABrU.js";
|
|
13
16
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
14
17
|
import { homedir, tmpdir } from "node:os";
|
|
15
18
|
import { join, posix } from "node:path";
|
|
16
19
|
import { z } from "zod";
|
|
17
20
|
import { createServer } from "node:net";
|
|
18
21
|
import { createHash, randomUUID } from "node:crypto";
|
|
22
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
19
23
|
import { spawn } from "node:child_process";
|
|
20
24
|
import { crc32, deflateSync, inflateRawSync } from "node:zlib";
|
|
21
25
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
@@ -900,9 +904,38 @@ function actionFromTool(name, input) {
|
|
|
900
904
|
y: Number(input.y)
|
|
901
905
|
};
|
|
902
906
|
}
|
|
907
|
+
/**
|
|
908
|
+
* Gemini models answer a point normalised to 0–1000 across the image they were shown, whatever
|
|
909
|
+
* the prompt asks, so their journeys are prompted in that scale and their points scaled back.
|
|
910
|
+
*/
|
|
911
|
+
const normalisesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
|
|
912
|
+
const NORMALISED = 1e3;
|
|
913
|
+
/** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
|
|
914
|
+
function scalePoints(action, page) {
|
|
915
|
+
const x = (n) => Math.round(n * page.width / NORMALISED);
|
|
916
|
+
const y = (n) => Math.round(n * page.height / NORMALISED);
|
|
917
|
+
switch (action.type) {
|
|
918
|
+
case "click":
|
|
919
|
+
case "hover":
|
|
920
|
+
case "scroll": return {
|
|
921
|
+
...action,
|
|
922
|
+
x: x(action.x),
|
|
923
|
+
y: y(action.y)
|
|
924
|
+
};
|
|
925
|
+
case "drag": return {
|
|
926
|
+
...action,
|
|
927
|
+
x: x(action.x),
|
|
928
|
+
y: y(action.y),
|
|
929
|
+
toX: x(action.toX),
|
|
930
|
+
toY: y(action.toY)
|
|
931
|
+
};
|
|
932
|
+
default: return action;
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
const systemPrompt = (normalised) => `You drive a web page from screenshots. Call one tool. ${normalised ? `Coordinates are normalised to 0-${NORMALISED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
|
|
903
936
|
const useTools = {
|
|
904
937
|
click: tool({
|
|
905
|
-
description: "Click at
|
|
938
|
+
description: "Click at (x, y) on the screenshot.",
|
|
906
939
|
inputSchema: z.object({
|
|
907
940
|
x: z.number(),
|
|
908
941
|
y: z.number()
|
|
@@ -966,6 +999,7 @@ async function runFakeScript(session, script, goal, remaining) {
|
|
|
966
999
|
}
|
|
967
1000
|
async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
968
1001
|
const model = await loadModel(modelName);
|
|
1002
|
+
const normalised = normalisesPoints(modelName);
|
|
969
1003
|
let moves = 0;
|
|
970
1004
|
let usage = null;
|
|
971
1005
|
while (moves < 32) {
|
|
@@ -980,7 +1014,7 @@ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
|
980
1014
|
let chosen;
|
|
981
1015
|
const result = await generateText({
|
|
982
1016
|
model,
|
|
983
|
-
system:
|
|
1017
|
+
system: systemPrompt(normalised),
|
|
984
1018
|
messages: [{
|
|
985
1019
|
role: "user",
|
|
986
1020
|
content: [{
|
|
@@ -999,6 +1033,7 @@ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
|
999
1033
|
usage = addUsage(usage, asUsage(result.usage ?? {}));
|
|
1000
1034
|
const call = result.toolCalls?.[0];
|
|
1001
1035
|
if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
|
|
1036
|
+
if (chosen && normalised) chosen = scalePoints(chosen, page);
|
|
1002
1037
|
if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
|
|
1003
1038
|
debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
|
|
1004
1039
|
debugTrace("check", "capture-use", {
|
|
@@ -1064,6 +1099,476 @@ function captureUseSpend(result) {
|
|
|
1064
1099
|
};
|
|
1065
1100
|
}
|
|
1066
1101
|
//#endregion
|
|
1102
|
+
//#region src/contract/attempt.ts
|
|
1103
|
+
const nullableNumber = z.number().nullable();
|
|
1104
|
+
const tokenUsageSchema = z.strictObject({
|
|
1105
|
+
input: z.number(),
|
|
1106
|
+
output: z.number(),
|
|
1107
|
+
cacheRead: z.number(),
|
|
1108
|
+
cacheWrite: z.number(),
|
|
1109
|
+
reasoning: nullableNumber
|
|
1110
|
+
}).describe("The tokens one turn used, as its harness reported them. `input` excludes cache reads and writes (so the prompt was `input + cacheRead + cacheWrite`); `reasoning` is part of `output`.");
|
|
1111
|
+
const fileChangeSchema = z.strictObject({
|
|
1112
|
+
path: z.string(),
|
|
1113
|
+
status: z.enum([
|
|
1114
|
+
"added",
|
|
1115
|
+
"modified",
|
|
1116
|
+
"deleted"
|
|
1117
|
+
])
|
|
1118
|
+
});
|
|
1119
|
+
const diffSchema = z.strictObject({
|
|
1120
|
+
patch: z.string(),
|
|
1121
|
+
files: z.array(fileChangeSchema)
|
|
1122
|
+
});
|
|
1123
|
+
const turnStatusSchema = z.enum([
|
|
1124
|
+
"completed",
|
|
1125
|
+
"aborted",
|
|
1126
|
+
"error"
|
|
1127
|
+
]);
|
|
1128
|
+
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]).describe("The transcript events a harness may be unable to report.");
|
|
1129
|
+
const at = {
|
|
1130
|
+
ts: z.string(),
|
|
1131
|
+
atMs: z.number()
|
|
1132
|
+
};
|
|
1133
|
+
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
1134
|
+
z.strictObject({
|
|
1135
|
+
...at,
|
|
1136
|
+
type: z.enum(["text", "reasoning"]),
|
|
1137
|
+
text: z.string(),
|
|
1138
|
+
durationMs: nullableNumber
|
|
1139
|
+
}),
|
|
1140
|
+
z.strictObject({
|
|
1141
|
+
...at,
|
|
1142
|
+
type: z.literal("file-edit"),
|
|
1143
|
+
toolCallId: z.string(),
|
|
1144
|
+
path: z.string(),
|
|
1145
|
+
kind: z.enum([
|
|
1146
|
+
"create",
|
|
1147
|
+
"modify",
|
|
1148
|
+
"delete"
|
|
1149
|
+
]),
|
|
1150
|
+
added: nullableNumber.describe("Line counts, or null when the harness didn't send the content (e.g. deletes)."),
|
|
1151
|
+
removed: nullableNumber,
|
|
1152
|
+
durationMs: nullableNumber,
|
|
1153
|
+
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1154
|
+
}),
|
|
1155
|
+
z.strictObject({
|
|
1156
|
+
...at,
|
|
1157
|
+
type: z.literal("web-search"),
|
|
1158
|
+
toolCallId: z.string(),
|
|
1159
|
+
query: z.string(),
|
|
1160
|
+
references: nullableNumber.describe("How many sources the harness says it used; Cursor sends no URLs or titles."),
|
|
1161
|
+
ok: z.boolean().optional().describe("Whether the search succeeded. Absent means it did."),
|
|
1162
|
+
durationMs: nullableNumber,
|
|
1163
|
+
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1164
|
+
}),
|
|
1165
|
+
z.strictObject({
|
|
1166
|
+
...at,
|
|
1167
|
+
type: z.literal("web-fetch"),
|
|
1168
|
+
toolCallId: z.string(),
|
|
1169
|
+
url: z.string(),
|
|
1170
|
+
ok: z.boolean(),
|
|
1171
|
+
durationMs: nullableNumber,
|
|
1172
|
+
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1173
|
+
}),
|
|
1174
|
+
z.strictObject({
|
|
1175
|
+
...at,
|
|
1176
|
+
type: z.literal("tool"),
|
|
1177
|
+
toolCallId: z.string(),
|
|
1178
|
+
tool: z.string(),
|
|
1179
|
+
input: z.unknown(),
|
|
1180
|
+
output: z.unknown(),
|
|
1181
|
+
ok: z.boolean(),
|
|
1182
|
+
durationMs: nullableNumber,
|
|
1183
|
+
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1184
|
+
}),
|
|
1185
|
+
z.strictObject({
|
|
1186
|
+
...at,
|
|
1187
|
+
type: z.literal("error"),
|
|
1188
|
+
message: z.string()
|
|
1189
|
+
}),
|
|
1190
|
+
z.strictObject({
|
|
1191
|
+
...at,
|
|
1192
|
+
type: z.literal("retry"),
|
|
1193
|
+
message: z.string()
|
|
1194
|
+
})
|
|
1195
|
+
]).describe("Normalised agent events. Each tool call is one event, merged with its result; `durationMs` is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null when the harness never reported the end. Long shell or edit calls set `stalled: true`.");
|
|
1196
|
+
const turnDataSchema = z.strictObject({
|
|
1197
|
+
prompt: z.string(),
|
|
1198
|
+
text: z.string().describe("The agent's final message for this turn."),
|
|
1199
|
+
status: turnStatusSchema,
|
|
1200
|
+
startedAt: z.string(),
|
|
1201
|
+
durationMs: z.number(),
|
|
1202
|
+
transcript: z.array(transcriptEventSchema),
|
|
1203
|
+
usage: tokenUsageSchema.nullable().describe("Tokens the turn used; null when the harness reports none."),
|
|
1204
|
+
model: z.string().nullable().describe("The model the harness says served the turn, e.g. `composer-2.5[fast=false]`; null when it reports none. It can differ from the requested one (an alias, or parameters it filled in)."),
|
|
1205
|
+
unreported: z.array(webEventTypeSchema).describe("Event types the harness couldn't report for this turn: the agent may have searched or fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops them from the stream) is `['web-search', 'web-fetch']`."),
|
|
1206
|
+
diff: diffSchema.describe("Workspace changes since the eval started, as of the end of this turn.")
|
|
1207
|
+
}).describe("A turn as data, as a record or another language's runner carries it: `Turn` without its methods.");
|
|
1208
|
+
const execResultSchema = z.strictObject({
|
|
1209
|
+
exitCode: z.number().int(),
|
|
1210
|
+
stdout: z.string(),
|
|
1211
|
+
stderr: z.string()
|
|
1212
|
+
});
|
|
1213
|
+
const execRetrySchema = z.strictObject({
|
|
1214
|
+
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional().describe("Extra tries after a non-zero exit, a whole number from 0. Default 0."),
|
|
1215
|
+
interval: z.number().min(0, "must be 0 or more milliseconds").optional().describe("Milliseconds to wait before each extra try. Default 1000.")
|
|
1216
|
+
}).describe("How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient.");
|
|
1217
|
+
/** A background command has no exit for `retries` to wait on. */
|
|
1218
|
+
const noBackgroundRetry = (options, ctx) => {
|
|
1219
|
+
if (!options.background) return;
|
|
1220
|
+
for (const key of ["retries", "interval"]) {
|
|
1221
|
+
if (options[key] === void 0) continue;
|
|
1222
|
+
ctx.addIssue({
|
|
1223
|
+
code: "custom",
|
|
1224
|
+
path: [key],
|
|
1225
|
+
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
};
|
|
1229
|
+
const execOptionsSchema = z.strictObject({
|
|
1230
|
+
background: z.boolean().optional(),
|
|
1231
|
+
...execRetrySchema.shape
|
|
1232
|
+
}).superRefine(noBackgroundRetry);
|
|
1233
|
+
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
1234
|
+
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
1235
|
+
const assertionRecordSchema = z.strictObject({
|
|
1236
|
+
label: z.string().describe("`expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt."),
|
|
1237
|
+
labelled: z.boolean().describe("The eval gave the label."),
|
|
1238
|
+
matcher: z.string().describe("`toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`."),
|
|
1239
|
+
status: z.enum([
|
|
1240
|
+
"pass",
|
|
1241
|
+
"fail",
|
|
1242
|
+
"error"
|
|
1243
|
+
]).describe("`error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report)."),
|
|
1244
|
+
soft: z.boolean(),
|
|
1245
|
+
message: z.string().optional().describe("Why it failed or couldn't answer, cut at 1 KB."),
|
|
1246
|
+
durationMs: z.number().optional().describe("Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`).")
|
|
1247
|
+
}).describe("One `expect` or `expect.soft` matcher call an attempt made.");
|
|
1248
|
+
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
1249
|
+
const captureSchema = z.strictObject({
|
|
1250
|
+
name: z.string(),
|
|
1251
|
+
port: z.number().describe("Container port that was tunneled."),
|
|
1252
|
+
path: z.string().describe("Path on the app, as passed in."),
|
|
1253
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).describe("What the author asked, including `'full-page'`."),
|
|
1254
|
+
file: z.strictObject({
|
|
1255
|
+
path: z.string(),
|
|
1256
|
+
sha256: z.string(),
|
|
1257
|
+
bytes: z.number()
|
|
1258
|
+
}).describe("`artifacts/<name>.png` under the attempt directory.")
|
|
1259
|
+
}).describe("What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`.");
|
|
1260
|
+
const judgeArtifactRefSchema = z.strictObject({
|
|
1261
|
+
name: z.string(),
|
|
1262
|
+
path: z.string().describe("`artifacts/<name>.png` under the attempt directory."),
|
|
1263
|
+
sha256: z.string(),
|
|
1264
|
+
bytes: z.number(),
|
|
1265
|
+
mediaType: z.literal("image/png")
|
|
1266
|
+
}).describe("A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`.");
|
|
1267
|
+
const verdictFields = {
|
|
1268
|
+
threshold: z.number(),
|
|
1269
|
+
question: z.string(),
|
|
1270
|
+
model: z.string(),
|
|
1271
|
+
gateway: gatewayNameSchema.optional().describe("The gateway from the config's `gateway` list the call went through; unset without one."),
|
|
1272
|
+
artifacts: z.array(judgeArtifactRefSchema).optional().describe("Captures this call sent the judge as file parts, when any.")
|
|
1273
|
+
};
|
|
1274
|
+
const verdictSchema = z.strictObject({
|
|
1275
|
+
verdict: z.enum(["pass", "fail"]),
|
|
1276
|
+
score: z.number().describe("The judge's probability that the answer is yes, from 0 to 1."),
|
|
1277
|
+
...verdictFields,
|
|
1278
|
+
resolvedModel: z.string().optional().describe("The model id the provider says served the call, when it reports one.")
|
|
1279
|
+
});
|
|
1280
|
+
const judgeFailureSchema = z.strictObject({
|
|
1281
|
+
verdict: z.literal("error"),
|
|
1282
|
+
score: z.null(),
|
|
1283
|
+
...verdictFields,
|
|
1284
|
+
error: z.string()
|
|
1285
|
+
}).describe("What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished, so the judges and checks after it still run.");
|
|
1286
|
+
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
1287
|
+
const patternSchema = z.strictObject({
|
|
1288
|
+
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
1289
|
+
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
1290
|
+
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
1291
|
+
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
1292
|
+
function hasAnsweredPattern(value) {
|
|
1293
|
+
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
1294
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1295
|
+
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
1296
|
+
}
|
|
1297
|
+
const answeredPatternSchema = z.strictObject({
|
|
1298
|
+
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
1299
|
+
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
1300
|
+
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
1301
|
+
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
1302
|
+
const matcherArgSchema = z.union([
|
|
1303
|
+
patternSchema,
|
|
1304
|
+
answeredPatternSchema,
|
|
1305
|
+
jsonValueSchema
|
|
1306
|
+
]).superRefine((value, ctx) => {
|
|
1307
|
+
if (!isPatternLike(value)) return;
|
|
1308
|
+
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
1309
|
+
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
1310
|
+
const matcherSubjectSchema = z.union([
|
|
1311
|
+
z.strictObject({ turn: turnIndex }),
|
|
1312
|
+
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
1313
|
+
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
1314
|
+
z.strictObject({
|
|
1315
|
+
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
1316
|
+
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
1317
|
+
}),
|
|
1318
|
+
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
1319
|
+
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
1320
|
+
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
1321
|
+
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
1322
|
+
const millisSchema = z.number().positive("more than zero milliseconds");
|
|
1323
|
+
const captureWaitSchema = z.enum(["load", "idle"]).describe("Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`). `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.");
|
|
1324
|
+
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]).describe("Same shape at top-level `wait` and `{ wait }` between instructions.");
|
|
1325
|
+
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
1326
|
+
const captureCallDataSchema = z.strictObject({
|
|
1327
|
+
name: nonEmpty.optional(),
|
|
1328
|
+
path: z.string().optional(),
|
|
1329
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
1330
|
+
timeout: durationSchema.optional(),
|
|
1331
|
+
wait: waitSchema.optional().describe("Look-ready once, before the first model shot or the saved PNG. Default `'load'`."),
|
|
1332
|
+
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
1333
|
+
model: z.string().optional().describe("Overrides cascaded `capture.model` for this call.")
|
|
1334
|
+
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
1335
|
+
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
1336
|
+
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
1337
|
+
const judgeCallDataSchema = z.strictObject({
|
|
1338
|
+
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
1339
|
+
...judgeFields,
|
|
1340
|
+
on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
|
|
1341
|
+
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
1342
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
|
|
1343
|
+
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
1344
|
+
z.strictObject({
|
|
1345
|
+
op: z.literal("send"),
|
|
1346
|
+
prompt: z.string().describe("The turn's prompt.")
|
|
1347
|
+
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
1348
|
+
z.strictObject({
|
|
1349
|
+
op: z.literal("exec"),
|
|
1350
|
+
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
1351
|
+
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
1352
|
+
...execRetrySchema.shape
|
|
1353
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
1354
|
+
z.strictObject({
|
|
1355
|
+
op: z.literal("serve"),
|
|
1356
|
+
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
1357
|
+
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
1358
|
+
}).describe("`c.serve(command, port?)`: a background process and its port."),
|
|
1359
|
+
z.strictObject({
|
|
1360
|
+
op: z.literal("kill"),
|
|
1361
|
+
handle: handleId
|
|
1362
|
+
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
1363
|
+
z.strictObject({
|
|
1364
|
+
op: z.literal("capture"),
|
|
1365
|
+
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
1366
|
+
options: captureCallDataSchema.optional()
|
|
1367
|
+
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
1368
|
+
z.strictObject({
|
|
1369
|
+
op: z.literal("readFile"),
|
|
1370
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
1371
|
+
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
1372
|
+
z.strictObject({
|
|
1373
|
+
op: z.literal("writeFile"),
|
|
1374
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
1375
|
+
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
1376
|
+
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
1377
|
+
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
1378
|
+
z.strictObject({
|
|
1379
|
+
op: z.literal("expect"),
|
|
1380
|
+
subject: matcherSubjectSchema,
|
|
1381
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
1382
|
+
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
1383
|
+
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
1384
|
+
soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
|
|
1385
|
+
label: z.string().optional().describe("`expect`'s second argument."),
|
|
1386
|
+
poll: z.strictObject({
|
|
1387
|
+
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
1388
|
+
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
1389
|
+
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
1390
|
+
}).superRefine((call, ctx) => {
|
|
1391
|
+
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
1392
|
+
code: "custom",
|
|
1393
|
+
path: ["subject"],
|
|
1394
|
+
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
1395
|
+
});
|
|
1396
|
+
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
1397
|
+
code: "custom",
|
|
1398
|
+
path: ["subject", "result"],
|
|
1399
|
+
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
1400
|
+
});
|
|
1401
|
+
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
1402
|
+
code: "custom",
|
|
1403
|
+
path: ["args"],
|
|
1404
|
+
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
1405
|
+
});
|
|
1406
|
+
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
1407
|
+
z.strictObject({
|
|
1408
|
+
op: z.literal("record"),
|
|
1409
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
1410
|
+
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
1411
|
+
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
1412
|
+
status: z.enum(["pass", "fail"]),
|
|
1413
|
+
message: z.string().optional().describe("Why it failed."),
|
|
1414
|
+
soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
|
|
1415
|
+
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
1416
|
+
z.strictObject({
|
|
1417
|
+
op: z.literal("judge"),
|
|
1418
|
+
question: nonEmpty.describe("A yes/no question about the graded turn or value."),
|
|
1419
|
+
options: judgeCallDataSchema.optional()
|
|
1420
|
+
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
1421
|
+
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
1422
|
+
const attemptErrorSchema = z.looseObject({
|
|
1423
|
+
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
1424
|
+
message: z.string()
|
|
1425
|
+
}).describe("Why a call failed, in reeval's words.");
|
|
1426
|
+
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
1427
|
+
const attemptRequestSchema = z.strictObject({
|
|
1428
|
+
id,
|
|
1429
|
+
call: attemptCallSchema
|
|
1430
|
+
}).describe("A call, numbered so its reply can find it.");
|
|
1431
|
+
z.union([
|
|
1432
|
+
z.discriminatedUnion("op", [
|
|
1433
|
+
z.strictObject({
|
|
1434
|
+
id,
|
|
1435
|
+
ok: z.literal(true),
|
|
1436
|
+
op: z.literal("send"),
|
|
1437
|
+
turn: turnDataSchema
|
|
1438
|
+
}),
|
|
1439
|
+
z.strictObject({
|
|
1440
|
+
id,
|
|
1441
|
+
ok: z.literal(true),
|
|
1442
|
+
op: z.literal("exec"),
|
|
1443
|
+
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
1444
|
+
}),
|
|
1445
|
+
z.strictObject({
|
|
1446
|
+
id,
|
|
1447
|
+
ok: z.literal(true),
|
|
1448
|
+
op: z.literal("serve"),
|
|
1449
|
+
handle: handleId,
|
|
1450
|
+
port: z.number().describe("The container port it was given.")
|
|
1451
|
+
}),
|
|
1452
|
+
z.strictObject({
|
|
1453
|
+
id,
|
|
1454
|
+
ok: z.literal(true),
|
|
1455
|
+
op: z.literal("kill")
|
|
1456
|
+
}),
|
|
1457
|
+
z.strictObject({
|
|
1458
|
+
id,
|
|
1459
|
+
ok: z.literal(true),
|
|
1460
|
+
op: z.literal("capture"),
|
|
1461
|
+
capture: captureSchema
|
|
1462
|
+
}),
|
|
1463
|
+
z.strictObject({
|
|
1464
|
+
id,
|
|
1465
|
+
ok: z.literal(true),
|
|
1466
|
+
op: z.literal("readFile"),
|
|
1467
|
+
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
1468
|
+
}),
|
|
1469
|
+
z.strictObject({
|
|
1470
|
+
id,
|
|
1471
|
+
ok: z.literal(true),
|
|
1472
|
+
op: z.literal("writeFile")
|
|
1473
|
+
}),
|
|
1474
|
+
z.strictObject({
|
|
1475
|
+
id,
|
|
1476
|
+
ok: z.literal(true),
|
|
1477
|
+
op: z.literal("expect"),
|
|
1478
|
+
assertion: assertionRecordSchema
|
|
1479
|
+
}),
|
|
1480
|
+
z.strictObject({
|
|
1481
|
+
id,
|
|
1482
|
+
ok: z.literal(true),
|
|
1483
|
+
op: z.literal("record"),
|
|
1484
|
+
assertion: assertionRecordSchema
|
|
1485
|
+
}),
|
|
1486
|
+
z.strictObject({
|
|
1487
|
+
id,
|
|
1488
|
+
ok: z.literal(true),
|
|
1489
|
+
op: z.literal("judge"),
|
|
1490
|
+
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
1491
|
+
})
|
|
1492
|
+
]),
|
|
1493
|
+
z.strictObject({
|
|
1494
|
+
id,
|
|
1495
|
+
ok: z.literal(true),
|
|
1496
|
+
op: z.literal("expect"),
|
|
1497
|
+
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
1498
|
+
}),
|
|
1499
|
+
z.strictObject({
|
|
1500
|
+
id,
|
|
1501
|
+
ok: z.literal(false),
|
|
1502
|
+
op: z.enum([
|
|
1503
|
+
"send",
|
|
1504
|
+
"exec",
|
|
1505
|
+
"readFile",
|
|
1506
|
+
"writeFile",
|
|
1507
|
+
"expect",
|
|
1508
|
+
"record",
|
|
1509
|
+
"judge",
|
|
1510
|
+
"serve",
|
|
1511
|
+
"kill",
|
|
1512
|
+
"capture"
|
|
1513
|
+
]),
|
|
1514
|
+
error: attemptErrorSchema
|
|
1515
|
+
})
|
|
1516
|
+
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
1517
|
+
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
1518
|
+
const cellSchema = z.strictObject({
|
|
1519
|
+
key: z.string(),
|
|
1520
|
+
harness: z.string(),
|
|
1521
|
+
environment: z.string().describe("The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix)."),
|
|
1522
|
+
sandbox: z.string(),
|
|
1523
|
+
role: cellRoleSchema,
|
|
1524
|
+
variant: z.string().optional().describe("The variant's name, for variant cells.")
|
|
1525
|
+
}).describe("One cell of an eval: a harness, an environment (`-` when none is set) and that environment's image (`sandbox` id: tag, dockerfile and install). With `compare`, the control is `baseline` (or the cascaded environment, or an empty `baseline`) and each variant is that environment on its own, including its sandbox.");
|
|
1526
|
+
z.strictObject({
|
|
1527
|
+
type: z.literal("start"),
|
|
1528
|
+
attempt: z.strictObject({
|
|
1529
|
+
id: nonEmpty,
|
|
1530
|
+
cell: cellSchema,
|
|
1531
|
+
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
1532
|
+
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
1533
|
+
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
1534
|
+
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
1535
|
+
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
1536
|
+
z.strictObject({
|
|
1537
|
+
type: z.literal("abort"),
|
|
1538
|
+
reason: z.enum(["timeout", "cancelled"]).describe("`timeout`: the eval's `timeout` ran out. `cancelled`: the run was stopped."),
|
|
1539
|
+
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
1540
|
+
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
1541
|
+
z.strictObject({
|
|
1542
|
+
type: z.literal("done"),
|
|
1543
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
|
|
1544
|
+
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
1545
|
+
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
1546
|
+
const OTHER_KEYS = [
|
|
1547
|
+
"id",
|
|
1548
|
+
"call",
|
|
1549
|
+
"turn",
|
|
1550
|
+
"attempt",
|
|
1551
|
+
"value",
|
|
1552
|
+
"regexp",
|
|
1553
|
+
"flags",
|
|
1554
|
+
"matchedBy",
|
|
1555
|
+
"answers",
|
|
1556
|
+
...Object.keys(judgeCallDataSchema.shape),
|
|
1557
|
+
...Object.keys(judgeStateSchema.shape),
|
|
1558
|
+
...Object.keys(captureSchema.shape),
|
|
1559
|
+
...Object.keys(captureCallDataSchema.shape),
|
|
1560
|
+
"port",
|
|
1561
|
+
"handle",
|
|
1562
|
+
"exec",
|
|
1563
|
+
"result",
|
|
1564
|
+
"readFile",
|
|
1565
|
+
"interval",
|
|
1566
|
+
"timeout",
|
|
1567
|
+
"ms"
|
|
1568
|
+
];
|
|
1569
|
+
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
1570
|
+
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
1571
|
+
//#endregion
|
|
1067
1572
|
//#region src/diff.ts
|
|
1068
1573
|
const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0";
|
|
1069
1574
|
const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE=${shellQuote(sandbox.workdir)} GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
|
|
@@ -1125,45 +1630,73 @@ const asCheck = (a, fn) => a.phases ? a.phases.checks(fn) : fn();
|
|
|
1125
1630
|
/** The attempt's sandbox, started (with its session) by the first call that needs it. */
|
|
1126
1631
|
const sandboxFor = async (a, who) => (await a.start(who))[0];
|
|
1127
1632
|
/**
|
|
1128
|
-
* `c.exec` runs with the checks' variables and secrets, each run recorded and
|
|
1129
|
-
*
|
|
1633
|
+
* `c.exec` runs with the checks' variables and secrets, each run recorded and timed, and runs
|
|
1634
|
+
* again on a non-zero exit while it has `retries` left; `c.readFile` reads the sandbox as is;
|
|
1635
|
+
* `c.writeFile` places a file the diff leaves out.
|
|
1130
1636
|
*/
|
|
1131
1637
|
function createChecks(a) {
|
|
1132
1638
|
const checkEnv = () => a.env?.checkVars ?? {};
|
|
1133
|
-
const execForeground = async (command) => {
|
|
1639
|
+
const execForeground = async (command, retries, interval) => {
|
|
1134
1640
|
const box = await sandboxFor(a, "c.exec()");
|
|
1135
1641
|
const start = performance.now();
|
|
1642
|
+
const tries = [];
|
|
1136
1643
|
a.report({
|
|
1137
1644
|
kind: "check-start",
|
|
1138
1645
|
command
|
|
1139
1646
|
});
|
|
1140
|
-
const r = await a.timed("checksMs",
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1647
|
+
const r = await a.timed("checksMs", async () => {
|
|
1648
|
+
for (;;) {
|
|
1649
|
+
const began = performance.now();
|
|
1650
|
+
const r = await asCheck(a, () => box.exec(command, {
|
|
1651
|
+
signal: a.signal,
|
|
1652
|
+
env: checkEnv()
|
|
1653
|
+
}));
|
|
1654
|
+
tries.push({
|
|
1655
|
+
...r,
|
|
1656
|
+
durationMs: Math.round(performance.now() - began)
|
|
1657
|
+
});
|
|
1658
|
+
if (r.exitCode === 0 || tries.length > retries) return r;
|
|
1659
|
+
debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
|
|
1660
|
+
await setTimeout$1(interval, void 0, { signal: a.signal });
|
|
1661
|
+
}
|
|
1662
|
+
});
|
|
1144
1663
|
const durationMs = Math.round(performance.now() - start);
|
|
1664
|
+
const retried = retries > 0 ? {
|
|
1665
|
+
retries,
|
|
1666
|
+
tries
|
|
1667
|
+
} : {};
|
|
1145
1668
|
a.record.checks.push({
|
|
1146
1669
|
command,
|
|
1147
1670
|
...r,
|
|
1148
|
-
durationMs
|
|
1671
|
+
durationMs,
|
|
1672
|
+
...retried
|
|
1149
1673
|
});
|
|
1150
|
-
|
|
1674
|
+
const of = retries > 0 ? ` on try ${tries.length} of ${retries + 1}` : "";
|
|
1675
|
+
debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
|
|
1151
1676
|
debugTrace("check", "output", {
|
|
1152
1677
|
command,
|
|
1153
1678
|
...r,
|
|
1154
|
-
durationMs
|
|
1679
|
+
durationMs,
|
|
1680
|
+
...retried
|
|
1155
1681
|
}, `check output: ${clip(command, 80)}`);
|
|
1156
1682
|
a.report({
|
|
1157
1683
|
kind: "check-end",
|
|
1158
1684
|
command,
|
|
1159
1685
|
exitCode: r.exitCode,
|
|
1160
|
-
ms: durationMs
|
|
1686
|
+
ms: durationMs,
|
|
1687
|
+
...retries > 0 ? {
|
|
1688
|
+
try: tries.length,
|
|
1689
|
+
tries: retries + 1
|
|
1690
|
+
} : {}
|
|
1161
1691
|
});
|
|
1162
1692
|
return r;
|
|
1163
1693
|
};
|
|
1164
1694
|
function exec(command, options) {
|
|
1165
|
-
|
|
1166
|
-
|
|
1695
|
+
const problem = options === void 0 ? void 0 : execOptionsProblem(options);
|
|
1696
|
+
if (problem) return Promise.reject(new UsageError(problem));
|
|
1697
|
+
if (options?.background) return import("./capture-CherbjzB.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
|
|
1698
|
+
const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
|
|
1699
|
+
return Object.defineProperty(run, SUBJECT, { value: command });
|
|
1167
1700
|
}
|
|
1168
1701
|
const readFile = async (path) => (await sandboxFor(a, "c.readFile()")).readFile(path);
|
|
1169
1702
|
const writeFile = async (path, contents) => {
|
|
@@ -1188,25 +1721,26 @@ function createChecks(a) {
|
|
|
1188
1721
|
const BG_LOG = "/tmp/reeval-bg.log";
|
|
1189
1722
|
const BG_TAG = "REEVAL_BACKGROUND";
|
|
1190
1723
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
1191
|
-
|
|
1724
|
+
const languageModel = (model) => typeof model === "string" && !isEvaluationModel(model) ? model : void 0;
|
|
1725
|
+
/**
|
|
1726
|
+
* `analysis.model`, else the attempt's judge (a profile's, group's or eval's over the config's),
|
|
1727
|
+
* whichever is first a language model.
|
|
1728
|
+
*/
|
|
1729
|
+
function defaultCaptureModel(a) {
|
|
1192
1730
|
try {
|
|
1193
|
-
|
|
1194
|
-
const analysis = config.analysis.model;
|
|
1195
|
-
if (typeof analysis === "string" && !isEvaluationModel(analysis)) return analysis;
|
|
1196
|
-
const judge = config.judge.model;
|
|
1197
|
-
if (typeof judge === "string" && !isEvaluationModel(judge)) return judge;
|
|
1731
|
+
return languageModel(getRunContext().config.analysis.model) ?? languageModel(a.spec.judge.model);
|
|
1198
1732
|
} catch {
|
|
1199
1733
|
return;
|
|
1200
1734
|
}
|
|
1201
1735
|
}
|
|
1202
|
-
function resolveUse(resolved, fakeSandbox) {
|
|
1736
|
+
function resolveUse(a, resolved, fakeSandbox) {
|
|
1203
1737
|
if (resolved.fake) return { fake: resolved.fake };
|
|
1204
1738
|
if (resolved.model) return { model: resolved.model };
|
|
1205
1739
|
if (!resolved.instructions.length) return void 0;
|
|
1206
|
-
const fallback = defaultCaptureModel();
|
|
1740
|
+
const fallback = defaultCaptureModel(a);
|
|
1207
1741
|
if (fallback) return { model: fallback };
|
|
1208
1742
|
if (fakeSandbox) return void 0;
|
|
1209
|
-
throw new Error("c.capture() instructions need a vision model. Set capture.model, pass model on the call, or analysis.model to a language model that accepts images. Tests pass fakeInstructions from reeval/testing.");
|
|
1743
|
+
throw new Error("c.capture() instructions need a vision model. Set capture.model, pass model on the call, or set analysis.model or the judge model to a language model that accepts images. Tests pass fakeInstructions from reeval/testing.");
|
|
1210
1744
|
}
|
|
1211
1745
|
/**
|
|
1212
1746
|
* Stops `pid` and its descendants under Git for Windows' MSYS. Native children (node.exe) are
|
|
@@ -1282,7 +1816,7 @@ async function capturePage(a, target, call) {
|
|
|
1282
1816
|
throw new Error(`nothing listening on port ${resolved.port}`);
|
|
1283
1817
|
}
|
|
1284
1818
|
const fake = a.spec.sandbox.kind === "fake";
|
|
1285
|
-
const use = resolveUse(resolved, fake);
|
|
1819
|
+
const use = resolveUse(a, resolved, fake);
|
|
1286
1820
|
a.captureNames.add(resolved.name);
|
|
1287
1821
|
const started = Date.now();
|
|
1288
1822
|
const remaining = () => Math.max(0, resolved.timeoutMs - (Date.now() - started));
|
|
@@ -1346,7 +1880,9 @@ async function shot(a, url, resolved, remaining, onMoves) {
|
|
|
1346
1880
|
use: resolved.use,
|
|
1347
1881
|
remaining,
|
|
1348
1882
|
loadModel: async (name) => {
|
|
1349
|
-
const
|
|
1883
|
+
const { config, profile } = getRunContext();
|
|
1884
|
+
const gateway = resolveModelGateway(name, void 0, gatewayScope(config, profile), "capture model");
|
|
1885
|
+
const route = judgeRoute(name, process.env, "capture model", gateway);
|
|
1350
1886
|
return providerModel({
|
|
1351
1887
|
name,
|
|
1352
1888
|
route: route.kind === "direct" ? "direct" : "gateway"
|
|
@@ -1374,6 +1910,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
|
|
|
1374
1910
|
for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
|
|
1375
1911
|
}
|
|
1376
1912
|
//#endregion
|
|
1377
|
-
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d,
|
|
1913
|
+
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, attemptRequestProblem as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, fakeInstructions as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
|
|
1378
1914
|
|
|
1379
|
-
//# sourceMappingURL=capture-
|
|
1915
|
+
//# sourceMappingURL=capture-CVRwRTnb.js.map
|