reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
2
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import { c as jsonValueSchema, i as durationSchema, n as dataProblem, p as
|
|
4
|
-
import {
|
|
3
|
+
import { c as jsonValueSchema, h as parseDuration, i as durationSchema, m as TIMER_LIMIT, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
4
|
+
import { y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
|
|
5
5
|
import { l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
6
|
-
import { Ct as
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
6
|
+
import { Ct as captureWindowSchema, Et as judgeModelNameSchema, Ot as judgeStateSchema, Tt as judgeFields, b as resolveModelGateway, v as gatewayScope, wt as fakeJudgeDataSchema } from "./config-k6m7XbAz.js";
|
|
7
|
+
import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
8
|
+
import { m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
9
|
+
import { i as formatDuration, n as clip } from "./text-DTigcA64.js";
|
|
9
10
|
import { s as cacheDir } from "./root-Dlff-zp8.js";
|
|
11
|
+
import { ft as shellQuote } from "./src-D91Hd6hT.js";
|
|
10
12
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
13
|
+
import { r as getRunContext, s as SUBJECT } from "./runtime-d56Gg3J5.js";
|
|
14
|
+
import { i as debugTrace, r as debugLog } from "./attempt-Ba2QerOK.js";
|
|
15
|
+
import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
|
|
16
|
+
import { i as hostModel } from "./model-Bq3G0OG1.js";
|
|
15
17
|
import { t as addUsage } from "./usage-BNY1ABrU.js";
|
|
16
18
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
17
19
|
import { homedir, tmpdir } from "node:os";
|
|
@@ -24,555 +26,1030 @@ import { spawn } from "node:child_process";
|
|
|
24
26
|
import { crc32, deflateSync, inflateRawSync } from "node:zlib";
|
|
25
27
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
26
28
|
import { generateText, stepCountIs, tool } from "ai";
|
|
27
|
-
//#region src/
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
return scrollY + action.dy;
|
|
153
|
-
}
|
|
154
|
-
const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
|
|
155
|
-
if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
|
|
156
|
-
type: "mouseWheel",
|
|
157
|
-
x: window.width / 2,
|
|
158
|
-
y: window.height / 2,
|
|
159
|
-
deltaX: 0,
|
|
160
|
-
deltaY: mapped.dy
|
|
161
|
-
});
|
|
162
|
-
const at = {
|
|
163
|
-
x: mapped.x,
|
|
164
|
-
y: mapped.y
|
|
165
|
-
};
|
|
166
|
-
if (action.type === "hover") {
|
|
167
|
-
await cdp.send("Input.dispatchMouseEvent", {
|
|
168
|
-
type: "mouseMoved",
|
|
169
|
-
...at
|
|
29
|
+
//#region src/contract/attempt.ts
|
|
30
|
+
const nullableNumber = z.number().nullable();
|
|
31
|
+
const tokenUsageSchema = z.strictObject({
|
|
32
|
+
input: z.number(),
|
|
33
|
+
output: z.number(),
|
|
34
|
+
cacheRead: z.number(),
|
|
35
|
+
cacheWrite: z.number(),
|
|
36
|
+
reasoning: nullableNumber
|
|
37
|
+
});
|
|
38
|
+
const fileChangeSchema = z.strictObject({
|
|
39
|
+
path: z.string(),
|
|
40
|
+
status: z.enum([
|
|
41
|
+
"added",
|
|
42
|
+
"modified",
|
|
43
|
+
"deleted"
|
|
44
|
+
])
|
|
45
|
+
});
|
|
46
|
+
const diffSchema = z.strictObject({
|
|
47
|
+
patch: z.string(),
|
|
48
|
+
files: z.array(fileChangeSchema)
|
|
49
|
+
});
|
|
50
|
+
const turnStatusSchema = z.enum([
|
|
51
|
+
"completed",
|
|
52
|
+
"aborted",
|
|
53
|
+
"error"
|
|
54
|
+
]);
|
|
55
|
+
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
|
|
56
|
+
const at = {
|
|
57
|
+
ts: z.string(),
|
|
58
|
+
atMs: z.number()
|
|
59
|
+
};
|
|
60
|
+
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
61
|
+
z.strictObject({
|
|
62
|
+
...at,
|
|
63
|
+
type: z.enum(["text", "reasoning"]),
|
|
64
|
+
text: z.string(),
|
|
65
|
+
durationMs: nullableNumber
|
|
66
|
+
}),
|
|
67
|
+
z.strictObject({
|
|
68
|
+
...at,
|
|
69
|
+
type: z.literal("file-edit"),
|
|
70
|
+
toolCallId: z.string(),
|
|
71
|
+
path: z.string(),
|
|
72
|
+
kind: z.enum([
|
|
73
|
+
"create",
|
|
74
|
+
"modify",
|
|
75
|
+
"write",
|
|
76
|
+
"delete"
|
|
77
|
+
]),
|
|
78
|
+
added: nullableNumber,
|
|
79
|
+
removed: nullableNumber,
|
|
80
|
+
durationMs: nullableNumber,
|
|
81
|
+
stalled: z.boolean().optional()
|
|
82
|
+
}),
|
|
83
|
+
z.strictObject({
|
|
84
|
+
...at,
|
|
85
|
+
type: z.literal("web-search"),
|
|
86
|
+
toolCallId: z.string(),
|
|
87
|
+
query: z.string(),
|
|
88
|
+
references: nullableNumber,
|
|
89
|
+
ok: z.boolean().optional(),
|
|
90
|
+
durationMs: nullableNumber,
|
|
91
|
+
stalled: z.boolean().optional()
|
|
92
|
+
}),
|
|
93
|
+
z.strictObject({
|
|
94
|
+
...at,
|
|
95
|
+
type: z.literal("web-fetch"),
|
|
96
|
+
toolCallId: z.string(),
|
|
97
|
+
url: z.string(),
|
|
98
|
+
ok: z.boolean(),
|
|
99
|
+
durationMs: nullableNumber,
|
|
100
|
+
stalled: z.boolean().optional()
|
|
101
|
+
}),
|
|
102
|
+
z.strictObject({
|
|
103
|
+
...at,
|
|
104
|
+
type: z.literal("tool"),
|
|
105
|
+
toolCallId: z.string(),
|
|
106
|
+
tool: z.string(),
|
|
107
|
+
input: z.unknown(),
|
|
108
|
+
output: z.unknown(),
|
|
109
|
+
ok: z.boolean(),
|
|
110
|
+
durationMs: nullableNumber,
|
|
111
|
+
stalled: z.boolean().optional()
|
|
112
|
+
}),
|
|
113
|
+
z.strictObject({
|
|
114
|
+
...at,
|
|
115
|
+
type: z.literal("error"),
|
|
116
|
+
message: z.string()
|
|
117
|
+
}),
|
|
118
|
+
z.strictObject({
|
|
119
|
+
...at,
|
|
120
|
+
type: z.literal("retry"),
|
|
121
|
+
message: z.string()
|
|
122
|
+
})
|
|
123
|
+
]);
|
|
124
|
+
const turnDataSchema = z.strictObject({
|
|
125
|
+
prompt: z.string(),
|
|
126
|
+
text: z.string(),
|
|
127
|
+
status: turnStatusSchema,
|
|
128
|
+
startedAt: z.string(),
|
|
129
|
+
durationMs: z.number(),
|
|
130
|
+
transcript: z.array(transcriptEventSchema),
|
|
131
|
+
usage: tokenUsageSchema.nullable(),
|
|
132
|
+
model: z.string().nullable(),
|
|
133
|
+
unreported: z.array(webEventTypeSchema),
|
|
134
|
+
diff: diffSchema
|
|
135
|
+
});
|
|
136
|
+
const execResultSchema = z.strictObject({
|
|
137
|
+
exitCode: z.number().int(),
|
|
138
|
+
stdout: z.string(),
|
|
139
|
+
stderr: z.string()
|
|
140
|
+
});
|
|
141
|
+
const execRetrySchema = z.strictObject({
|
|
142
|
+
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
143
|
+
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
144
|
+
});
|
|
145
|
+
/** A background command has no exit for `retries` to wait on. */
|
|
146
|
+
const noBackgroundRetry = (options, ctx) => {
|
|
147
|
+
if (!options.background) return;
|
|
148
|
+
for (const key of ["retries", "interval"]) {
|
|
149
|
+
if (options[key] === void 0) continue;
|
|
150
|
+
ctx.addIssue({
|
|
151
|
+
code: "custom",
|
|
152
|
+
path: [key],
|
|
153
|
+
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
170
154
|
});
|
|
171
|
-
return mapped.scrollY;
|
|
172
155
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
156
|
+
};
|
|
157
|
+
const execOptionsSchema = z.strictObject({
|
|
158
|
+
background: z.boolean().optional(),
|
|
159
|
+
...execRetrySchema.shape
|
|
160
|
+
}).superRefine(noBackgroundRetry);
|
|
161
|
+
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
162
|
+
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
163
|
+
const assertionRecordSchema = z.strictObject({
|
|
164
|
+
label: z.string(),
|
|
165
|
+
labelled: z.boolean(),
|
|
166
|
+
matcher: z.string(),
|
|
167
|
+
status: z.enum([
|
|
168
|
+
"pass",
|
|
169
|
+
"fail",
|
|
170
|
+
"error"
|
|
171
|
+
]),
|
|
172
|
+
soft: z.boolean(),
|
|
173
|
+
message: z.string().optional(),
|
|
174
|
+
durationMs: z.number().optional()
|
|
175
|
+
});
|
|
176
|
+
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
177
|
+
const captureSchema = z.strictObject({
|
|
178
|
+
name: z.string(),
|
|
179
|
+
port: z.number(),
|
|
180
|
+
path: z.string(),
|
|
181
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]),
|
|
182
|
+
file: z.strictObject({
|
|
183
|
+
path: z.string(),
|
|
184
|
+
sha256: z.string(),
|
|
185
|
+
bytes: z.number()
|
|
186
|
+
})
|
|
187
|
+
});
|
|
188
|
+
const judgeArtifactRefSchema = z.strictObject({
|
|
189
|
+
name: z.string(),
|
|
190
|
+
path: z.string(),
|
|
191
|
+
sha256: z.string(),
|
|
192
|
+
bytes: z.number(),
|
|
193
|
+
mediaType: z.literal("image/png")
|
|
194
|
+
});
|
|
195
|
+
const verdictFields = {
|
|
196
|
+
threshold: z.number(),
|
|
197
|
+
question: z.string(),
|
|
198
|
+
model: z.string(),
|
|
199
|
+
gateway: gatewayNameSchema.optional(),
|
|
200
|
+
artifacts: z.array(judgeArtifactRefSchema).optional()
|
|
201
|
+
};
|
|
202
|
+
const verdictSchema = z.strictObject({
|
|
203
|
+
verdict: z.enum(["pass", "fail"]),
|
|
204
|
+
score: z.number(),
|
|
205
|
+
...verdictFields,
|
|
206
|
+
resolvedModel: z.string().optional(),
|
|
207
|
+
upstream: z.string().optional(),
|
|
208
|
+
costUsd: z.number().optional()
|
|
209
|
+
});
|
|
210
|
+
const judgeFailureSchema = z.strictObject({
|
|
211
|
+
verdict: z.literal("error"),
|
|
212
|
+
score: z.null(),
|
|
213
|
+
...verdictFields,
|
|
214
|
+
error: z.string()
|
|
215
|
+
});
|
|
216
|
+
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
217
|
+
const patternSchema = z.strictObject({
|
|
218
|
+
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
219
|
+
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
220
|
+
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
221
|
+
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
222
|
+
function hasAnsweredPattern(value) {
|
|
223
|
+
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
224
|
+
if (typeof value !== "object" || value === null) return false;
|
|
225
|
+
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
226
|
+
}
|
|
227
|
+
const answeredPatternSchema = z.strictObject({
|
|
228
|
+
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
229
|
+
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
230
|
+
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
231
|
+
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
232
|
+
const matcherArgSchema = z.union([
|
|
233
|
+
patternSchema,
|
|
234
|
+
answeredPatternSchema,
|
|
235
|
+
jsonValueSchema
|
|
236
|
+
]).superRefine((value, ctx) => {
|
|
237
|
+
if (!isPatternLike(value)) return;
|
|
238
|
+
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
239
|
+
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
240
|
+
const matcherSubjectSchema = z.union([
|
|
241
|
+
z.strictObject({ turn: turnIndex }),
|
|
242
|
+
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
243
|
+
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
244
|
+
z.strictObject({
|
|
245
|
+
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
246
|
+
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
247
|
+
}),
|
|
248
|
+
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
249
|
+
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
250
|
+
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
251
|
+
const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
|
|
252
|
+
const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
|
|
253
|
+
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
254
|
+
const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
|
|
255
|
+
const captureWaitSchema = z.enum(["load", "idle"]);
|
|
256
|
+
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
|
|
257
|
+
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
258
|
+
const captureCallDataSchema = z.strictObject({
|
|
259
|
+
name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
|
|
260
|
+
path: z.string().optional(),
|
|
261
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
262
|
+
timeout: durationSchema.optional(),
|
|
263
|
+
wait: waitSchema.optional(),
|
|
264
|
+
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
265
|
+
model: z.string().optional()
|
|
266
|
+
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
267
|
+
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
268
|
+
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
269
|
+
const judgeCallDataSchema = z.strictObject({
|
|
270
|
+
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
271
|
+
...judgeFields,
|
|
272
|
+
on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
|
|
273
|
+
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
274
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
|
|
275
|
+
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
276
|
+
z.strictObject({
|
|
277
|
+
op: z.literal("send"),
|
|
278
|
+
prompt: z.string().describe("The turn's prompt.")
|
|
279
|
+
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
280
|
+
z.strictObject({
|
|
281
|
+
op: z.literal("exec"),
|
|
282
|
+
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
283
|
+
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
284
|
+
...execRetrySchema.shape
|
|
285
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
286
|
+
z.strictObject({
|
|
287
|
+
op: z.literal("serve"),
|
|
288
|
+
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
289
|
+
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
290
|
+
}).describe("`c.serve(command, port?)`: a background process and its port."),
|
|
291
|
+
z.strictObject({
|
|
292
|
+
op: z.literal("kill"),
|
|
293
|
+
handle: handleId
|
|
294
|
+
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
295
|
+
z.strictObject({
|
|
296
|
+
op: z.literal("capture"),
|
|
297
|
+
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
298
|
+
options: captureCallDataSchema.optional()
|
|
299
|
+
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
300
|
+
z.strictObject({
|
|
301
|
+
op: z.literal("readFile"),
|
|
302
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
303
|
+
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
304
|
+
z.strictObject({
|
|
305
|
+
op: z.literal("writeFile"),
|
|
306
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
307
|
+
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
308
|
+
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
309
|
+
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
310
|
+
z.strictObject({
|
|
311
|
+
op: z.literal("expect"),
|
|
312
|
+
subject: matcherSubjectSchema,
|
|
313
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
314
|
+
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
315
|
+
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
316
|
+
soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
|
|
317
|
+
label: z.string().optional().describe("`expect`'s second argument."),
|
|
318
|
+
poll: z.strictObject({
|
|
319
|
+
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
320
|
+
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
321
|
+
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
322
|
+
}).superRefine((call, ctx) => {
|
|
323
|
+
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
324
|
+
code: "custom",
|
|
325
|
+
path: ["subject"],
|
|
326
|
+
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
178
327
|
});
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
clickCount: 1
|
|
328
|
+
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
329
|
+
code: "custom",
|
|
330
|
+
path: ["subject", "result"],
|
|
331
|
+
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
184
332
|
});
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
clickCount: 1
|
|
333
|
+
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
334
|
+
code: "custom",
|
|
335
|
+
path: ["args"],
|
|
336
|
+
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
190
337
|
});
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
})
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
338
|
+
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
339
|
+
z.strictObject({
|
|
340
|
+
op: z.literal("record"),
|
|
341
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
342
|
+
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
343
|
+
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
344
|
+
status: z.enum(["pass", "fail"]),
|
|
345
|
+
message: z.string().optional().describe("Why it failed."),
|
|
346
|
+
soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
|
|
347
|
+
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
348
|
+
z.strictObject({
|
|
349
|
+
op: z.literal("judge"),
|
|
350
|
+
question: nonEmpty.describe("A yes/no question about the graded turn or value."),
|
|
351
|
+
options: judgeCallDataSchema.optional()
|
|
352
|
+
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
353
|
+
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
354
|
+
const attemptErrorSchema = z.looseObject({
|
|
355
|
+
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
356
|
+
message: z.string()
|
|
357
|
+
}).describe("Why a call failed, in reeval's words.");
|
|
358
|
+
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
359
|
+
const attemptRequestSchema = z.strictObject({
|
|
360
|
+
id,
|
|
361
|
+
call: attemptCallSchema
|
|
362
|
+
}).describe("A call, numbered so its reply can find it.");
|
|
363
|
+
z.union([
|
|
364
|
+
z.discriminatedUnion("op", [
|
|
365
|
+
z.strictObject({
|
|
366
|
+
id,
|
|
367
|
+
ok: z.literal(true),
|
|
368
|
+
op: z.literal("send"),
|
|
369
|
+
turn: turnDataSchema
|
|
370
|
+
}),
|
|
371
|
+
z.strictObject({
|
|
372
|
+
id,
|
|
373
|
+
ok: z.literal(true),
|
|
374
|
+
op: z.literal("exec"),
|
|
375
|
+
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
376
|
+
}),
|
|
377
|
+
z.strictObject({
|
|
378
|
+
id,
|
|
379
|
+
ok: z.literal(true),
|
|
380
|
+
op: z.literal("serve"),
|
|
381
|
+
handle: handleId,
|
|
382
|
+
port: z.number().describe("The container port it was given.")
|
|
383
|
+
}),
|
|
384
|
+
z.strictObject({
|
|
385
|
+
id,
|
|
386
|
+
ok: z.literal(true),
|
|
387
|
+
op: z.literal("kill")
|
|
388
|
+
}),
|
|
389
|
+
z.strictObject({
|
|
390
|
+
id,
|
|
391
|
+
ok: z.literal(true),
|
|
392
|
+
op: z.literal("capture"),
|
|
393
|
+
capture: captureSchema
|
|
394
|
+
}),
|
|
395
|
+
z.strictObject({
|
|
396
|
+
id,
|
|
397
|
+
ok: z.literal(true),
|
|
398
|
+
op: z.literal("readFile"),
|
|
399
|
+
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
400
|
+
}),
|
|
401
|
+
z.strictObject({
|
|
402
|
+
id,
|
|
403
|
+
ok: z.literal(true),
|
|
404
|
+
op: z.literal("writeFile")
|
|
405
|
+
}),
|
|
406
|
+
z.strictObject({
|
|
407
|
+
id,
|
|
408
|
+
ok: z.literal(true),
|
|
409
|
+
op: z.literal("expect"),
|
|
410
|
+
assertion: assertionRecordSchema
|
|
411
|
+
}),
|
|
412
|
+
z.strictObject({
|
|
413
|
+
id,
|
|
414
|
+
ok: z.literal(true),
|
|
415
|
+
op: z.literal("record"),
|
|
416
|
+
assertion: assertionRecordSchema
|
|
417
|
+
}),
|
|
418
|
+
z.strictObject({
|
|
419
|
+
id,
|
|
420
|
+
ok: z.literal(true),
|
|
421
|
+
op: z.literal("judge"),
|
|
422
|
+
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
423
|
+
})
|
|
424
|
+
]),
|
|
425
|
+
z.strictObject({
|
|
426
|
+
id,
|
|
427
|
+
ok: z.literal(true),
|
|
428
|
+
op: z.literal("expect"),
|
|
429
|
+
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
430
|
+
}),
|
|
431
|
+
z.strictObject({
|
|
432
|
+
id,
|
|
433
|
+
ok: z.literal(false),
|
|
434
|
+
op: z.enum([
|
|
435
|
+
"send",
|
|
436
|
+
"exec",
|
|
437
|
+
"readFile",
|
|
438
|
+
"writeFile",
|
|
439
|
+
"expect",
|
|
440
|
+
"record",
|
|
441
|
+
"judge",
|
|
442
|
+
"serve",
|
|
443
|
+
"kill",
|
|
444
|
+
"capture"
|
|
445
|
+
]),
|
|
446
|
+
error: attemptErrorSchema
|
|
447
|
+
})
|
|
448
|
+
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
449
|
+
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
450
|
+
const cellSchema = z.strictObject({
|
|
451
|
+
key: z.string(),
|
|
452
|
+
harness: z.string(),
|
|
453
|
+
environment: z.string(),
|
|
454
|
+
sandbox: z.string(),
|
|
455
|
+
role: cellRoleSchema,
|
|
456
|
+
variant: z.string().optional()
|
|
457
|
+
});
|
|
458
|
+
z.strictObject({
|
|
459
|
+
type: z.literal("start"),
|
|
460
|
+
attempt: z.strictObject({
|
|
461
|
+
id: nonEmpty,
|
|
462
|
+
cell: cellSchema,
|
|
463
|
+
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
464
|
+
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
465
|
+
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
466
|
+
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
467
|
+
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
468
|
+
z.strictObject({
|
|
469
|
+
type: z.literal("abort"),
|
|
470
|
+
reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
|
|
471
|
+
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
472
|
+
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
473
|
+
z.strictObject({
|
|
474
|
+
type: z.literal("done"),
|
|
475
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
|
|
476
|
+
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
477
|
+
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
478
|
+
const OTHER_KEYS = [
|
|
479
|
+
"id",
|
|
480
|
+
"call",
|
|
481
|
+
"turn",
|
|
482
|
+
"attempt",
|
|
483
|
+
"value",
|
|
484
|
+
"regexp",
|
|
485
|
+
"flags",
|
|
486
|
+
"matchedBy",
|
|
487
|
+
"answers",
|
|
488
|
+
...Object.keys(judgeCallDataSchema.shape),
|
|
489
|
+
...Object.keys(judgeStateSchema.shape),
|
|
490
|
+
...Object.keys(captureSchema.shape),
|
|
491
|
+
...Object.keys(captureCallDataSchema.shape),
|
|
492
|
+
"port",
|
|
493
|
+
"handle",
|
|
494
|
+
"exec",
|
|
495
|
+
"result",
|
|
496
|
+
"readFile",
|
|
497
|
+
"interval",
|
|
498
|
+
"timeout",
|
|
499
|
+
"ms"
|
|
500
|
+
];
|
|
501
|
+
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
502
|
+
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
503
|
+
//#endregion
|
|
504
|
+
//#region src/capture/fake-instructions.ts
|
|
505
|
+
const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
|
|
506
|
+
function isWaitStep(step) {
|
|
507
|
+
return typeof step === "object" && step !== null && !Array.isArray(step) && "wait" in step;
|
|
329
508
|
}
|
|
330
|
-
|
|
331
|
-
|
|
509
|
+
/** One list: action lists and `{ wait }` inlined. Returns branded `instructions`. */
|
|
510
|
+
function fakeInstructions(steps) {
|
|
511
|
+
if (!Array.isArray(steps)) throw new Error(`fakeInstructions() is an array of action lists and { wait } (got ${JSON.stringify(steps)})`);
|
|
512
|
+
const instructions = [];
|
|
513
|
+
let goals = 0;
|
|
514
|
+
for (const [i, step] of steps.entries()) {
|
|
515
|
+
if (isWaitStep(step)) {
|
|
516
|
+
instructions.push({ wait: step.wait });
|
|
517
|
+
continue;
|
|
518
|
+
}
|
|
519
|
+
if (!Array.isArray(step)) throw new Error(`fakeInstructions()[${i}] is an action list or { wait } (got ${JSON.stringify(step)})`);
|
|
520
|
+
instructions.push(`fake[${goals}]`);
|
|
521
|
+
goals++;
|
|
522
|
+
}
|
|
523
|
+
Object.defineProperty(instructions, FAKE_INSTRUCTIONS, {
|
|
524
|
+
value: steps,
|
|
525
|
+
enumerable: false
|
|
526
|
+
});
|
|
527
|
+
return instructions;
|
|
332
528
|
}
|
|
333
|
-
function
|
|
334
|
-
|
|
529
|
+
function fakeInstructionSteps(value) {
|
|
530
|
+
if (typeof value === "object" && value !== null && FAKE_INSTRUCTIONS in value) return value[FAKE_INSTRUCTIONS];
|
|
335
531
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
chunks.push(value);
|
|
357
|
-
received += value.length;
|
|
358
|
-
if (received - last >= 262144 || total && received === total) {
|
|
359
|
-
report(received);
|
|
360
|
-
last = received;
|
|
532
|
+
//#endregion
|
|
533
|
+
//#region src/capture/zip.ts
|
|
534
|
+
const LOCAL = 67324752;
|
|
535
|
+
/** Extracts a zip (store or deflate) into `files` keyed by posix path. */
|
|
536
|
+
function unzip(bytes) {
|
|
537
|
+
const buf = Buffer.from(bytes);
|
|
538
|
+
const files = /* @__PURE__ */ new Map();
|
|
539
|
+
let i = 0;
|
|
540
|
+
while (i + 30 <= buf.length) {
|
|
541
|
+
if (buf.readUInt32LE(i) !== LOCAL) break;
|
|
542
|
+
const method = buf.readUInt16LE(i + 8);
|
|
543
|
+
const comp = buf.readUInt32LE(i + 18);
|
|
544
|
+
const nameLen = buf.readUInt16LE(i + 26);
|
|
545
|
+
const extra = buf.readUInt16LE(i + 28);
|
|
546
|
+
const name = buf.toString("utf8", i + 30, i + 30 + nameLen);
|
|
547
|
+
const start = i + 30 + nameLen + extra;
|
|
548
|
+
const raw = buf.subarray(start, start + comp);
|
|
549
|
+
if (name.endsWith("/")) {
|
|
550
|
+
i = start + comp;
|
|
551
|
+
continue;
|
|
361
552
|
}
|
|
553
|
+
if (method === 0) files.set(name, Buffer.from(raw));
|
|
554
|
+
else if (method === 8) files.set(name, Buffer.from(inflateRawSync(raw)));
|
|
555
|
+
else throw new Error(`zip method ${method} is not supported (${name})`);
|
|
556
|
+
i = start + comp;
|
|
362
557
|
}
|
|
363
|
-
|
|
364
|
-
return Buffer.concat(chunks);
|
|
558
|
+
return files;
|
|
365
559
|
}
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
throw new Error(`zip has no ${binaryName()} (saw ${names || "nothing"})`);
|
|
373
|
-
}
|
|
374
|
-
const prefix = entry[0].slice(0, -binaryName().length);
|
|
375
|
-
mkdirSync(dest, { recursive: true });
|
|
376
|
-
for (const [name, data] of files) {
|
|
377
|
-
const rel = prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
|
|
378
|
-
if (!rel || rel.endsWith("/")) continue;
|
|
379
|
-
const target = join(dest, ...rel.split("/"));
|
|
380
|
-
mkdirSync(join(target, ".."), { recursive: true });
|
|
381
|
-
writeFileSync(target, data);
|
|
382
|
-
}
|
|
383
|
-
const bin = join(dest, binaryName());
|
|
384
|
-
if (!existsSync(bin)) writeFileSync(bin, entry[1]);
|
|
385
|
-
if (process.platform !== "win32") chmodSync(bin, 493);
|
|
560
|
+
//#endregion
|
|
561
|
+
//#region src/capture/browser.ts
|
|
562
|
+
const isMs = (wait) => typeof wait === "object" && wait !== null;
|
|
563
|
+
/** The CDP event `'idle'` waits for; `'load'` is `Page.loadEventFired`. */
|
|
564
|
+
function captureWaitLabel(wait) {
|
|
565
|
+
return wait === "idle" ? "networkAlmostIdle" : "load";
|
|
386
566
|
}
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
*/
|
|
390
|
-
async function ensureChrome(cacheRoot, options = { cacheLabel: cacheRoot }) {
|
|
391
|
-
const platform = chromePlatform();
|
|
392
|
-
const dest = join(cacheRoot, CHROME_NAME, CHROME_VERSION, platform);
|
|
393
|
-
const bin = join(dest, binaryName());
|
|
394
|
-
if (existsSync(bin)) return bin;
|
|
395
|
-
mkdirSync(join(cacheRoot, CHROME_NAME, CHROME_VERSION), { recursive: true });
|
|
396
|
-
const partial = `${dest}.${process.pid}.partial`;
|
|
397
|
-
rmSync(partial, {
|
|
398
|
-
recursive: true,
|
|
399
|
-
force: true
|
|
400
|
-
});
|
|
401
|
-
try {
|
|
402
|
-
extractBinary(await downloadZip(chromeUrl(platform), options.onProgress, options.cacheLabel, options.signal), partial, platform);
|
|
403
|
-
try {
|
|
404
|
-
renameSync(partial, dest);
|
|
405
|
-
} catch (error) {
|
|
406
|
-
rmSync(partial, {
|
|
407
|
-
recursive: true,
|
|
408
|
-
force: true
|
|
409
|
-
});
|
|
410
|
-
if (!existsSync(bin)) throw error;
|
|
411
|
-
}
|
|
412
|
-
} catch (error) {
|
|
413
|
-
rmSync(partial, {
|
|
414
|
-
recursive: true,
|
|
415
|
-
force: true
|
|
416
|
-
});
|
|
417
|
-
if (existsSync(bin)) return bin;
|
|
418
|
-
throw error;
|
|
419
|
-
}
|
|
420
|
-
return bin;
|
|
567
|
+
function captureWaitTimeout(wait) {
|
|
568
|
+
return /* @__PURE__ */ new Error(`c.capture() timed out waiting for ${captureWaitLabel(wait)}`);
|
|
421
569
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
const p = this.#pending.get(msg.id);
|
|
433
|
-
if (!p) return;
|
|
434
|
-
this.#pending.delete(msg.id);
|
|
435
|
-
if (msg.error) p.reject(new Error(msg.error.message));
|
|
436
|
-
else p.resolve(msg.result);
|
|
437
|
-
return;
|
|
438
|
-
}
|
|
439
|
-
if (msg.method) for (const fn of this.#events.get(msg.method) ?? []) fn(msg);
|
|
440
|
-
});
|
|
570
|
+
function waitTimeout(wait, detail) {
|
|
571
|
+
if (isMs(wait)) return /* @__PURE__ */ new Error(`c.capture() timed out${detail ? ` on ${detail}` : ""} waiting ${wait.ms}ms`);
|
|
572
|
+
return captureWaitTimeout(wait);
|
|
573
|
+
}
|
|
574
|
+
async function waitForPage(cdp, wait, timeoutMs, nav) {
|
|
575
|
+
if (isMs(wait)) {
|
|
576
|
+
if (timeoutMs <= 0) throw waitTimeout(wait);
|
|
577
|
+
const ms = Math.min(wait.ms, timeoutMs);
|
|
578
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
579
|
+
return;
|
|
441
580
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
581
|
+
const page = listenForCaptureWait(cdp, wait);
|
|
582
|
+
page.arm();
|
|
583
|
+
await page.ready(nav, timeoutMs);
|
|
584
|
+
}
|
|
585
|
+
const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
|
|
586
|
+
/** Map a full-page click onto the live window, scrolling with the wheel if needed. */
|
|
587
|
+
function mapPagePoint(x, y, window, page, scrollY) {
|
|
588
|
+
const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
|
|
589
|
+
const target = y >= window.height || y < 0 ? clamp(y - window.height / 2, 0, maxY) : scrollY;
|
|
590
|
+
const nextY = clamp(target, 0, maxY);
|
|
591
|
+
return {
|
|
592
|
+
x: clamp(x, 0, window.width - 1),
|
|
593
|
+
y: clamp(y - nextY, 0, window.height - 1),
|
|
594
|
+
scrollY: nextY,
|
|
595
|
+
dy: nextY - scrollY
|
|
596
|
+
};
|
|
597
|
+
}
|
|
598
|
+
async function dispatchInput(cdp, action, window, page, scrollY = 0) {
|
|
599
|
+
if (action.type === "done") return scrollY;
|
|
600
|
+
if (action.type === "type") {
|
|
601
|
+
await cdp.send("Input.insertText", { text: action.text });
|
|
602
|
+
return scrollY;
|
|
446
603
|
}
|
|
447
|
-
|
|
448
|
-
const
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
604
|
+
if (action.type === "key") {
|
|
605
|
+
const key = action.key;
|
|
606
|
+
const vk = key === "Tab" ? 9 : 13;
|
|
607
|
+
await cdp.send("Input.dispatchKeyEvent", {
|
|
608
|
+
type: "keyDown",
|
|
609
|
+
key,
|
|
610
|
+
code: key,
|
|
611
|
+
windowsVirtualKeyCode: vk
|
|
612
|
+
});
|
|
613
|
+
await cdp.send("Input.dispatchKeyEvent", {
|
|
614
|
+
type: "keyUp",
|
|
615
|
+
key,
|
|
616
|
+
code: key,
|
|
617
|
+
windowsVirtualKeyCode: vk
|
|
618
|
+
});
|
|
619
|
+
return scrollY;
|
|
458
620
|
}
|
|
459
|
-
|
|
460
|
-
|
|
621
|
+
if (action.type === "scroll") {
|
|
622
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
623
|
+
type: "mouseWheel",
|
|
624
|
+
x: action.x,
|
|
625
|
+
y: action.y,
|
|
626
|
+
deltaX: 0,
|
|
627
|
+
deltaY: action.dy
|
|
628
|
+
});
|
|
629
|
+
return scrollY + action.dy;
|
|
461
630
|
}
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
const port = Number(line);
|
|
470
|
-
if (port > 0) {
|
|
471
|
-
resolve(port);
|
|
472
|
-
return;
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
if (Date.now() > deadline) {
|
|
476
|
-
reject(/* @__PURE__ */ new Error("chrome-headless-shell did not open a DevTools port"));
|
|
477
|
-
return;
|
|
478
|
-
}
|
|
479
|
-
setTimeout(tick, 50);
|
|
480
|
-
};
|
|
481
|
-
tick();
|
|
631
|
+
const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
|
|
632
|
+
if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
|
|
633
|
+
type: "mouseWheel",
|
|
634
|
+
x: window.width / 2,
|
|
635
|
+
y: window.height / 2,
|
|
636
|
+
deltaX: 0,
|
|
637
|
+
deltaY: mapped.dy
|
|
482
638
|
});
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
}
|
|
487
|
-
async function pageDebuggerUrl(port, timeoutMs) {
|
|
488
|
-
const deadline = Date.now() + timeoutMs;
|
|
489
|
-
const listPages = async () => {
|
|
490
|
-
const res = await fetch(`http://127.0.0.1:${port}/json/list`);
|
|
491
|
-
if (!res.ok) return void 0;
|
|
492
|
-
return pickPageDebuggerUrl(await res.json());
|
|
639
|
+
const at = {
|
|
640
|
+
x: mapped.x,
|
|
641
|
+
y: mapped.y
|
|
493
642
|
};
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
|
|
501
|
-
if (!version.webSocketDebuggerUrl) throw new Error("chrome-headless-shell reported no DevTools URL");
|
|
502
|
-
const browser = await cdpConnect(version.webSocketDebuggerUrl, timeoutMs);
|
|
503
|
-
try {
|
|
504
|
-
await browser.send("Target.createTarget", { url: "about:blank" });
|
|
505
|
-
} finally {
|
|
506
|
-
browser.close();
|
|
643
|
+
if (action.type === "hover") {
|
|
644
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
645
|
+
type: "mouseMoved",
|
|
646
|
+
...at
|
|
647
|
+
});
|
|
648
|
+
return mapped.scrollY;
|
|
507
649
|
}
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
const ws = new WebSocket(url);
|
|
514
|
-
await new Promise((resolve, reject) => {
|
|
515
|
-
const t = setTimeout(() => reject(/* @__PURE__ */ new Error("CDP websocket timed out")), timeoutMs);
|
|
516
|
-
ws.addEventListener("open", () => {
|
|
517
|
-
clearTimeout(t);
|
|
518
|
-
resolve();
|
|
650
|
+
if (action.type === "click") {
|
|
651
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
652
|
+
type: "mouseMoved",
|
|
653
|
+
...at,
|
|
654
|
+
button: "left"
|
|
519
655
|
});
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
656
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
657
|
+
type: "mousePressed",
|
|
658
|
+
...at,
|
|
659
|
+
button: "left",
|
|
660
|
+
clickCount: 1
|
|
661
|
+
});
|
|
662
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
663
|
+
type: "mouseReleased",
|
|
664
|
+
...at,
|
|
665
|
+
button: "left",
|
|
666
|
+
clickCount: 1
|
|
523
667
|
});
|
|
668
|
+
return mapped.scrollY;
|
|
669
|
+
}
|
|
670
|
+
const end = mapPagePoint(action.toX, action.toY, window, page, mapped.scrollY);
|
|
671
|
+
if (end.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
|
|
672
|
+
type: "mouseWheel",
|
|
673
|
+
x: window.width / 2,
|
|
674
|
+
y: window.height / 2,
|
|
675
|
+
deltaX: 0,
|
|
676
|
+
deltaY: end.dy
|
|
524
677
|
});
|
|
525
|
-
|
|
678
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
679
|
+
type: "mouseMoved",
|
|
680
|
+
x: at.x,
|
|
681
|
+
y: at.y,
|
|
682
|
+
button: "left"
|
|
683
|
+
});
|
|
684
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
685
|
+
type: "mousePressed",
|
|
686
|
+
x: at.x,
|
|
687
|
+
y: at.y,
|
|
688
|
+
button: "left",
|
|
689
|
+
clickCount: 1
|
|
690
|
+
});
|
|
691
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
692
|
+
type: "mouseMoved",
|
|
693
|
+
x: end.x,
|
|
694
|
+
y: end.y,
|
|
695
|
+
button: "left"
|
|
696
|
+
});
|
|
697
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
698
|
+
type: "mouseReleased",
|
|
699
|
+
x: end.x,
|
|
700
|
+
y: end.y,
|
|
701
|
+
button: "left",
|
|
702
|
+
clickCount: 1
|
|
703
|
+
});
|
|
704
|
+
return end.scrollY;
|
|
705
|
+
}
|
|
706
|
+
function lifecycleParams(msg) {
|
|
707
|
+
const body = msg;
|
|
708
|
+
return body.params ?? body;
|
|
526
709
|
}
|
|
527
710
|
/**
|
|
528
|
-
*
|
|
529
|
-
*
|
|
711
|
+
* Arms after `Page.enable` so about:blank's events cannot resolve the wait.
|
|
712
|
+
* Call `arm` immediately before `Page.navigate`.
|
|
530
713
|
*/
|
|
531
|
-
function
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
714
|
+
function listenForCaptureWait(cdp, wait) {
|
|
715
|
+
const method = wait === "idle" ? "Page.lifecycleEvent" : "Page.loadEventFired";
|
|
716
|
+
const pending = [];
|
|
717
|
+
let deliver;
|
|
718
|
+
let started = false;
|
|
719
|
+
cdp.on(method, (msg) => {
|
|
720
|
+
if (!started) return;
|
|
721
|
+
if (deliver) deliver(msg);
|
|
722
|
+
else pending.push(msg);
|
|
723
|
+
});
|
|
724
|
+
const matches = (msg, loaderId) => {
|
|
725
|
+
if (wait === "load") return true;
|
|
726
|
+
const params = lifecycleParams(msg);
|
|
727
|
+
if (params.name !== "networkAlmostIdle") return false;
|
|
728
|
+
return !loaderId || params.loaderId === loaderId;
|
|
729
|
+
};
|
|
730
|
+
return {
|
|
731
|
+
arm() {
|
|
732
|
+
started = true;
|
|
733
|
+
},
|
|
734
|
+
ready(nav, timeoutMs) {
|
|
735
|
+
return new Promise((resolve, reject) => {
|
|
736
|
+
const timer = setTimeout(() => reject(captureWaitTimeout(wait)), timeoutMs);
|
|
737
|
+
const finish = () => {
|
|
738
|
+
clearTimeout(timer);
|
|
739
|
+
resolve();
|
|
740
|
+
};
|
|
741
|
+
deliver = (msg) => {
|
|
742
|
+
if (matches(msg, nav?.loaderId)) finish();
|
|
743
|
+
};
|
|
744
|
+
if (pending.some((msg) => matches(msg, nav?.loaderId))) finish();
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
const QUIET_IN_FLIGHT = 2;
|
|
750
|
+
function requestId(msg) {
|
|
751
|
+
const body = msg;
|
|
752
|
+
return (body.params ?? body).requestId;
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* Counts the tab's requests from `Network.enable` on, for a journey's `{ wait: 'idle' }` beat.
|
|
756
|
+
* Chromium emits `networkAlmostIdle` once per document, so after a click that only fetches it
|
|
757
|
+
* never comes again. As in Chromium, every request that starts or ends restarts the quiet window.
|
|
758
|
+
*/
|
|
759
|
+
function trackNetwork(cdp, quietMs = 500) {
|
|
760
|
+
const inFlight = /* @__PURE__ */ new Set();
|
|
761
|
+
let changed;
|
|
762
|
+
cdp.on("Network.requestWillBeSent", (msg) => {
|
|
763
|
+
const id = requestId(msg);
|
|
764
|
+
if (id === void 0) return;
|
|
765
|
+
inFlight.add(id);
|
|
766
|
+
changed?.();
|
|
767
|
+
});
|
|
768
|
+
const end = (msg) => {
|
|
769
|
+
const id = requestId(msg);
|
|
770
|
+
if (id === void 0 || !inFlight.delete(id)) return;
|
|
771
|
+
changed?.();
|
|
772
|
+
};
|
|
773
|
+
cdp.on("Network.loadingFinished", end);
|
|
774
|
+
cdp.on("Network.loadingFailed", end);
|
|
775
|
+
return { quiet(timeoutMs) {
|
|
776
|
+
return new Promise((resolve, reject) => {
|
|
777
|
+
let settle;
|
|
778
|
+
const stop = () => {
|
|
779
|
+
clearTimeout(timer);
|
|
780
|
+
clearTimeout(settle);
|
|
781
|
+
changed = void 0;
|
|
782
|
+
};
|
|
783
|
+
const timer = setTimeout(() => {
|
|
784
|
+
stop();
|
|
785
|
+
reject(captureWaitTimeout("idle"));
|
|
786
|
+
}, timeoutMs);
|
|
787
|
+
changed = () => {
|
|
788
|
+
clearTimeout(settle);
|
|
789
|
+
if (inFlight.size > QUIET_IN_FLIGHT) return;
|
|
790
|
+
settle = setTimeout(() => {
|
|
791
|
+
stop();
|
|
792
|
+
resolve();
|
|
793
|
+
}, quietMs);
|
|
794
|
+
};
|
|
795
|
+
changed();
|
|
538
796
|
});
|
|
539
|
-
}
|
|
540
|
-
scheduleChromeProfileSweep(dir, rm);
|
|
541
|
-
}
|
|
797
|
+
} };
|
|
542
798
|
}
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
retryDelay: 50
|
|
551
|
-
});
|
|
552
|
-
} catch {}
|
|
553
|
-
}, 250).unref();
|
|
799
|
+
/** Pinned Chrome for Testing `chrome-headless-shell` build. */
|
|
800
|
+
const CHROME_VERSION = "140.0.7339.185";
|
|
801
|
+
const CHROME_NAME = "chrome-headless-shell";
|
|
802
|
+
function chromePlatform() {
|
|
803
|
+
if (process.platform === "darwin") return process.arch === "arm64" ? "mac-arm64" : "mac-x64";
|
|
804
|
+
if (process.platform === "win32") return "win64";
|
|
805
|
+
return "linux64";
|
|
554
806
|
}
|
|
555
|
-
function
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
807
|
+
function chromeUrl(platform) {
|
|
808
|
+
return `https://storage.googleapis.com/chrome-for-testing-public/${CHROME_VERSION}/${platform}/${CHROME_NAME}-${platform}.zip`;
|
|
809
|
+
}
|
|
810
|
+
function binaryName() {
|
|
811
|
+
return process.platform === "win32" ? `${CHROME_NAME}.exe` : CHROME_NAME;
|
|
812
|
+
}
|
|
813
|
+
async function downloadZip(url, onProgress, cacheLabel, signal) {
|
|
814
|
+
const res = await fetch(url, { signal });
|
|
815
|
+
if (!res.ok) throw new Error(`download failed: HTTP ${res.status}`);
|
|
816
|
+
const total = Number(res.headers.get("content-length")) || null;
|
|
817
|
+
const report = (received) => onProgress?.({
|
|
818
|
+
name: CHROME_NAME,
|
|
819
|
+
version: CHROME_VERSION,
|
|
820
|
+
bytes: total,
|
|
821
|
+
received,
|
|
822
|
+
cache: cacheLabel
|
|
565
823
|
});
|
|
824
|
+
report(0);
|
|
825
|
+
if (!res.body) return Buffer.from(await res.arrayBuffer());
|
|
826
|
+
const reader = res.body.getReader();
|
|
827
|
+
const chunks = [];
|
|
828
|
+
let received = 0;
|
|
829
|
+
let last = 0;
|
|
830
|
+
for (;;) {
|
|
831
|
+
const { done, value } = await reader.read();
|
|
832
|
+
if (done) break;
|
|
833
|
+
chunks.push(value);
|
|
834
|
+
received += value.length;
|
|
835
|
+
if (received - last >= 262144 || total && received === total) {
|
|
836
|
+
report(received);
|
|
837
|
+
last = received;
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
if (received !== last) report(received);
|
|
841
|
+
return Buffer.concat(chunks);
|
|
566
842
|
}
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
843
|
+
function extractBinary(zip, dest, platform) {
|
|
844
|
+
const files = unzip(zip);
|
|
845
|
+
const needle = `/${binaryName()}`;
|
|
846
|
+
const entry = [...files.entries()].find(([name]) => name.endsWith(needle) || name === binaryName() || name.endsWith(`\\${binaryName()}`));
|
|
847
|
+
if (!entry) {
|
|
848
|
+
const names = [...files.keys()].slice(0, 8).join(", ");
|
|
849
|
+
throw new Error(`zip has no ${binaryName()} (saw ${names || "nothing"})`);
|
|
850
|
+
}
|
|
851
|
+
const prefix = entry[0].slice(0, -binaryName().length);
|
|
852
|
+
mkdirSync(dest, { recursive: true });
|
|
853
|
+
for (const [name, data] of files) {
|
|
854
|
+
const rel = prefix && name.startsWith(prefix) ? name.slice(prefix.length) : name;
|
|
855
|
+
if (!rel || rel.endsWith("/")) continue;
|
|
856
|
+
const target = join(dest, ...rel.split("/"));
|
|
857
|
+
mkdirSync(join(target, ".."), { recursive: true });
|
|
858
|
+
writeFileSync(target, data);
|
|
859
|
+
}
|
|
860
|
+
const bin = join(dest, binaryName());
|
|
861
|
+
if (!existsSync(bin)) writeFileSync(bin, entry[1]);
|
|
862
|
+
if (process.platform !== "win32") chmodSync(bin, 493);
|
|
574
863
|
}
|
|
575
|
-
/**
|
|
864
|
+
/**
|
|
865
|
+
* Returns the cached binary, downloading on a miss. `onProgress` runs only for a download.
|
|
866
|
+
*/
|
|
867
|
+
async function ensureChrome(cacheRoot, options = { cacheLabel: cacheRoot }) {
|
|
868
|
+
const platform = chromePlatform();
|
|
869
|
+
const dest = join(cacheRoot, CHROME_NAME, CHROME_VERSION, platform);
|
|
870
|
+
const bin = join(dest, binaryName());
|
|
871
|
+
if (existsSync(bin)) return bin;
|
|
872
|
+
mkdirSync(join(cacheRoot, CHROME_NAME, CHROME_VERSION), { recursive: true });
|
|
873
|
+
const partial = `${dest}.${process.pid}.partial`;
|
|
874
|
+
rmSync(partial, {
|
|
875
|
+
recursive: true,
|
|
876
|
+
force: true
|
|
877
|
+
});
|
|
878
|
+
try {
|
|
879
|
+
extractBinary(await downloadZip(chromeUrl(platform), options.onProgress, options.cacheLabel, options.signal), partial, platform);
|
|
880
|
+
try {
|
|
881
|
+
renameSync(partial, dest);
|
|
882
|
+
} catch (error) {
|
|
883
|
+
rmSync(partial, {
|
|
884
|
+
recursive: true,
|
|
885
|
+
force: true
|
|
886
|
+
});
|
|
887
|
+
if (!existsSync(bin)) throw error;
|
|
888
|
+
}
|
|
889
|
+
} catch (error) {
|
|
890
|
+
rmSync(partial, {
|
|
891
|
+
recursive: true,
|
|
892
|
+
force: true
|
|
893
|
+
});
|
|
894
|
+
if (existsSync(bin)) return bin;
|
|
895
|
+
throw error;
|
|
896
|
+
}
|
|
897
|
+
return bin;
|
|
898
|
+
}
|
|
899
|
+
var Cdp = class {
|
|
900
|
+
#ws;
|
|
901
|
+
#next = 0;
|
|
902
|
+
#pending = /* @__PURE__ */ new Map();
|
|
903
|
+
#events = /* @__PURE__ */ new Map();
|
|
904
|
+
constructor(ws) {
|
|
905
|
+
this.#ws = ws;
|
|
906
|
+
ws.addEventListener("message", (ev) => {
|
|
907
|
+
const msg = JSON.parse(String(ev.data));
|
|
908
|
+
if (msg.id !== void 0) {
|
|
909
|
+
const p = this.#pending.get(msg.id);
|
|
910
|
+
if (!p) return;
|
|
911
|
+
this.#pending.delete(msg.id);
|
|
912
|
+
if (msg.error) p.reject(new Error(msg.error.message));
|
|
913
|
+
else p.resolve(msg.result);
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
if (msg.method) for (const fn of this.#events.get(msg.method) ?? []) fn(msg);
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
on(method, fn) {
|
|
920
|
+
const list = this.#events.get(method) ?? [];
|
|
921
|
+
list.push(fn);
|
|
922
|
+
this.#events.set(method, list);
|
|
923
|
+
}
|
|
924
|
+
send(method, params) {
|
|
925
|
+
const id = ++this.#next;
|
|
926
|
+
this.#ws.send(JSON.stringify({
|
|
927
|
+
id,
|
|
928
|
+
method,
|
|
929
|
+
params
|
|
930
|
+
}));
|
|
931
|
+
return new Promise((resolve, reject) => this.#pending.set(id, {
|
|
932
|
+
resolve,
|
|
933
|
+
reject
|
|
934
|
+
}));
|
|
935
|
+
}
|
|
936
|
+
close() {
|
|
937
|
+
this.#ws.close();
|
|
938
|
+
}
|
|
939
|
+
};
|
|
940
|
+
function waitForPortFile(file, timeoutMs) {
|
|
941
|
+
const deadline = Date.now() + timeoutMs;
|
|
942
|
+
return new Promise((resolve, reject) => {
|
|
943
|
+
const tick = () => {
|
|
944
|
+
if (existsSync(file)) {
|
|
945
|
+
const line = readFileSync(file, "utf8").split(/\r?\n/)[0];
|
|
946
|
+
const port = Number(line);
|
|
947
|
+
if (port > 0) {
|
|
948
|
+
resolve(port);
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
if (Date.now() > deadline) {
|
|
953
|
+
reject(/* @__PURE__ */ new Error("chrome-headless-shell did not open a DevTools port"));
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
setTimeout(tick, 50);
|
|
957
|
+
};
|
|
958
|
+
tick();
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
function pickPageDebuggerUrl(targets) {
|
|
962
|
+
return targets.find((t) => t.webSocketDebuggerUrl && t.type !== "browser")?.webSocketDebuggerUrl;
|
|
963
|
+
}
|
|
964
|
+
async function pageDebuggerUrl(port, timeoutMs) {
|
|
965
|
+
const deadline = Date.now() + timeoutMs;
|
|
966
|
+
const listPages = async () => {
|
|
967
|
+
const res = await fetch(`http://127.0.0.1:${port}/json/list`);
|
|
968
|
+
if (!res.ok) return void 0;
|
|
969
|
+
return pickPageDebuggerUrl(await res.json());
|
|
970
|
+
};
|
|
971
|
+
for (;;) {
|
|
972
|
+
const url = await listPages();
|
|
973
|
+
if (url) return url;
|
|
974
|
+
if (Date.now() > deadline) break;
|
|
975
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
976
|
+
}
|
|
977
|
+
const version = await (await fetch(`http://127.0.0.1:${port}/json/version`)).json();
|
|
978
|
+
if (!version.webSocketDebuggerUrl) throw new Error("chrome-headless-shell reported no DevTools URL");
|
|
979
|
+
const browser = await cdpConnect(version.webSocketDebuggerUrl, timeoutMs);
|
|
980
|
+
try {
|
|
981
|
+
await browser.send("Target.createTarget", { url: "about:blank" });
|
|
982
|
+
} finally {
|
|
983
|
+
browser.close();
|
|
984
|
+
}
|
|
985
|
+
const created = await listPages();
|
|
986
|
+
if (!created) throw new Error("chrome-headless-shell reported no page target");
|
|
987
|
+
return created;
|
|
988
|
+
}
|
|
989
|
+
async function cdpConnect(url, timeoutMs) {
|
|
990
|
+
const ws = new WebSocket(url);
|
|
991
|
+
await new Promise((resolve, reject) => {
|
|
992
|
+
const t = setTimeout(() => reject(/* @__PURE__ */ new Error("CDP websocket timed out")), timeoutMs);
|
|
993
|
+
ws.addEventListener("open", () => {
|
|
994
|
+
clearTimeout(t);
|
|
995
|
+
resolve();
|
|
996
|
+
});
|
|
997
|
+
ws.addEventListener("error", () => {
|
|
998
|
+
clearTimeout(t);
|
|
999
|
+
reject(/* @__PURE__ */ new Error("CDP websocket failed"));
|
|
1000
|
+
});
|
|
1001
|
+
});
|
|
1002
|
+
return new Cdp(ws);
|
|
1003
|
+
}
|
|
1004
|
+
/**
|
|
1005
|
+
* Drops a chrome-headless-shell profile. Chrome can still write after SIGKILL,
|
|
1006
|
+
* so `ENOTEMPTY` / `EBUSY` must not replace a PNG already in hand.
|
|
1007
|
+
*/
|
|
1008
|
+
function removeChromeProfile(dir, rm = rmSync) {
|
|
1009
|
+
try {
|
|
1010
|
+
rm(dir, {
|
|
1011
|
+
recursive: true,
|
|
1012
|
+
force: true,
|
|
1013
|
+
maxRetries: 5,
|
|
1014
|
+
retryDelay: 20
|
|
1015
|
+
});
|
|
1016
|
+
} catch {
|
|
1017
|
+
scheduleChromeProfileSweep(dir, rm);
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
function scheduleChromeProfileSweep(dir, rm) {
|
|
1021
|
+
setTimeout(() => {
|
|
1022
|
+
try {
|
|
1023
|
+
rm(dir, {
|
|
1024
|
+
recursive: true,
|
|
1025
|
+
force: true,
|
|
1026
|
+
maxRetries: 5,
|
|
1027
|
+
retryDelay: 50
|
|
1028
|
+
});
|
|
1029
|
+
} catch {}
|
|
1030
|
+
}, 250).unref();
|
|
1031
|
+
}
|
|
1032
|
+
function waitForExit(child, timeoutMs) {
|
|
1033
|
+
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve();
|
|
1034
|
+
return new Promise((resolve) => {
|
|
1035
|
+
const finish = () => {
|
|
1036
|
+
clearTimeout(timer);
|
|
1037
|
+
child.off("exit", finish);
|
|
1038
|
+
resolve();
|
|
1039
|
+
};
|
|
1040
|
+
const timer = setTimeout(finish, timeoutMs);
|
|
1041
|
+
child.once("exit", finish);
|
|
1042
|
+
});
|
|
1043
|
+
}
|
|
1044
|
+
/** Kills the shell and drops its profile. Never throws: the PNG is already in hand. */
|
|
1045
|
+
async function disposeChrome(child, userData) {
|
|
1046
|
+
try {
|
|
1047
|
+
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
|
1048
|
+
await waitForExit(child, 1e3);
|
|
1049
|
+
} catch {}
|
|
1050
|
+
removeChromeProfile(userData);
|
|
1051
|
+
}
|
|
1052
|
+
/** One Chrome process, one tab. `fn` must not keep the session after it returns. */
|
|
576
1053
|
async function withChrome(bin, window, timeoutMs, fn) {
|
|
577
1054
|
const userData = mkdtempSync(join(tmpdir(), `reeval-chrome-${process.pid}-`));
|
|
578
1055
|
const child = spawn(bin, [
|
|
@@ -729,7 +1206,6 @@ const DEFAULT_VIEWPORT = {
|
|
|
729
1206
|
width: 1280,
|
|
730
1207
|
height: 720
|
|
731
1208
|
};
|
|
732
|
-
const CAPTURE_NAME = /^[a-z0-9_-]+$/;
|
|
733
1209
|
const CAPTURE_WAITS = ["load", "idle"];
|
|
734
1210
|
/** `'load'` when the call omits `wait`. `'load'`, `'idle'` or `{ ms }`. */
|
|
735
1211
|
function resolveWait(wait) {
|
|
@@ -788,9 +1264,12 @@ function resolveCaptureTarget(target) {
|
|
|
788
1264
|
if (target !== null && typeof target === "object" && typeof target.port === "number" && typeof target.kill === "function") return resolveCaptureTarget(target.port);
|
|
789
1265
|
throw new Error("c.capture() needs a port or a ServeHandle as the first argument");
|
|
790
1266
|
}
|
|
791
|
-
/**
|
|
1267
|
+
/**
|
|
1268
|
+
* Artifact name from a path: `/login` → `login`, `/foo/bar` → `foo-bar`, `/` → `root`. Always
|
|
1269
|
+
* a valid capture name: leading `_` and `-` go, and it's cut at the name's length limit.
|
|
1270
|
+
*/
|
|
792
1271
|
function nameFromPath(path) {
|
|
793
|
-
return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(
|
|
1272
|
+
return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^[_-]+/, "").slice(0, 100).replace(/-+$/, "") || "root";
|
|
794
1273
|
}
|
|
795
1274
|
function resolveCaptureCall(target, call, cascade) {
|
|
796
1275
|
if (call && Object.hasOwn(call, "port")) throw new Error("c.capture() takes the port as the first argument, not in the options");
|
|
@@ -831,7 +1310,7 @@ function checkCallModel(model) {
|
|
|
831
1310
|
if (model.startsWith("typesafe-ai/")) throw new Error(`c.capture() model "${model}" is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
|
|
832
1311
|
}
|
|
833
1312
|
function checkCaptureName(name) {
|
|
834
|
-
if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name
|
|
1313
|
+
if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name ${CAPTURE_NAME_RULE} (got ${JSON.stringify(name)})`);
|
|
835
1314
|
}
|
|
836
1315
|
/** A free host port. Closed before the promise resolves, so the caller can bind it. */
|
|
837
1316
|
function allocatePort() {
|
|
@@ -905,672 +1384,202 @@ function actionFromTool(name, input) {
|
|
|
905
1384
|
};
|
|
906
1385
|
}
|
|
907
1386
|
/**
|
|
908
|
-
* Gemini models answer a point
|
|
1387
|
+
* Gemini models answer a point normalized to 0–1000 across the image they were shown, whatever
|
|
909
1388
|
* the prompt asks, so their journeys are prompted in that scale and their points scaled back.
|
|
910
1389
|
*/
|
|
911
|
-
const
|
|
912
|
-
const
|
|
913
|
-
/** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
|
|
914
|
-
function scalePoints(action, page) {
|
|
915
|
-
const x = (n) => Math.round(n * page.width /
|
|
916
|
-
const y = (n) => Math.round(n * page.height /
|
|
917
|
-
switch (action.type) {
|
|
918
|
-
case "click":
|
|
919
|
-
case "hover":
|
|
920
|
-
case "scroll": return {
|
|
921
|
-
...action,
|
|
922
|
-
x: x(action.x),
|
|
923
|
-
y: y(action.y)
|
|
924
|
-
};
|
|
925
|
-
case "drag": return {
|
|
926
|
-
...action,
|
|
927
|
-
x: x(action.x),
|
|
928
|
-
y: y(action.y),
|
|
929
|
-
toX: x(action.toX),
|
|
930
|
-
toY: y(action.toY)
|
|
931
|
-
};
|
|
932
|
-
default: return action;
|
|
933
|
-
}
|
|
934
|
-
}
|
|
935
|
-
const systemPrompt = (
|
|
936
|
-
const useTools = {
|
|
937
|
-
click: tool({
|
|
938
|
-
description: "Click at (x, y) on the screenshot.",
|
|
939
|
-
inputSchema: z.object({
|
|
940
|
-
x: z.number(),
|
|
941
|
-
y: z.number()
|
|
942
|
-
})
|
|
943
|
-
}),
|
|
944
|
-
type: tool({
|
|
945
|
-
description: "Insert text into the focused field. Click the field first.",
|
|
946
|
-
inputSchema: z.object({ text: z.string() })
|
|
947
|
-
}),
|
|
948
|
-
key: tool({
|
|
949
|
-
description: "Press Tab or Enter.",
|
|
950
|
-
inputSchema: z.object({ key: z.enum(["Tab", "Enter"]) })
|
|
951
|
-
}),
|
|
952
|
-
scroll: tool({
|
|
953
|
-
description: "Wheel-scroll a constrained scroller at (x, y).",
|
|
954
|
-
inputSchema: z.object({
|
|
955
|
-
x: z.number(),
|
|
956
|
-
y: z.number(),
|
|
957
|
-
dy: z.number()
|
|
958
|
-
})
|
|
959
|
-
}),
|
|
960
|
-
drag: tool({
|
|
961
|
-
description: "Drag from (x, y) to (toX, toY).",
|
|
962
|
-
inputSchema: z.object({
|
|
963
|
-
x: z.number(),
|
|
964
|
-
y: z.number(),
|
|
965
|
-
toX: z.number(),
|
|
966
|
-
toY: z.number()
|
|
967
|
-
})
|
|
968
|
-
}),
|
|
969
|
-
hover: tool({
|
|
970
|
-
description: "Move the pointer to (x, y).",
|
|
971
|
-
inputSchema: z.object({
|
|
972
|
-
x: z.number(),
|
|
973
|
-
y: z.number()
|
|
974
|
-
})
|
|
975
|
-
}),
|
|
976
|
-
done: tool({
|
|
977
|
-
description: "The goal is finished. Leave the loop.",
|
|
978
|
-
inputSchema: z.object({})
|
|
979
|
-
})
|
|
980
|
-
};
|
|
981
|
-
async function runFakeScript(session, script, goal, remaining) {
|
|
982
|
-
let moves = 0;
|
|
983
|
-
for (const action of script) {
|
|
984
|
-
remainingOrThrow(remaining, JSON.stringify(goal));
|
|
985
|
-
if (action.type === "done") return moves;
|
|
986
|
-
if (moves >= 32) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
|
|
987
|
-
const page = pngSize(await session.screenshot(true));
|
|
988
|
-
debugTrace("check", "capture-use", {
|
|
989
|
-
goal,
|
|
990
|
-
action,
|
|
991
|
-
page
|
|
992
|
-
}, `capture use ${action.type}`);
|
|
993
|
-
await session.dispatch(action, page);
|
|
994
|
-
moves++;
|
|
995
|
-
await settle();
|
|
996
|
-
}
|
|
997
|
-
while (remaining() > 0) await settle();
|
|
998
|
-
throw new Error(`c.capture() timed out on ${JSON.stringify(goal)}`);
|
|
999
|
-
}
|
|
1000
|
-
async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
1001
|
-
const model = await loadModel(modelName);
|
|
1002
|
-
const normalised = normalisesPoints(modelName);
|
|
1003
|
-
let moves = 0;
|
|
1004
|
-
let usage = null;
|
|
1005
|
-
while (moves < 32) {
|
|
1006
|
-
remainingOrThrow(remaining, JSON.stringify(goal));
|
|
1007
|
-
const ephemeral = await session.screenshot(true);
|
|
1008
|
-
const page = pngSize(ephemeral);
|
|
1009
|
-
debugTrace("check", "capture-shot", {
|
|
1010
|
-
goal,
|
|
1011
|
-
bytes: ephemeral.length,
|
|
1012
|
-
page
|
|
1013
|
-
}, false);
|
|
1014
|
-
let chosen;
|
|
1015
|
-
const result = await generateText({
|
|
1016
|
-
model,
|
|
1017
|
-
system: systemPrompt(normalised),
|
|
1018
|
-
messages: [{
|
|
1019
|
-
role: "user",
|
|
1020
|
-
content: [{
|
|
1021
|
-
type: "text",
|
|
1022
|
-
text: goal
|
|
1023
|
-
}, {
|
|
1024
|
-
type: "image",
|
|
1025
|
-
image: ephemeral
|
|
1026
|
-
}]
|
|
1027
|
-
}],
|
|
1028
|
-
tools: useTools,
|
|
1029
|
-
toolChoice: "required",
|
|
1030
|
-
stopWhen: stepCountIs(1),
|
|
1031
|
-
abortSignal: AbortSignal.timeout(remainingOrThrow(remaining, JSON.stringify(goal)))
|
|
1032
|
-
});
|
|
1033
|
-
usage = addUsage(usage, asUsage(result.usage ?? {}));
|
|
1034
|
-
const call = result.toolCalls?.[0];
|
|
1035
|
-
if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
|
|
1036
|
-
if (chosen && normalised) chosen = scalePoints(chosen, page);
|
|
1037
|
-
if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
|
|
1038
|
-
debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
|
|
1039
|
-
debugTrace("check", "capture-use", {
|
|
1040
|
-
goal,
|
|
1041
|
-
action: chosen,
|
|
1042
|
-
page
|
|
1043
|
-
}, `capture use ${chosen.type}`);
|
|
1044
|
-
if (chosen.type === "done") return {
|
|
1045
|
-
moves,
|
|
1046
|
-
usage
|
|
1047
|
-
};
|
|
1048
|
-
await session.dispatch(chosen, page);
|
|
1049
|
-
moves++;
|
|
1050
|
-
await settle();
|
|
1051
|
-
}
|
|
1052
|
-
throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
|
|
1053
|
-
}
|
|
1054
|
-
/**
|
|
1055
|
-
* Top-level wait, then the instruction loop, then the saved PNG.
|
|
1056
|
-
* Throws before `screenshot` when the budget or move cap is hit.
|
|
1057
|
-
*/
|
|
1058
|
-
async function runJourney(session, url, options) {
|
|
1059
|
-
const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
|
|
1060
|
-
remainingOrThrow(remaining, "navigate");
|
|
1061
|
-
await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining()));
|
|
1062
|
-
let moves = 0;
|
|
1063
|
-
let usage = null;
|
|
1064
|
-
let stringIndex = 0;
|
|
1065
|
-
for (const step of instructions) {
|
|
1066
|
-
if (typeof step !== "string") {
|
|
1067
|
-
remainingOrThrow(remaining, "wait");
|
|
1068
|
-
await session.wait(step.wait, remaining(), void 0);
|
|
1069
|
-
continue;
|
|
1070
|
-
}
|
|
1071
|
-
remainingOrThrow(remaining, JSON.stringify(step));
|
|
1072
|
-
debugLog("check", `capture instruction ${JSON.stringify(step)}`);
|
|
1073
|
-
if (use && "fake" in use) {
|
|
1074
|
-
const script = use.fake[stringIndex] ?? [];
|
|
1075
|
-
moves += await runFakeScript(session, script, step, remaining);
|
|
1076
|
-
} else if (use && "model" in use) {
|
|
1077
|
-
if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
|
|
1078
|
-
const ran = await runModelGoal(session, step, use.model, remaining, options.loadModel);
|
|
1079
|
-
moves += ran.moves;
|
|
1080
|
-
usage = addUsage(usage, ran.usage);
|
|
1081
|
-
}
|
|
1082
|
-
stringIndex++;
|
|
1083
|
-
await settle();
|
|
1084
|
-
}
|
|
1085
|
-
remainingOrThrow(remaining, "screenshot");
|
|
1086
|
-
return {
|
|
1087
|
-
png: await session.screenshot(fullPage),
|
|
1088
|
-
moves,
|
|
1089
|
-
model: use && "model" in use ? use.model : use && "fake" in use ? "fake" : void 0,
|
|
1090
|
-
usage
|
|
1091
|
-
};
|
|
1092
|
-
}
|
|
1093
|
-
function captureUseSpend(result) {
|
|
1094
|
-
if (!result.model) return void 0;
|
|
1095
|
-
return {
|
|
1096
|
-
model: result.model,
|
|
1097
|
-
moves: result.moves,
|
|
1098
|
-
usage: result.usage
|
|
1099
|
-
};
|
|
1100
|
-
}
|
|
1101
|
-
//#endregion
|
|
1102
|
-
//#region src/contract/attempt.ts
|
|
1103
|
-
const nullableNumber = z.number().nullable();
|
|
1104
|
-
const tokenUsageSchema = z.strictObject({
|
|
1105
|
-
input: z.number(),
|
|
1106
|
-
output: z.number(),
|
|
1107
|
-
cacheRead: z.number(),
|
|
1108
|
-
cacheWrite: z.number(),
|
|
1109
|
-
reasoning: nullableNumber
|
|
1110
|
-
}).describe("The tokens one turn used, as its harness reported them. `input` excludes cache reads and writes (so the prompt was `input + cacheRead + cacheWrite`); `reasoning` is part of `output`.");
|
|
1111
|
-
const fileChangeSchema = z.strictObject({
|
|
1112
|
-
path: z.string(),
|
|
1113
|
-
status: z.enum([
|
|
1114
|
-
"added",
|
|
1115
|
-
"modified",
|
|
1116
|
-
"deleted"
|
|
1117
|
-
])
|
|
1118
|
-
});
|
|
1119
|
-
const diffSchema = z.strictObject({
|
|
1120
|
-
patch: z.string(),
|
|
1121
|
-
files: z.array(fileChangeSchema)
|
|
1122
|
-
});
|
|
1123
|
-
const turnStatusSchema = z.enum([
|
|
1124
|
-
"completed",
|
|
1125
|
-
"aborted",
|
|
1126
|
-
"error"
|
|
1127
|
-
]);
|
|
1128
|
-
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]).describe("The transcript events a harness may be unable to report.");
|
|
1129
|
-
const at = {
|
|
1130
|
-
ts: z.string(),
|
|
1131
|
-
atMs: z.number()
|
|
1132
|
-
};
|
|
1133
|
-
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
1134
|
-
z.strictObject({
|
|
1135
|
-
...at,
|
|
1136
|
-
type: z.enum(["text", "reasoning"]),
|
|
1137
|
-
text: z.string(),
|
|
1138
|
-
durationMs: nullableNumber
|
|
1139
|
-
}),
|
|
1140
|
-
z.strictObject({
|
|
1141
|
-
...at,
|
|
1142
|
-
type: z.literal("file-edit"),
|
|
1143
|
-
toolCallId: z.string(),
|
|
1144
|
-
path: z.string(),
|
|
1145
|
-
kind: z.enum([
|
|
1146
|
-
"create",
|
|
1147
|
-
"modify",
|
|
1148
|
-
"delete"
|
|
1149
|
-
]),
|
|
1150
|
-
added: nullableNumber.describe("Line counts, or null when the harness didn't send the content (e.g. deletes)."),
|
|
1151
|
-
removed: nullableNumber,
|
|
1152
|
-
durationMs: nullableNumber,
|
|
1153
|
-
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1154
|
-
}),
|
|
1155
|
-
z.strictObject({
|
|
1156
|
-
...at,
|
|
1157
|
-
type: z.literal("web-search"),
|
|
1158
|
-
toolCallId: z.string(),
|
|
1159
|
-
query: z.string(),
|
|
1160
|
-
references: nullableNumber.describe("How many sources the harness says it used; Cursor sends no URLs or titles."),
|
|
1161
|
-
ok: z.boolean().optional().describe("Whether the search succeeded. Absent means it did."),
|
|
1162
|
-
durationMs: nullableNumber,
|
|
1163
|
-
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1164
|
-
}),
|
|
1165
|
-
z.strictObject({
|
|
1166
|
-
...at,
|
|
1167
|
-
type: z.literal("web-fetch"),
|
|
1168
|
-
toolCallId: z.string(),
|
|
1169
|
-
url: z.string(),
|
|
1170
|
-
ok: z.boolean(),
|
|
1171
|
-
durationMs: nullableNumber,
|
|
1172
|
-
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1173
|
-
}),
|
|
1174
|
-
z.strictObject({
|
|
1175
|
-
...at,
|
|
1176
|
-
type: z.literal("tool"),
|
|
1177
|
-
toolCallId: z.string(),
|
|
1178
|
-
tool: z.string(),
|
|
1179
|
-
input: z.unknown(),
|
|
1180
|
-
output: z.unknown(),
|
|
1181
|
-
ok: z.boolean(),
|
|
1182
|
-
durationMs: nullableNumber,
|
|
1183
|
-
stalled: z.boolean().optional().describe("True when a shell-like tool or edit ran at least two minutes.")
|
|
1184
|
-
}),
|
|
1185
|
-
z.strictObject({
|
|
1186
|
-
...at,
|
|
1187
|
-
type: z.literal("error"),
|
|
1188
|
-
message: z.string()
|
|
1189
|
-
}),
|
|
1190
|
-
z.strictObject({
|
|
1191
|
-
...at,
|
|
1192
|
-
type: z.literal("retry"),
|
|
1193
|
-
message: z.string()
|
|
1194
|
-
})
|
|
1195
|
-
]).describe("Normalised agent events. Each tool call is one event, merged with its result; `durationMs` is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null when the harness never reported the end. Long shell or edit calls set `stalled: true`.");
|
|
1196
|
-
const turnDataSchema = z.strictObject({
|
|
1197
|
-
prompt: z.string(),
|
|
1198
|
-
text: z.string().describe("The agent's final message for this turn."),
|
|
1199
|
-
status: turnStatusSchema,
|
|
1200
|
-
startedAt: z.string(),
|
|
1201
|
-
durationMs: z.number(),
|
|
1202
|
-
transcript: z.array(transcriptEventSchema),
|
|
1203
|
-
usage: tokenUsageSchema.nullable().describe("Tokens the turn used; null when the harness reports none."),
|
|
1204
|
-
model: z.string().nullable().describe("The model the harness says served the turn, e.g. `composer-2.5[fast=false]`; null when it reports none. It can differ from the requested one (an alias, or parameters it filled in)."),
|
|
1205
|
-
unreported: z.array(webEventTypeSchema).describe("Event types the harness couldn't report for this turn: the agent may have searched or fetched with no event in `transcript`. `toHaveSearched()` and `toHaveFetched()` refuse such a turn. Usually `[]`; a Cursor turn whose web calls the adapter couldn't recover (its SDK drops them from the stream) is `['web-search', 'web-fetch']`."),
|
|
1206
|
-
diff: diffSchema.describe("Workspace changes since the eval started, as of the end of this turn.")
|
|
1207
|
-
}).describe("A turn as data, as a record or another language's runner carries it: `Turn` without its methods.");
|
|
1208
|
-
const execResultSchema = z.strictObject({
|
|
1209
|
-
exitCode: z.number().int(),
|
|
1210
|
-
stdout: z.string(),
|
|
1211
|
-
stderr: z.string()
|
|
1212
|
-
});
|
|
1213
|
-
const execRetrySchema = z.strictObject({
|
|
1214
|
-
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional().describe("Extra tries after a non-zero exit, a whole number from 0. Default 0."),
|
|
1215
|
-
interval: z.number().min(0, "must be 0 or more milliseconds").optional().describe("Milliseconds to wait before each extra try. Default 1000.")
|
|
1216
|
-
}).describe("How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient.");
|
|
1217
|
-
/** A background command has no exit for `retries` to wait on. */
|
|
1218
|
-
const noBackgroundRetry = (options, ctx) => {
|
|
1219
|
-
if (!options.background) return;
|
|
1220
|
-
for (const key of ["retries", "interval"]) {
|
|
1221
|
-
if (options[key] === void 0) continue;
|
|
1222
|
-
ctx.addIssue({
|
|
1223
|
-
code: "custom",
|
|
1224
|
-
path: [key],
|
|
1225
|
-
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
1226
|
-
});
|
|
1227
|
-
}
|
|
1228
|
-
};
|
|
1229
|
-
const execOptionsSchema = z.strictObject({
|
|
1230
|
-
background: z.boolean().optional(),
|
|
1231
|
-
...execRetrySchema.shape
|
|
1232
|
-
}).superRefine(noBackgroundRetry);
|
|
1233
|
-
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
1234
|
-
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
1235
|
-
const assertionRecordSchema = z.strictObject({
|
|
1236
|
-
label: z.string().describe("`expect`'s second argument, or `<matcher> <first argument>` (or the command `c.exec` ran, or the judge's question), cut at 60 characters; ` #2`, ` #3` on repeats within the attempt."),
|
|
1237
|
-
labelled: z.boolean().describe("The eval gave the label."),
|
|
1238
|
-
matcher: z.string().describe("`toHaveFile`, `not.toBe`, `resolves.toBe`, `poll.toBe`."),
|
|
1239
|
-
status: z.enum([
|
|
1240
|
-
"pass",
|
|
1241
|
-
"fail",
|
|
1242
|
-
"error"
|
|
1243
|
-
]).describe("`error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report)."),
|
|
1244
|
-
soft: z.boolean(),
|
|
1245
|
-
message: z.string().optional().describe("Why it failed or couldn't answer, cut at 1 KB."),
|
|
1246
|
-
durationMs: z.number().optional().describe("Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`).")
|
|
1247
|
-
}).describe("One `expect` or `expect.soft` matcher call an attempt made.");
|
|
1248
|
-
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
1249
|
-
const captureSchema = z.strictObject({
|
|
1250
|
-
name: z.string(),
|
|
1251
|
-
port: z.number().describe("Container port that was tunneled."),
|
|
1252
|
-
path: z.string().describe("Path on the app, as passed in."),
|
|
1253
|
-
viewport: z.union([captureWindowSchema, z.literal("full-page")]).describe("What the author asked, including `'full-page'`."),
|
|
1254
|
-
file: z.strictObject({
|
|
1255
|
-
path: z.string(),
|
|
1256
|
-
sha256: z.string(),
|
|
1257
|
-
bytes: z.number()
|
|
1258
|
-
}).describe("`artifacts/<name>.png` under the attempt directory.")
|
|
1259
|
-
}).describe("What `c.capture()` returns: the PNG is also in the attempt record under `artifacts/`.");
|
|
1260
|
-
const judgeArtifactRefSchema = z.strictObject({
|
|
1261
|
-
name: z.string(),
|
|
1262
|
-
path: z.string().describe("`artifacts/<name>.png` under the attempt directory."),
|
|
1263
|
-
sha256: z.string(),
|
|
1264
|
-
bytes: z.number(),
|
|
1265
|
-
mediaType: z.literal("image/png")
|
|
1266
|
-
}).describe("A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png`.");
|
|
1267
|
-
const verdictFields = {
|
|
1268
|
-
threshold: z.number(),
|
|
1269
|
-
question: z.string(),
|
|
1270
|
-
model: z.string(),
|
|
1271
|
-
gateway: gatewayNameSchema.optional().describe("The gateway from the config's `gateway` list the call went through; unset without one."),
|
|
1272
|
-
artifacts: z.array(judgeArtifactRefSchema).optional().describe("Captures this call sent the judge as file parts, when any.")
|
|
1273
|
-
};
|
|
1274
|
-
const verdictSchema = z.strictObject({
|
|
1275
|
-
verdict: z.enum(["pass", "fail"]),
|
|
1276
|
-
score: z.number().describe("The judge's probability that the answer is yes, from 0 to 1."),
|
|
1277
|
-
...verdictFields,
|
|
1278
|
-
resolvedModel: z.string().optional().describe("The model id the provider says served the call, when it reports one.")
|
|
1279
|
-
});
|
|
1280
|
-
const judgeFailureSchema = z.strictObject({
|
|
1281
|
-
verdict: z.literal("error"),
|
|
1282
|
-
score: z.null(),
|
|
1283
|
-
...verdictFields,
|
|
1284
|
-
error: z.string()
|
|
1285
|
-
}).describe("What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished, so the judges and checks after it still run.");
|
|
1286
|
-
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
1287
|
-
const patternSchema = z.strictObject({
|
|
1288
|
-
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
1289
|
-
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
1290
|
-
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
1291
|
-
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
1292
|
-
function hasAnsweredPattern(value) {
|
|
1293
|
-
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
1294
|
-
if (typeof value !== "object" || value === null) return false;
|
|
1295
|
-
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
1296
|
-
}
|
|
1297
|
-
const answeredPatternSchema = z.strictObject({
|
|
1298
|
-
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
1299
|
-
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
1300
|
-
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
1301
|
-
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
1302
|
-
const matcherArgSchema = z.union([
|
|
1303
|
-
patternSchema,
|
|
1304
|
-
answeredPatternSchema,
|
|
1305
|
-
jsonValueSchema
|
|
1306
|
-
]).superRefine((value, ctx) => {
|
|
1307
|
-
if (!isPatternLike(value)) return;
|
|
1308
|
-
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
1309
|
-
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
1310
|
-
const matcherSubjectSchema = z.union([
|
|
1311
|
-
z.strictObject({ turn: turnIndex }),
|
|
1312
|
-
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
1313
|
-
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
1314
|
-
z.strictObject({
|
|
1315
|
-
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
1316
|
-
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
1317
|
-
}),
|
|
1318
|
-
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
1319
|
-
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
1320
|
-
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
1321
|
-
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
1322
|
-
const millisSchema = z.number().positive("more than zero milliseconds");
|
|
1323
|
-
const captureWaitSchema = z.enum(["load", "idle"]).describe("Chromium look-ready events. `'load'` is `window` load (`Page.loadEventFired`). `'idle'` is `Page.lifecycleEvent` `networkAlmostIdle`.");
|
|
1324
|
-
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]).describe("Same shape at top-level `wait` and `{ wait }` between instructions.");
|
|
1325
|
-
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
1326
|
-
const captureCallDataSchema = z.strictObject({
|
|
1327
|
-
name: nonEmpty.optional(),
|
|
1328
|
-
path: z.string().optional(),
|
|
1329
|
-
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
1330
|
-
timeout: durationSchema.optional(),
|
|
1331
|
-
wait: waitSchema.optional().describe("Look-ready once, before the first model shot or the saved PNG. Default `'load'`."),
|
|
1332
|
-
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
1333
|
-
model: z.string().optional().describe("Overrides cascaded `capture.model` for this call.")
|
|
1334
|
-
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
1335
|
-
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
1336
|
-
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
1337
|
-
const judgeCallDataSchema = z.strictObject({
|
|
1338
|
-
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
1339
|
-
...judgeFields,
|
|
1340
|
-
on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
|
|
1341
|
-
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
1342
|
-
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
|
|
1343
|
-
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
1344
|
-
z.strictObject({
|
|
1345
|
-
op: z.literal("send"),
|
|
1346
|
-
prompt: z.string().describe("The turn's prompt.")
|
|
1347
|
-
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
1348
|
-
z.strictObject({
|
|
1349
|
-
op: z.literal("exec"),
|
|
1350
|
-
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
1351
|
-
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
1352
|
-
...execRetrySchema.shape
|
|
1353
|
-
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
1354
|
-
z.strictObject({
|
|
1355
|
-
op: z.literal("serve"),
|
|
1356
|
-
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
1357
|
-
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
1358
|
-
}).describe("`c.serve(command, port?)`: a background process and its port."),
|
|
1359
|
-
z.strictObject({
|
|
1360
|
-
op: z.literal("kill"),
|
|
1361
|
-
handle: handleId
|
|
1362
|
-
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
1363
|
-
z.strictObject({
|
|
1364
|
-
op: z.literal("capture"),
|
|
1365
|
-
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
1366
|
-
options: captureCallDataSchema.optional()
|
|
1367
|
-
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
1368
|
-
z.strictObject({
|
|
1369
|
-
op: z.literal("readFile"),
|
|
1370
|
-
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
1371
|
-
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
1372
|
-
z.strictObject({
|
|
1373
|
-
op: z.literal("writeFile"),
|
|
1374
|
-
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
1375
|
-
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
1376
|
-
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
1377
|
-
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
1378
|
-
z.strictObject({
|
|
1379
|
-
op: z.literal("expect"),
|
|
1380
|
-
subject: matcherSubjectSchema,
|
|
1381
|
-
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
1382
|
-
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
1383
|
-
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
1384
|
-
soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
|
|
1385
|
-
label: z.string().optional().describe("`expect`'s second argument."),
|
|
1386
|
-
poll: z.strictObject({
|
|
1387
|
-
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
1388
|
-
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
1389
|
-
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
1390
|
-
}).superRefine((call, ctx) => {
|
|
1391
|
-
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
1392
|
-
code: "custom",
|
|
1393
|
-
path: ["subject"],
|
|
1394
|
-
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
1395
|
-
});
|
|
1396
|
-
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
1397
|
-
code: "custom",
|
|
1398
|
-
path: ["subject", "result"],
|
|
1399
|
-
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
1400
|
-
});
|
|
1401
|
-
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
1402
|
-
code: "custom",
|
|
1403
|
-
path: ["args"],
|
|
1404
|
-
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
1405
|
-
});
|
|
1406
|
-
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
1407
|
-
z.strictObject({
|
|
1408
|
-
op: z.literal("record"),
|
|
1409
|
-
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
1410
|
-
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
1411
|
-
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
1412
|
-
status: z.enum(["pass", "fail"]),
|
|
1413
|
-
message: z.string().optional().describe("Why it failed."),
|
|
1414
|
-
soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
|
|
1415
|
-
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
1416
|
-
z.strictObject({
|
|
1417
|
-
op: z.literal("judge"),
|
|
1418
|
-
question: nonEmpty.describe("A yes/no question about the graded turn or value."),
|
|
1419
|
-
options: judgeCallDataSchema.optional()
|
|
1420
|
-
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
1421
|
-
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
1422
|
-
const attemptErrorSchema = z.looseObject({
|
|
1423
|
-
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
1424
|
-
message: z.string()
|
|
1425
|
-
}).describe("Why a call failed, in reeval's words.");
|
|
1426
|
-
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
1427
|
-
const attemptRequestSchema = z.strictObject({
|
|
1428
|
-
id,
|
|
1429
|
-
call: attemptCallSchema
|
|
1430
|
-
}).describe("A call, numbered so its reply can find it.");
|
|
1431
|
-
z.union([
|
|
1432
|
-
z.discriminatedUnion("op", [
|
|
1433
|
-
z.strictObject({
|
|
1434
|
-
id,
|
|
1435
|
-
ok: z.literal(true),
|
|
1436
|
-
op: z.literal("send"),
|
|
1437
|
-
turn: turnDataSchema
|
|
1438
|
-
}),
|
|
1439
|
-
z.strictObject({
|
|
1440
|
-
id,
|
|
1441
|
-
ok: z.literal(true),
|
|
1442
|
-
op: z.literal("exec"),
|
|
1443
|
-
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
1444
|
-
}),
|
|
1445
|
-
z.strictObject({
|
|
1446
|
-
id,
|
|
1447
|
-
ok: z.literal(true),
|
|
1448
|
-
op: z.literal("serve"),
|
|
1449
|
-
handle: handleId,
|
|
1450
|
-
port: z.number().describe("The container port it was given.")
|
|
1451
|
-
}),
|
|
1452
|
-
z.strictObject({
|
|
1453
|
-
id,
|
|
1454
|
-
ok: z.literal(true),
|
|
1455
|
-
op: z.literal("kill")
|
|
1456
|
-
}),
|
|
1457
|
-
z.strictObject({
|
|
1458
|
-
id,
|
|
1459
|
-
ok: z.literal(true),
|
|
1460
|
-
op: z.literal("capture"),
|
|
1461
|
-
capture: captureSchema
|
|
1462
|
-
}),
|
|
1463
|
-
z.strictObject({
|
|
1464
|
-
id,
|
|
1465
|
-
ok: z.literal(true),
|
|
1466
|
-
op: z.literal("readFile"),
|
|
1467
|
-
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
1468
|
-
}),
|
|
1469
|
-
z.strictObject({
|
|
1470
|
-
id,
|
|
1471
|
-
ok: z.literal(true),
|
|
1472
|
-
op: z.literal("writeFile")
|
|
1473
|
-
}),
|
|
1474
|
-
z.strictObject({
|
|
1475
|
-
id,
|
|
1476
|
-
ok: z.literal(true),
|
|
1477
|
-
op: z.literal("expect"),
|
|
1478
|
-
assertion: assertionRecordSchema
|
|
1479
|
-
}),
|
|
1480
|
-
z.strictObject({
|
|
1481
|
-
id,
|
|
1482
|
-
ok: z.literal(true),
|
|
1483
|
-
op: z.literal("record"),
|
|
1484
|
-
assertion: assertionRecordSchema
|
|
1485
|
-
}),
|
|
1486
|
-
z.strictObject({
|
|
1487
|
-
id,
|
|
1488
|
-
ok: z.literal(true),
|
|
1489
|
-
op: z.literal("judge"),
|
|
1490
|
-
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
1390
|
+
const normalizesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
|
|
1391
|
+
const NORMALIZED = 1e3;
|
|
1392
|
+
/** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
|
|
1393
|
+
function scalePoints(action, page) {
|
|
1394
|
+
const x = (n) => Math.round(n * page.width / NORMALIZED);
|
|
1395
|
+
const y = (n) => Math.round(n * page.height / NORMALIZED);
|
|
1396
|
+
switch (action.type) {
|
|
1397
|
+
case "click":
|
|
1398
|
+
case "hover":
|
|
1399
|
+
case "scroll": return {
|
|
1400
|
+
...action,
|
|
1401
|
+
x: x(action.x),
|
|
1402
|
+
y: y(action.y)
|
|
1403
|
+
};
|
|
1404
|
+
case "drag": return {
|
|
1405
|
+
...action,
|
|
1406
|
+
x: x(action.x),
|
|
1407
|
+
y: y(action.y),
|
|
1408
|
+
toX: x(action.toX),
|
|
1409
|
+
toY: y(action.toY)
|
|
1410
|
+
};
|
|
1411
|
+
default: return action;
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
|
|
1415
|
+
const useTools = {
|
|
1416
|
+
click: tool({
|
|
1417
|
+
description: "Click at (x, y) on the screenshot.",
|
|
1418
|
+
inputSchema: z.object({
|
|
1419
|
+
x: z.number(),
|
|
1420
|
+
y: z.number()
|
|
1491
1421
|
})
|
|
1492
|
-
]),
|
|
1493
|
-
z.strictObject({
|
|
1494
|
-
id,
|
|
1495
|
-
ok: z.literal(true),
|
|
1496
|
-
op: z.literal("expect"),
|
|
1497
|
-
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
1498
1422
|
}),
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1423
|
+
type: tool({
|
|
1424
|
+
description: "Insert text into the focused field. Click the field first.",
|
|
1425
|
+
inputSchema: z.object({ text: z.string() })
|
|
1426
|
+
}),
|
|
1427
|
+
key: tool({
|
|
1428
|
+
description: "Press Tab or Enter.",
|
|
1429
|
+
inputSchema: z.object({ key: z.enum(["Tab", "Enter"]) })
|
|
1430
|
+
}),
|
|
1431
|
+
scroll: tool({
|
|
1432
|
+
description: "Wheel-scroll a constrained scroller at (x, y).",
|
|
1433
|
+
inputSchema: z.object({
|
|
1434
|
+
x: z.number(),
|
|
1435
|
+
y: z.number(),
|
|
1436
|
+
dy: z.number()
|
|
1437
|
+
})
|
|
1438
|
+
}),
|
|
1439
|
+
drag: tool({
|
|
1440
|
+
description: "Drag from (x, y) to (toX, toY).",
|
|
1441
|
+
inputSchema: z.object({
|
|
1442
|
+
x: z.number(),
|
|
1443
|
+
y: z.number(),
|
|
1444
|
+
toX: z.number(),
|
|
1445
|
+
toY: z.number()
|
|
1446
|
+
})
|
|
1447
|
+
}),
|
|
1448
|
+
hover: tool({
|
|
1449
|
+
description: "Move the pointer to (x, y).",
|
|
1450
|
+
inputSchema: z.object({
|
|
1451
|
+
x: z.number(),
|
|
1452
|
+
y: z.number()
|
|
1453
|
+
})
|
|
1454
|
+
}),
|
|
1455
|
+
done: tool({
|
|
1456
|
+
description: "The goal is finished. Leave the loop.",
|
|
1457
|
+
inputSchema: z.object({})
|
|
1515
1458
|
})
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
}
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
const
|
|
1459
|
+
};
|
|
1460
|
+
async function runFakeScript(session, script, goal, remaining) {
|
|
1461
|
+
let moves = 0;
|
|
1462
|
+
for (const action of script) {
|
|
1463
|
+
remainingOrThrow(remaining, JSON.stringify(goal));
|
|
1464
|
+
if (action.type === "done") return moves;
|
|
1465
|
+
if (moves >= 32) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
|
|
1466
|
+
const page = pngSize(await session.screenshot(true));
|
|
1467
|
+
debugTrace("check", "capture-use", {
|
|
1468
|
+
goal,
|
|
1469
|
+
action,
|
|
1470
|
+
page
|
|
1471
|
+
}, `capture use ${action.type}`);
|
|
1472
|
+
await session.dispatch(action, page);
|
|
1473
|
+
moves++;
|
|
1474
|
+
await settle();
|
|
1475
|
+
}
|
|
1476
|
+
while (remaining() > 0) await settle();
|
|
1477
|
+
throw new Error(`c.capture() timed out on ${JSON.stringify(goal)}`);
|
|
1478
|
+
}
|
|
1479
|
+
async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
1480
|
+
const model = await loadModel(modelName);
|
|
1481
|
+
const normalized = normalizesPoints(modelName);
|
|
1482
|
+
let moves = 0;
|
|
1483
|
+
let usage = null;
|
|
1484
|
+
while (moves < 32) {
|
|
1485
|
+
remainingOrThrow(remaining, JSON.stringify(goal));
|
|
1486
|
+
const ephemeral = await session.screenshot(true);
|
|
1487
|
+
const page = pngSize(ephemeral);
|
|
1488
|
+
debugTrace("check", "capture-shot", {
|
|
1489
|
+
goal,
|
|
1490
|
+
bytes: ephemeral.length,
|
|
1491
|
+
page
|
|
1492
|
+
}, false);
|
|
1493
|
+
let chosen;
|
|
1494
|
+
const result = await generateText({
|
|
1495
|
+
model,
|
|
1496
|
+
system: systemPrompt(normalized),
|
|
1497
|
+
messages: [{
|
|
1498
|
+
role: "user",
|
|
1499
|
+
content: [{
|
|
1500
|
+
type: "text",
|
|
1501
|
+
text: goal
|
|
1502
|
+
}, {
|
|
1503
|
+
type: "image",
|
|
1504
|
+
image: ephemeral
|
|
1505
|
+
}]
|
|
1506
|
+
}],
|
|
1507
|
+
tools: useTools,
|
|
1508
|
+
toolChoice: "required",
|
|
1509
|
+
stopWhen: stepCountIs(1),
|
|
1510
|
+
abortSignal: AbortSignal.timeout(remainingOrThrow(remaining, JSON.stringify(goal)))
|
|
1511
|
+
});
|
|
1512
|
+
usage = addUsage(usage, asUsage(result.usage ?? {}));
|
|
1513
|
+
const call = result.toolCalls?.[0];
|
|
1514
|
+
if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
|
|
1515
|
+
if (chosen && normalized) chosen = scalePoints(chosen, page);
|
|
1516
|
+
if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
|
|
1517
|
+
debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
|
|
1518
|
+
debugTrace("check", "capture-use", {
|
|
1519
|
+
goal,
|
|
1520
|
+
action: chosen,
|
|
1521
|
+
page
|
|
1522
|
+
}, `capture use ${chosen.type}`);
|
|
1523
|
+
if (chosen.type === "done") return {
|
|
1524
|
+
moves,
|
|
1525
|
+
usage
|
|
1526
|
+
};
|
|
1527
|
+
await session.dispatch(chosen, page);
|
|
1528
|
+
moves++;
|
|
1529
|
+
await settle();
|
|
1530
|
+
}
|
|
1531
|
+
throw new Error(`c.capture() instruction ${JSON.stringify(goal)} did not finish in 32 moves`);
|
|
1532
|
+
}
|
|
1533
|
+
/**
|
|
1534
|
+
* Top-level wait, then the instruction loop, then the saved PNG.
|
|
1535
|
+
* Throws before `screenshot` when the budget or move cap is hit.
|
|
1536
|
+
*/
|
|
1537
|
+
async function runJourney(session, url, options) {
|
|
1538
|
+
const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
|
|
1539
|
+
remainingOrThrow(remaining, "navigate");
|
|
1540
|
+
await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining()));
|
|
1541
|
+
let moves = 0;
|
|
1542
|
+
let usage = null;
|
|
1543
|
+
let stringIndex = 0;
|
|
1544
|
+
for (const step of instructions) {
|
|
1545
|
+
if (typeof step !== "string") {
|
|
1546
|
+
remainingOrThrow(remaining, "wait");
|
|
1547
|
+
await session.wait(step.wait, remaining(), void 0);
|
|
1548
|
+
continue;
|
|
1549
|
+
}
|
|
1550
|
+
remainingOrThrow(remaining, JSON.stringify(step));
|
|
1551
|
+
debugLog("check", `capture instruction ${JSON.stringify(step)}`);
|
|
1552
|
+
if (use && "fake" in use) {
|
|
1553
|
+
const script = use.fake[stringIndex] ?? [];
|
|
1554
|
+
moves += await runFakeScript(session, script, step, remaining);
|
|
1555
|
+
} else if (use && "model" in use) {
|
|
1556
|
+
if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
|
|
1557
|
+
const ran = await runModelGoal(session, step, use.model, remaining, options.loadModel);
|
|
1558
|
+
moves += ran.moves;
|
|
1559
|
+
usage = addUsage(usage, ran.usage);
|
|
1560
|
+
}
|
|
1561
|
+
stringIndex++;
|
|
1562
|
+
await settle();
|
|
1563
|
+
}
|
|
1564
|
+
remainingOrThrow(remaining, "screenshot");
|
|
1565
|
+
return {
|
|
1566
|
+
png: await session.screenshot(fullPage),
|
|
1567
|
+
moves,
|
|
1568
|
+
model: use && "model" in use ? use.model : use && "fake" in use ? "fake" : void 0,
|
|
1569
|
+
usage
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
function captureUseSpend(result) {
|
|
1573
|
+
if (!result.model) return void 0;
|
|
1574
|
+
return {
|
|
1575
|
+
model: result.model,
|
|
1576
|
+
moves: result.moves,
|
|
1577
|
+
usage: result.usage
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1571
1580
|
//#endregion
|
|
1572
1581
|
//#region src/diff.ts
|
|
1573
|
-
const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0";
|
|
1582
|
+
const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0 -c core.quotePath=false";
|
|
1574
1583
|
const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE=${shellQuote(sandbox.workdir)} GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
|
|
1575
1584
|
/** Points HEAD at a commit of `$tree` with no parent, then drops every object nothing reaches. */
|
|
1576
1585
|
const RESET_HEAD = `c=$(${GIT} commit-tree -m base "$tree") && ${GIT} update-ref HEAD "$c" && ${GIT} prune --expire=now`;
|
|
@@ -1598,20 +1607,22 @@ async function acceptIntoSnapshot(sandbox, path, signal) {
|
|
|
1598
1607
|
async function diffSinceSnapshot(sandbox, signal) {
|
|
1599
1608
|
const env = `${snapshotEnv(sandbox)} && ${GIT} add -A && `;
|
|
1600
1609
|
const patch = await sandbox.execOwn(`${env}${GIT} diff --cached --binary HEAD`, { signal });
|
|
1601
|
-
const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
|
|
1610
|
+
const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames -z HEAD`, { signal });
|
|
1602
1611
|
if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
|
|
1603
1612
|
const status = {
|
|
1604
1613
|
A: "added",
|
|
1605
1614
|
M: "modified",
|
|
1606
1615
|
D: "deleted"
|
|
1607
1616
|
};
|
|
1608
|
-
const
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1617
|
+
const fields = names.stdout.split("\0");
|
|
1618
|
+
const files = [];
|
|
1619
|
+
for (let i = 0; i + 1 < fields.length; i += 2) {
|
|
1620
|
+
const code = fields[i];
|
|
1621
|
+
files.push({
|
|
1622
|
+
path: fields[i + 1],
|
|
1612
1623
|
status: status[code[0]] ?? "modified"
|
|
1613
|
-
};
|
|
1614
|
-
}
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1615
1626
|
return {
|
|
1616
1627
|
patch: patch.stdout,
|
|
1617
1628
|
files
|
|
@@ -1656,7 +1667,7 @@ function createChecks(a) {
|
|
|
1656
1667
|
durationMs: Math.round(performance.now() - began)
|
|
1657
1668
|
});
|
|
1658
1669
|
if (r.exitCode === 0 || tries.length > retries) return r;
|
|
1659
|
-
debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
|
|
1670
|
+
debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
|
|
1660
1671
|
await setTimeout$1(interval, void 0, { signal: a.signal });
|
|
1661
1672
|
}
|
|
1662
1673
|
});
|
|
@@ -1672,13 +1683,13 @@ function createChecks(a) {
|
|
|
1672
1683
|
...retried
|
|
1673
1684
|
});
|
|
1674
1685
|
const of = retries > 0 ? ` on try ${tries.length} of ${retries + 1}` : "";
|
|
1675
|
-
debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
|
|
1686
|
+
debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
|
|
1676
1687
|
debugTrace("check", "output", {
|
|
1677
1688
|
command,
|
|
1678
1689
|
...r,
|
|
1679
1690
|
durationMs,
|
|
1680
1691
|
...retried
|
|
1681
|
-
}, `check output: ${clip(command, 80)}`);
|
|
1692
|
+
}, `check output: ${clip(redactSecrets(command), 80)}`);
|
|
1682
1693
|
a.report({
|
|
1683
1694
|
kind: "check-end",
|
|
1684
1695
|
command,
|
|
@@ -1694,7 +1705,7 @@ function createChecks(a) {
|
|
|
1694
1705
|
function exec(command, options) {
|
|
1695
1706
|
const problem = options === void 0 ? void 0 : execOptionsProblem(options);
|
|
1696
1707
|
if (problem) return Promise.reject(new UsageError(problem));
|
|
1697
|
-
if (options?.background) return import("./capture-
|
|
1708
|
+
if (options?.background) return import("./capture-DSK8O5bA.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
|
|
1698
1709
|
const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
|
|
1699
1710
|
return Object.defineProperty(run, SUBJECT, { value: command });
|
|
1700
1711
|
}
|
|
@@ -1881,11 +1892,11 @@ async function shot(a, url, resolved, remaining, onMoves) {
|
|
|
1881
1892
|
remaining,
|
|
1882
1893
|
loadModel: async (name) => {
|
|
1883
1894
|
const { config, profile } = getRunContext();
|
|
1884
|
-
const
|
|
1885
|
-
const
|
|
1886
|
-
return
|
|
1887
|
-
|
|
1888
|
-
|
|
1895
|
+
const scope = gatewayScope(config, profile);
|
|
1896
|
+
const gateway = resolveModelGateway(name, void 0, scope, "capture model");
|
|
1897
|
+
return hostModel(name, gateway, "capture model", {
|
|
1898
|
+
fallbacks: openRouterFallbacks(scope.gateways),
|
|
1899
|
+
...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
|
|
1889
1900
|
});
|
|
1890
1901
|
}
|
|
1891
1902
|
});
|
|
@@ -1910,6 +1921,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
|
|
|
1910
1921
|
for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
|
|
1911
1922
|
}
|
|
1912
1923
|
//#endregion
|
|
1913
|
-
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d,
|
|
1924
|
+
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, attemptRequestProblem as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
|
|
1914
1925
|
|
|
1915
|
-
//# sourceMappingURL=capture-
|
|
1926
|
+
//# sourceMappingURL=capture-BM_3100g.js.map
|