reeval 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-BIlJWXff.js → adapter-CptK032x.js} +2 -2
- package/dist/{adapter-BIlJWXff.js.map → adapter-CptK032x.js.map} +1 -1
- package/dist/{agent-CE-qdvmh.js → agent-B63OzUX9.js} +124 -11
- package/dist/agent-B63OzUX9.js.map +1 -0
- package/dist/{analysis-CTBb_C0Z.js → analysis-BnG_1x_O.js} +4 -4
- package/dist/{analysis-CTBb_C0Z.js.map → analysis-BnG_1x_O.js.map} +1 -1
- package/dist/{app-files-YMmDAHA8.js → app-files-sgqwfjtn.js} +4 -4
- package/dist/app-files-sgqwfjtn.js.map +1 -0
- package/dist/assemble-CxBHkavM.js +1 -0
- package/dist/{assemble-7RI7Nuhz.js → assemble-Me7Whp-y.js} +62 -196
- package/dist/assemble-Me7Whp-y.js.map +1 -0
- package/dist/{attempt-CtKGvaQR.js → attempt-Dp7Wtm8M.js} +2 -2
- package/dist/{attempt-CtKGvaQR.js.map → attempt-Dp7Wtm8M.js.map} +1 -1
- package/dist/{attempt-host-CJM_wEA4.js → attempt-host-Cik0SoIn.js} +17 -9
- package/dist/{attempt-host-CJM_wEA4.js.map → attempt-host-Cik0SoIn.js.map} +1 -1
- package/dist/bridge/index.mjs +9 -5
- package/dist/{capture-DF8cHsJh.js → capture-C3RmjPph.js} +392 -609
- package/dist/capture-C3RmjPph.js.map +1 -0
- package/dist/capture-YPwxp_iK.js +1 -0
- package/dist/{cli-Bm9ntSxz.js → cli-BBOlXKPG.js} +1078 -459
- package/dist/cli-BBOlXKPG.js.map +1 -0
- package/dist/{cli-spec-ChkleEVm.js → cli-spec-DZ0bnP0_.js} +9 -8
- package/dist/cli-spec-DZ0bnP0_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-CF-DOjOU.js → client-tZ2WBDxf.js} +20 -13
- package/dist/client-tZ2WBDxf.js.map +1 -0
- package/dist/{command-Dk6wggV-.js → command-CFQ9c1hk.js} +13 -13
- package/dist/{command-Dk6wggV-.js.map → command-CFQ9c1hk.js.map} +1 -1
- package/dist/{command-DPF8hnWS.js → command-DhiwsZl1.js} +9 -9
- package/dist/{command-DPF8hnWS.js.map → command-DhiwsZl1.js.map} +1 -1
- package/dist/{command-DDkz6jsD.js → command-fO0BFsTq.js} +105 -37
- package/dist/command-fO0BFsTq.js.map +1 -0
- package/dist/{command-B7MslY_w.js → command-qGS802uX.js} +17 -15
- package/dist/command-qGS802uX.js.map +1 -0
- package/dist/{commands-DHT-nhEJ.js → commands-CC60TOT5.js} +64 -18
- package/dist/commands-CC60TOT5.js.map +1 -0
- package/dist/{commands-DQBayPCV.js → commands-xYTMNTPv.js} +1 -1
- package/dist/{config-BLL5JqeX.js → config-C2UtS9vy.js} +745 -80
- package/dist/config-C2UtS9vy.js.map +1 -0
- package/dist/{context-I6nq3uS7.js → context-BajfiH0S.js} +3 -3
- package/dist/{context-I6nq3uS7.js.map → context-BajfiH0S.js.map} +1 -1
- package/dist/{contrasts-CjOU_pfD.js → contrasts-BjOg7wyr.js} +3 -3
- package/dist/{contrasts-CjOU_pfD.js.map → contrasts-BjOg7wyr.js.map} +1 -1
- package/dist/cost-BJU0MAji.js +1 -0
- package/dist/{cost-e4IppZpJ.js → cost-BuVDamui.js} +4 -4
- package/dist/{cost-e4IppZpJ.js.map → cost-BuVDamui.js.map} +1 -1
- package/dist/{cursor-catalog-BeBJTk6A.js → cursor-catalog-BTK_apL-.js} +2 -2
- package/dist/{cursor-catalog-BeBJTk6A.js.map → cursor-catalog-BTK_apL-.js.map} +1 -1
- package/dist/{db-DPiWC7Br.js → db-C5SZ0510.js} +79 -18
- package/dist/db-C5SZ0510.js.map +1 -0
- package/dist/db-puzLzX4k.js +2 -0
- package/dist/{default-effort-ciIXMA7h.js → default-effort-BVmx7r7L.js} +8 -5
- package/dist/default-effort-BVmx7r7L.js.map +1 -0
- package/dist/{default-effort-B7c4V9Qd.js → default-effort-FEidNAIi.js} +1 -1
- package/dist/{dist-CGwueB0T.js → dist-ByYk_wAs.js} +58 -290
- package/dist/dist-ByYk_wAs.js.map +1 -0
- package/dist/{docker-Bcrc39Z8.js → docker-B2F8-PGC.js} +4 -4
- package/dist/{docker-Bcrc39Z8.js.map → docker-B2F8-PGC.js.map} +1 -1
- package/dist/{docker-D83s_ADn.js → docker-BlSopHpZ.js} +1 -1
- package/dist/{docker-BUZOxh62.js → docker-Dto8ROCq.js} +5 -5
- package/dist/{docker-BUZOxh62.js.map → docker-Dto8ROCq.js.map} +1 -1
- package/dist/{doctor-CIhEAzbw.js → doctor-BcvfKpBX.js} +20 -20
- package/dist/{doctor-CIhEAzbw.js.map → doctor-BcvfKpBX.js.map} +1 -1
- package/dist/{env-T86lAqUD.js → env-8x1O8I1M.js} +2 -2
- package/dist/{env-T86lAqUD.js.map → env-8x1O8I1M.js.map} +1 -1
- package/dist/{environment-Ccytlpgf.js → environment-CC1RV21n.js} +3 -3
- package/dist/{environment-Ccytlpgf.js.map → environment-CC1RV21n.js.map} +1 -1
- package/dist/{eval-source-D4xqc0iO.js → eval-source-CzxdgFBW.js} +18 -10
- package/dist/eval-source-CzxdgFBW.js.map +1 -0
- package/dist/evaluate-DToSB2bt.d.ts +101 -0
- package/dist/{evaluate-DNzyRQFe.js → evaluate-NTNJ8T9H.js} +135 -54
- package/dist/evaluate-NTNJ8T9H.js.map +1 -0
- package/dist/{files-D_CBK9Zo.js → files-CZXrkPja.js} +6 -6
- package/dist/{files-D_CBK9Zo.js.map → files-CZXrkPja.js.map} +1 -1
- package/dist/files-DFtuRQJx.js +1 -0
- package/dist/{files-3ZqUBbwg.js → files-DsXULUi5.js} +4 -4
- package/dist/{files-3ZqUBbwg.js.map → files-DsXULUi5.js.map} +1 -1
- package/dist/{files-B2U5RezT.js → files-aAzoGNCB.js} +5 -5
- package/dist/{files-B2U5RezT.js.map → files-aAzoGNCB.js.map} +1 -1
- package/dist/files-oOYNR7ZH.js +1 -0
- package/dist/{gate-BoDKeuJu.js → gate-Cp_pQoEe.js} +2 -2
- package/dist/{gate-BoDKeuJu.js.map → gate-Cp_pQoEe.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-s-qf-vww.js → git-C1MB5bjr.js} +3 -3
- package/dist/{git-s-qf-vww.js.map → git-C1MB5bjr.js.map} +1 -1
- package/dist/harness-DGsRZDSo.js +105 -0
- package/dist/harness-DGsRZDSo.js.map +1 -0
- package/dist/{harness-Ch2P81sx.js → harness-DYSC4zMh.js} +65 -110
- package/dist/harness-DYSC4zMh.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{images-h1VBM7TP.js → images-CNayXCp2.js} +8 -8
- package/dist/{images-h1VBM7TP.js.map → images-CNayXCp2.js.map} +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +5 -5
- package/dist/init-8o6W3_fn.js +1 -0
- package/dist/{init-Z1cRnE3E.js → init-D5HtmyMI.js} +5 -5
- package/dist/{init-Z1cRnE3E.js.map → init-D5HtmyMI.js.map} +1 -1
- package/dist/ink-list-Ca-5dunO.js +184 -0
- package/dist/ink-list-Ca-5dunO.js.map +1 -0
- package/dist/{invocations-z6hUD15n.js → invocations-DrGXF9dT.js} +4 -4
- package/dist/{invocations-z6hUD15n.js.map → invocations-DrGXF9dT.js.map} +1 -1
- package/dist/{judge-ByEUlzOM.js → judge-DYOF69Oe.js} +5 -5
- package/dist/{judge-ByEUlzOM.js.map → judge-DYOF69Oe.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/judgments-m-mb0zv8.js +246 -0
- package/dist/judgments-m-mb0zv8.js.map +1 -0
- package/dist/list-CM_YSJXq.js +374 -0
- package/dist/list-CM_YSJXq.js.map +1 -0
- package/dist/{live-D22qY6S_.js → live-B71o2TkX.js} +22 -4
- package/dist/live-B71o2TkX.js.map +1 -0
- package/dist/live-DQc8jr6o.js +1 -0
- package/dist/{live-DpfJfc33.js → live-uRURFDMZ.js} +7 -7
- package/dist/{live-DpfJfc33.js.map → live-uRURFDMZ.js.map} +1 -1
- package/dist/{matchers-B4ZtlJyP.js → matchers-Cs8aDqdU.js} +3 -3
- package/dist/{matchers-B4ZtlJyP.js.map → matchers-Cs8aDqdU.js.map} +1 -1
- package/dist/{model-CFSjV-HJ.js → model-1L19Zukh.js} +5 -5
- package/dist/{model-CFSjV-HJ.js.map → model-1L19Zukh.js.map} +1 -1
- package/dist/{model-BL_10Neu.js → model-BxBWFOZn.js} +44 -20
- package/dist/{model-BL_10Neu.js.map → model-BxBWFOZn.js.map} +1 -1
- package/dist/{model-D1TBKeZj.js → model-CXVVB4RK.js} +1 -1
- package/dist/{model-Bn7ndUQZ.js → model-Ce9l9_rF.js} +27 -254
- package/dist/model-Ce9l9_rF.js.map +1 -0
- package/dist/model-Cxzo4cjw.js +1 -0
- package/dist/{node-CwP1G178.js → node-DhIRSrrK.js} +22 -17
- package/dist/{node-CwP1G178.js.map → node-DhIRSrrK.js.map} +1 -1
- package/dist/{opencode-providers-yNKyXD49.js → opencode-providers-Co7plCea.js} +4 -15
- package/dist/{opencode-providers-yNKyXD49.js.map → opencode-providers-Co7plCea.js.map} +1 -1
- package/dist/{openrouter-models-C8nzPKG2.js → openrouter-models-DNvLRYM7.js} +2 -2
- package/dist/{openrouter-models-C8nzPKG2.js.map → openrouter-models-DNvLRYM7.js.map} +1 -1
- package/dist/{patch-LFunkZ-L.js → patch-ug5HpMOj.js} +3 -3
- package/dist/{patch-LFunkZ-L.js.map → patch-ug5HpMOj.js.map} +1 -1
- package/dist/{plan-BrfiMcHh.js → plan-DqnzfDH1.js} +15 -14
- package/dist/plan-DqnzfDH1.js.map +1 -0
- package/dist/{plugins-CW-sQMg3.js → plugins-B2KQ0y3P.js} +4 -4
- package/dist/{plugins-CW-sQMg3.js.map → plugins-B2KQ0y3P.js.map} +1 -1
- package/dist/{preflight-CH-7KyhS.js → preflight-DVXEvzFU.js} +1 -1
- package/dist/{preflight-BnFpd3Kv.js → preflight-DyzmvK7i.js} +3 -3
- package/dist/{preflight-BnFpd3Kv.js.map → preflight-DyzmvK7i.js.map} +1 -1
- package/dist/{prepare-0cA77eFD.js → prepare-21WMQZyt.js} +7 -7
- package/dist/{prepare-0cA77eFD.js.map → prepare-21WMQZyt.js.map} +1 -1
- package/dist/{prepare-CRyEwb5q.js → prepare-BcSRa7-3.js} +2 -2
- package/dist/{prices-C4BRuSe8.js → prices-DYJA0tJN.js} +2 -2
- package/dist/{prices-C4BRuSe8.js.map → prices-DYJA0tJN.js.map} +1 -1
- package/dist/{provenance-CKIG_1LB.js → provenance-iRxjp4Pu.js} +4 -4
- package/dist/{provenance-CKIG_1LB.js.map → provenance-iRxjp4Pu.js.map} +1 -1
- package/dist/{providers-CCl8ElNW.js → providers-D0ob_tra.js} +1 -1
- package/dist/{providers-Dt6BEN_1.js → providers-vXMXuwn-.js} +5 -5
- package/dist/{providers-Dt6BEN_1.js.map → providers-vXMXuwn-.js.map} +1 -1
- package/dist/{query-BV-zcw2t.js → query-BhaKQ07P.js} +3 -3
- package/dist/{query-BV-zcw2t.js.map → query-BhaKQ07P.js.map} +1 -1
- package/dist/query-CvgUsvep.js +1 -0
- package/dist/{questions-AyLUMHYX.js → questions-BzdLdV3C.js} +39 -9
- package/dist/questions-BzdLdV3C.js.map +1 -0
- package/dist/{record-J45gJise.js → record-DhzwkSXc.js} +22 -2
- package/dist/{record-J45gJise.js.map → record-DhzwkSXc.js.map} +1 -1
- package/dist/{redact-BEEGc37e.js → redact-BhjtnjrU.js} +11 -7
- package/dist/redact-BhjtnjrU.js.map +1 -0
- package/dist/remarks-CAvTSq16.js +1 -0
- package/dist/{remarks-UIZTSIhd.js → remarks-CauU408t.js} +231 -25
- package/dist/remarks-CauU408t.js.map +1 -0
- package/dist/{remarks-0EXMh6AD.js → remarks-D9Lcd1fF.js} +3 -2
- package/dist/remarks-D9Lcd1fF.js.map +1 -0
- package/dist/{remarks-CgO1nIoG.js → remarks-NMgPd30U.js} +4 -3
- package/dist/{remarks-CgO1nIoG.js.map → remarks-NMgPd30U.js.map} +1 -1
- package/dist/{remote-BA9-hhgB.js → remote-Dwy3fyo3.js} +5 -5
- package/dist/{remote-BA9-hhgB.js.map → remote-Dwy3fyo3.js.map} +1 -1
- package/dist/{resolve-FHjE-dN3.js → resolve-B1zMnPwK.js} +7 -7
- package/dist/{resolve-FHjE-dN3.js.map → resolve-B1zMnPwK.js.map} +1 -1
- package/dist/resolve-BIIeVT12.js +1 -0
- package/dist/{results-CFE6Nwb0.js → results-CfC0id_u.js} +7 -7
- package/dist/{results-CFE6Nwb0.js.map → results-CfC0id_u.js.map} +1 -1
- package/dist/{resume-iaTxWRew.js → resume-Dkhrpjow.js} +123 -8
- package/dist/resume-Dkhrpjow.js.map +1 -0
- package/dist/{runtime-SEkGS1Rx.js → runtime-DcvQcr36.js} +2 -2
- package/dist/{runtime-SEkGS1Rx.js.map → runtime-DcvQcr36.js.map} +1 -1
- package/dist/{sandbox-CAsXlGro.js → sandbox-8zCrtuCf.js} +2 -2
- package/dist/{sandbox-CAsXlGro.js.map → sandbox-8zCrtuCf.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-0tuj8VSa.js +1 -0
- package/dist/{server-C9_kV1IP.js → server-Dplq8g0r.js} +8 -8
- package/dist/{server-C9_kV1IP.js.map → server-Dplq8g0r.js.map} +1 -1
- package/dist/{source-BszY6yAC.js → source-NWHqn5-o.js} +2 -2
- package/dist/{source-BszY6yAC.js.map → source-NWHqn5-o.js.map} +1 -1
- package/dist/source-llwLewOU.js +1 -0
- package/dist/{src-Dt6BWDcJ.js → src-B9oE_mNS.js} +1 -1
- package/dist/{src-DaYQ1O9Y.js → src-BQ6BiXoc.js} +86 -31
- package/dist/{src-DaYQ1O9Y.js.map → src-BQ6BiXoc.js.map} +1 -1
- package/dist/src-BQXdf-Gw.js +1448 -0
- package/dist/src-BQXdf-Gw.js.map +1 -0
- package/dist/{src-DVGVWOAB.js → src-BXnv0ViU.js} +1 -1
- package/dist/src-BiS8YNN_.js +3 -0
- package/dist/{src-CZANB4pb.js → src-BkH4GOxu.js} +1 -1
- package/dist/{src-8fy4nSWU.js → src-BvM0jWuZ.js} +28 -2
- package/dist/{src-8fy4nSWU.js.map → src-BvM0jWuZ.js.map} +1 -1
- package/dist/src-C-tH5ebd.js +2 -0
- package/dist/{src-umdk2alV.js → src-CUpoVVb8.js} +2 -2
- package/dist/src-Cg1bFs9g.js +2 -0
- package/dist/{src-CWrKJO92.js → src-D5t4OiRt.js} +2 -2
- package/dist/{src-CWrKJO92.js.map → src-D5t4OiRt.js.map} +1 -1
- package/dist/{src-DoxUxkU3.js → src-DV_UorZv.js} +1 -1
- package/dist/{src-DMiwtprP.js → src-KlDOGD0B.js} +4 -4
- package/dist/{src-DMiwtprP.js.map → src-KlDOGD0B.js.map} +1 -1
- package/dist/{src-BYpY_vXK.js → src-XxUpr2sa.js} +1 -1
- package/dist/{state-DfcikRK3.js → state-DDCGUhva.js} +16 -16
- package/dist/{state-DfcikRK3.js.map → state-DDCGUhva.js.map} +1 -1
- package/dist/state-DZbAO95a.js +1 -0
- package/dist/{stats-D-Gqwqt0.js → stats-C18W4xPX.js} +63 -8
- package/dist/stats-C18W4xPX.js.map +1 -0
- package/dist/suite-M8_7xio7.js +1294 -0
- package/dist/suite-M8_7xio7.js.map +1 -0
- package/dist/suite-u2KJnGpY.js +2 -0
- package/dist/{summary-BmooCwQv.js → summary-Uzc9osfV.js} +47 -13
- package/dist/summary-Uzc9osfV.js.map +1 -0
- package/dist/testing.d.ts +51 -37
- package/dist/testing.js +4 -4
- package/dist/{text-D--Q5KsA.js → text-B8-W3Cvn.js} +2 -4
- package/dist/{text-D--Q5KsA.js.map → text-B8-W3Cvn.js.map} +1 -1
- package/dist/{transcript-C_IpYMRk.js → transcript-Cg4SeqRX.js} +3 -3
- package/dist/{transcript-C_IpYMRk.js.map → transcript-Cg4SeqRX.js.map} +1 -1
- package/dist/{types-BlM-aN9_.d.ts → types-DGvexkcp.d.ts} +116 -66
- package/dist/{vertex-6uSpZ5L8.js → vertex-BLQMayc1.js} +5 -2
- package/dist/vertex-BLQMayc1.js.map +1 -0
- package/dist/vertex-h_peaZTN.js +1 -0
- package/dist/view/index.d.ts +143 -34
- package/dist/view/index.js +9 -8
- package/dist/{views-DPng-2-z.js → views-BISrsIjw.js} +6 -6
- package/dist/{views-DPng-2-z.js.map → views-BISrsIjw.js.map} +1 -1
- package/dist/views-Bfab8fM9.js +2 -0
- package/dist/{worker-entry-lt5otEb5.js → worker-entry-DJD_nHkS.js} +5 -5
- package/dist/{worker-entry-lt5otEb5.js.map → worker-entry-DJD_nHkS.js.map} +1 -1
- package/dist/workspace-B8EOCuxl.js +1 -0
- package/dist/{workspace-1O8x4oDc.js → workspace-BO1VhUR9.js} +5 -5
- package/dist/{workspace-1O8x4oDc.js.map → workspace-BO1VhUR9.js.map} +1 -1
- package/docker/Dockerfile +4 -4
- package/package.json +14 -12
- package/skills/eval-rubric/SKILL.md +22 -3
- package/dist/agent-CE-qdvmh.js.map +0 -1
- package/dist/app-files-YMmDAHA8.js.map +0 -1
- package/dist/assemble-7RI7Nuhz.js.map +0 -1
- package/dist/assemble-CANqwtwD.js +0 -1
- package/dist/capture-DF8cHsJh.js.map +0 -1
- package/dist/capture-DagM1npp.js +0 -1
- package/dist/cli-Bm9ntSxz.js.map +0 -1
- package/dist/cli-spec-ChkleEVm.js.map +0 -1
- package/dist/client-CF-DOjOU.js.map +0 -1
- package/dist/command-B7MslY_w.js.map +0 -1
- package/dist/command-DDkz6jsD.js.map +0 -1
- package/dist/commands-DHT-nhEJ.js.map +0 -1
- package/dist/config-BLL5JqeX.js.map +0 -1
- package/dist/cost-CMzY9VXv.js +0 -1
- package/dist/db-DPiWC7Br.js.map +0 -1
- package/dist/db-DysoTexX.js +0 -2
- package/dist/default-effort-ciIXMA7h.js.map +0 -1
- package/dist/dist-CGwueB0T.js.map +0 -1
- package/dist/eval-source-D4xqc0iO.js.map +0 -1
- package/dist/evaluate-BEGAQQuo.d.ts +0 -43
- package/dist/evaluate-DNzyRQFe.js.map +0 -1
- package/dist/files-Bw2NQecs.js +0 -1
- package/dist/files-Cp1jp0Nq.js +0 -1
- package/dist/harness-Ch2P81sx.js.map +0 -1
- package/dist/init-DJ3WFUt8.js +0 -1
- package/dist/live-D22qY6S_.js.map +0 -1
- package/dist/live-DUpitP_9.js +0 -1
- package/dist/model-Bn7ndUQZ.js.map +0 -1
- package/dist/model-CR3-qY-z.js +0 -1
- package/dist/plan-BrfiMcHh.js.map +0 -1
- package/dist/query-DSSMxkmw.js +0 -1
- package/dist/questions-AyLUMHYX.js.map +0 -1
- package/dist/redact-BEEGc37e.js.map +0 -1
- package/dist/remarks-0EXMh6AD.js.map +0 -1
- package/dist/remarks-BL18Id09.js +0 -1
- package/dist/remarks-UIZTSIhd.js.map +0 -1
- package/dist/resolve-C56s8_EF.js +0 -1
- package/dist/resume-iaTxWRew.js.map +0 -1
- package/dist/server-BYfqpMD0.js +0 -1
- package/dist/source-C4Q5psjD.js +0 -1
- package/dist/src-BBmCEE-M.js +0 -3
- package/dist/src-BWcgyARO.js +0 -640
- package/dist/src-BWcgyARO.js.map +0 -1
- package/dist/src-CnrLy_r_.js +0 -2
- package/dist/src-DqIQQTu1.js +0 -2
- package/dist/state-haxTOljN.js +0 -1
- package/dist/stats-D-Gqwqt0.js.map +0 -1
- package/dist/suite-BVcBo_ua.js +0 -544
- package/dist/suite-BVcBo_ua.js.map +0 -1
- package/dist/suite-CmeXyn66.js +0 -2
- package/dist/summary-BmooCwQv.js.map +0 -1
- package/dist/vertex-6uSpZ5L8.js.map +0 -1
- package/dist/vertex-BaAz44YN.js +0 -1
- package/dist/views-CBAsDkCB.js +0 -2
- package/dist/workspace-It7dvpbg.js +0 -1
- /package/dist/bridge/{pnpm-lock.yaml → pnpm-lockfile.yaml} +0 -0
|
@@ -1,21 +1,20 @@
|
|
|
1
1
|
import { n as isOneOf } from "./object-CeQ8dgUC.js";
|
|
2
2
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
3
|
-
import {
|
|
4
|
-
import { x as gatewayNameSchema } from "./harness-Ch2P81sx.js";
|
|
3
|
+
import { h as parseDuration } from "./shared-ogqPJ0uo.js";
|
|
5
4
|
import { l as sha256 } from "./ids-DuK6xl4w.js";
|
|
6
|
-
import {
|
|
7
|
-
import { y as vertexSettingsIn } from "./vertex-
|
|
8
|
-
import {
|
|
9
|
-
import { a as plural, i as formatDuration, n as clip } from "./text-
|
|
5
|
+
import { Ct as execOptionsProblem, b as resolveModelGateway, bt as CAPTURE_NAME, ct as parseProbe, lt as probeScript, ot as awaitListening, st as hostListening, ut as servePort, v as gatewayScope, xt as CAPTURE_NAME_RULE } from "./config-C2UtS9vy.js";
|
|
6
|
+
import { y as vertexSettingsIn } from "./vertex-BLQMayc1.js";
|
|
7
|
+
import { f as redactJson, h as redactSecrets } from "./redact-BhjtnjrU.js";
|
|
8
|
+
import { a as plural, i as formatDuration, n as clip } from "./text-B8-W3Cvn.js";
|
|
10
9
|
import { s as cacheDir } from "./root-Co0jhOT5.js";
|
|
11
|
-
import { mt as shellQuote } from "./src-
|
|
10
|
+
import { mt as shellQuote } from "./src-BQ6BiXoc.js";
|
|
12
11
|
import { t as InfraError } from "./types-C4OkYhZg.js";
|
|
13
|
-
import { r as getRunContext, s as SUBJECT } from "./runtime-
|
|
14
|
-
import { i as debugTrace, r as debugLog } from "./attempt-
|
|
15
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
16
|
-
import { i as hostModel } from "./model-
|
|
12
|
+
import { r as getRunContext, s as SUBJECT } from "./runtime-DcvQcr36.js";
|
|
13
|
+
import { i as debugTrace, r as debugLog } from "./attempt-Dp7Wtm8M.js";
|
|
14
|
+
import { r as openRouterFallbacks } from "./openrouter-models-DNvLRYM7.js";
|
|
15
|
+
import { i as hostModel } from "./model-BxBWFOZn.js";
|
|
17
16
|
import { t as addUsage } from "./usage-DJiDnh2D.js";
|
|
18
|
-
import { t as untraced } from "./docker-
|
|
17
|
+
import { t as untraced } from "./docker-B2F8-PGC.js";
|
|
19
18
|
import { n as pageLines } from "./capture-DMBZE9cs.js";
|
|
20
19
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
21
20
|
import { homedir, tmpdir } from "node:os";
|
|
@@ -29,517 +28,6 @@ import http from "node:http";
|
|
|
29
28
|
import { crc32, deflateSync, inflateRawSync, inflateSync } from "node:zlib";
|
|
30
29
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
31
30
|
import { generateText, stepCountIs, tool } from "ai";
|
|
32
|
-
// src/contract/attempt.ts
|
|
33
|
-
const nullableNumber = z.number().nullable();
|
|
34
|
-
const tokenUsageSchema = z.strictObject({
|
|
35
|
-
input: z.number(),
|
|
36
|
-
output: z.number(),
|
|
37
|
-
cacheRead: z.number(),
|
|
38
|
-
cacheWrite: z.number(),
|
|
39
|
-
reasoning: nullableNumber
|
|
40
|
-
});
|
|
41
|
-
const fileChangeSchema = z.strictObject({
|
|
42
|
-
path: z.string(),
|
|
43
|
-
status: z.enum([
|
|
44
|
-
"added",
|
|
45
|
-
"modified",
|
|
46
|
-
"deleted"
|
|
47
|
-
])
|
|
48
|
-
});
|
|
49
|
-
const diffSchema = z.strictObject({
|
|
50
|
-
patch: z.string(),
|
|
51
|
-
files: z.array(fileChangeSchema)
|
|
52
|
-
});
|
|
53
|
-
const turnStatusSchema = z.enum([
|
|
54
|
-
"completed",
|
|
55
|
-
"aborted",
|
|
56
|
-
"error"
|
|
57
|
-
]);
|
|
58
|
-
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
|
|
59
|
-
const at = {
|
|
60
|
-
ts: z.string(),
|
|
61
|
-
atMs: z.number()
|
|
62
|
-
};
|
|
63
|
-
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
64
|
-
z.strictObject({
|
|
65
|
-
...at,
|
|
66
|
-
type: z.enum(["text", "reasoning"]),
|
|
67
|
-
text: z.string(),
|
|
68
|
-
durationMs: nullableNumber
|
|
69
|
-
}),
|
|
70
|
-
z.strictObject({
|
|
71
|
-
...at,
|
|
72
|
-
type: z.literal("file-edit"),
|
|
73
|
-
toolCallId: z.string(),
|
|
74
|
-
path: z.string(),
|
|
75
|
-
kind: z.enum([
|
|
76
|
-
"create",
|
|
77
|
-
"modify",
|
|
78
|
-
"write",
|
|
79
|
-
"delete"
|
|
80
|
-
]),
|
|
81
|
-
added: nullableNumber,
|
|
82
|
-
removed: nullableNumber,
|
|
83
|
-
durationMs: nullableNumber,
|
|
84
|
-
stalled: z.boolean().optional()
|
|
85
|
-
}),
|
|
86
|
-
z.strictObject({
|
|
87
|
-
...at,
|
|
88
|
-
type: z.literal("web-search"),
|
|
89
|
-
toolCallId: z.string(),
|
|
90
|
-
query: z.string(),
|
|
91
|
-
references: nullableNumber,
|
|
92
|
-
ok: z.boolean().nullable().optional(),
|
|
93
|
-
durationMs: nullableNumber,
|
|
94
|
-
stalled: z.boolean().optional()
|
|
95
|
-
}),
|
|
96
|
-
z.strictObject({
|
|
97
|
-
...at,
|
|
98
|
-
type: z.literal("web-fetch"),
|
|
99
|
-
toolCallId: z.string(),
|
|
100
|
-
url: z.string(),
|
|
101
|
-
ok: z.boolean().nullable(),
|
|
102
|
-
durationMs: nullableNumber,
|
|
103
|
-
stalled: z.boolean().optional()
|
|
104
|
-
}),
|
|
105
|
-
z.strictObject({
|
|
106
|
-
...at,
|
|
107
|
-
type: z.literal("tool"),
|
|
108
|
-
toolCallId: z.string(),
|
|
109
|
-
tool: z.string(),
|
|
110
|
-
input: z.unknown(),
|
|
111
|
-
output: z.unknown(),
|
|
112
|
-
ok: z.boolean().nullable(),
|
|
113
|
-
durationMs: nullableNumber,
|
|
114
|
-
stalled: z.boolean().optional()
|
|
115
|
-
}),
|
|
116
|
-
z.strictObject({
|
|
117
|
-
...at,
|
|
118
|
-
type: z.literal("error"),
|
|
119
|
-
message: z.string()
|
|
120
|
-
}),
|
|
121
|
-
z.strictObject({
|
|
122
|
-
...at,
|
|
123
|
-
type: z.literal("retry"),
|
|
124
|
-
message: z.string()
|
|
125
|
-
})
|
|
126
|
-
]);
|
|
127
|
-
const turnDataSchema = z.strictObject({
|
|
128
|
-
prompt: z.string(),
|
|
129
|
-
text: z.string(),
|
|
130
|
-
status: turnStatusSchema,
|
|
131
|
-
startedAt: z.string(),
|
|
132
|
-
durationMs: z.number(),
|
|
133
|
-
transcript: z.array(transcriptEventSchema),
|
|
134
|
-
usage: tokenUsageSchema.nullable(),
|
|
135
|
-
model: z.string().nullable(),
|
|
136
|
-
unreported: z.array(webEventTypeSchema),
|
|
137
|
-
diff: diffSchema
|
|
138
|
-
});
|
|
139
|
-
const execResultSchema = z.strictObject({
|
|
140
|
-
exitCode: z.number().int(),
|
|
141
|
-
stdout: z.string(),
|
|
142
|
-
stderr: z.string()
|
|
143
|
-
});
|
|
144
|
-
const execRetrySchema = z.strictObject({
|
|
145
|
-
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
146
|
-
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
147
|
-
});
|
|
148
|
-
/** A background command has no exit for `retries` to wait on. */
|
|
149
|
-
const noBackgroundRetry = (options, ctx) => {
|
|
150
|
-
if (!options.background) return;
|
|
151
|
-
for (const key of ["retries", "interval"]) {
|
|
152
|
-
if (options[key] === void 0) continue;
|
|
153
|
-
ctx.addIssue({
|
|
154
|
-
code: "custom",
|
|
155
|
-
path: [key],
|
|
156
|
-
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
};
|
|
160
|
-
const execOptionsSchema = z.strictObject({
|
|
161
|
-
background: z.boolean().optional(),
|
|
162
|
-
...execRetrySchema.shape
|
|
163
|
-
}).superRefine(noBackgroundRetry);
|
|
164
|
-
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
165
|
-
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
166
|
-
const assertionRecordSchema = z.strictObject({
|
|
167
|
-
label: z.string(),
|
|
168
|
-
labelled: z.boolean(),
|
|
169
|
-
matcher: z.string(),
|
|
170
|
-
status: z.enum([
|
|
171
|
-
"pass",
|
|
172
|
-
"fail",
|
|
173
|
-
"error"
|
|
174
|
-
]),
|
|
175
|
-
soft: z.boolean(),
|
|
176
|
-
at: z.string().optional(),
|
|
177
|
-
message: z.string().optional(),
|
|
178
|
-
durationMs: z.number().optional()
|
|
179
|
-
});
|
|
180
|
-
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
181
|
-
const captureConsoleMessageSchema = z.strictObject({
|
|
182
|
-
level: z.enum([
|
|
183
|
-
"error",
|
|
184
|
-
"warning",
|
|
185
|
-
"info",
|
|
186
|
-
"log",
|
|
187
|
-
"debug"
|
|
188
|
-
]),
|
|
189
|
-
text: z.string(),
|
|
190
|
-
source: z.string(),
|
|
191
|
-
url: z.string().optional()
|
|
192
|
-
});
|
|
193
|
-
const captureFailedRequestSchema = z.strictObject({
|
|
194
|
-
method: z.string(),
|
|
195
|
-
url: z.string(),
|
|
196
|
-
status: z.number().optional(),
|
|
197
|
-
error: z.string().optional()
|
|
198
|
-
});
|
|
199
|
-
const capturePageFields = {
|
|
200
|
-
status: z.number().optional(),
|
|
201
|
-
finalUrl: z.string(),
|
|
202
|
-
console: z.array(captureConsoleMessageSchema),
|
|
203
|
-
errors: z.array(z.string()),
|
|
204
|
-
failedRequests: z.array(captureFailedRequestSchema),
|
|
205
|
-
dropped: z.number().optional()
|
|
206
|
-
};
|
|
207
|
-
z.strictObject(capturePageFields);
|
|
208
|
-
const captureSchema = z.strictObject({
|
|
209
|
-
name: z.string(),
|
|
210
|
-
port: z.number(),
|
|
211
|
-
path: z.string(),
|
|
212
|
-
viewport: z.union([captureWindowSchema, z.literal("full-page")]),
|
|
213
|
-
file: z.strictObject({
|
|
214
|
-
path: z.string(),
|
|
215
|
-
sha256: z.string(),
|
|
216
|
-
bytes: z.number()
|
|
217
|
-
}),
|
|
218
|
-
...capturePageFields
|
|
219
|
-
});
|
|
220
|
-
const judgeArtifactRefSchema = z.strictObject({
|
|
221
|
-
name: z.string(),
|
|
222
|
-
path: z.string(),
|
|
223
|
-
sha256: z.string(),
|
|
224
|
-
bytes: z.number(),
|
|
225
|
-
mediaType: z.literal("image/png")
|
|
226
|
-
});
|
|
227
|
-
const verdictFields = {
|
|
228
|
-
threshold: z.number(),
|
|
229
|
-
question: z.string(),
|
|
230
|
-
model: z.string(),
|
|
231
|
-
gateway: gatewayNameSchema.optional(),
|
|
232
|
-
artifacts: z.array(judgeArtifactRefSchema).optional()
|
|
233
|
-
};
|
|
234
|
-
const verdictSchema = z.strictObject({
|
|
235
|
-
verdict: z.enum(["pass", "fail"]),
|
|
236
|
-
score: z.number(),
|
|
237
|
-
...verdictFields,
|
|
238
|
-
resolvedModel: z.string().optional(),
|
|
239
|
-
upstream: z.string().optional(),
|
|
240
|
-
costUsd: z.number().optional()
|
|
241
|
-
});
|
|
242
|
-
const judgeFailureSchema = z.strictObject({
|
|
243
|
-
verdict: z.literal("error"),
|
|
244
|
-
score: z.null(),
|
|
245
|
-
...verdictFields,
|
|
246
|
-
error: z.string()
|
|
247
|
-
});
|
|
248
|
-
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
249
|
-
const patternSchema = z.strictObject({
|
|
250
|
-
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
251
|
-
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
252
|
-
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
253
|
-
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
254
|
-
function hasAnsweredPattern(value) {
|
|
255
|
-
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
256
|
-
if (typeof value !== "object" || value === null) return false;
|
|
257
|
-
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
258
|
-
}
|
|
259
|
-
const answeredPatternSchema = z.strictObject({
|
|
260
|
-
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
261
|
-
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
262
|
-
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
263
|
-
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
264
|
-
const matcherArgSchema = z.union([
|
|
265
|
-
patternSchema,
|
|
266
|
-
answeredPatternSchema,
|
|
267
|
-
jsonValueSchema
|
|
268
|
-
]).superRefine((value, ctx) => {
|
|
269
|
-
if (!isPatternLike(value)) return;
|
|
270
|
-
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
271
|
-
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
272
|
-
const matcherSubjectSchema = z.union([
|
|
273
|
-
z.strictObject({ turn: turnIndex }),
|
|
274
|
-
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
275
|
-
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
276
|
-
z.strictObject({
|
|
277
|
-
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
278
|
-
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
279
|
-
}),
|
|
280
|
-
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
281
|
-
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
282
|
-
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
283
|
-
const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
|
|
284
|
-
const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
|
|
285
|
-
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
286
|
-
const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
|
|
287
|
-
const captureWaitSchema = z.enum(["load", "idle"]);
|
|
288
|
-
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
|
|
289
|
-
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
290
|
-
const captureCallDataSchema = z.strictObject({
|
|
291
|
-
name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
|
|
292
|
-
path: z.string().optional(),
|
|
293
|
-
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
294
|
-
timeout: durationSchema.optional(),
|
|
295
|
-
wait: waitSchema.optional(),
|
|
296
|
-
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
297
|
-
model: z.string().optional()
|
|
298
|
-
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
299
|
-
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
300
|
-
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
301
|
-
const judgeCallDataSchema = z.strictObject({
|
|
302
|
-
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
303
|
-
...judgeFields,
|
|
304
|
-
...sharedInputFields,
|
|
305
|
-
contents: z.array(nonEmpty).optional(),
|
|
306
|
-
turns: turnsSchema.optional(),
|
|
307
|
-
on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
|
|
308
|
-
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
309
|
-
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
|
|
310
|
-
const callSiteSchema = z.strictObject({
|
|
311
|
-
file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
|
|
312
|
-
line: z.number().int().min(1).describe("From 1.")
|
|
313
|
-
}).describe("Where the eval made the call, which the record names beside the check's outcome.");
|
|
314
|
-
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
315
|
-
z.strictObject({
|
|
316
|
-
op: z.literal("send"),
|
|
317
|
-
prompt: z.string().describe("The turn's prompt.")
|
|
318
|
-
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
319
|
-
z.strictObject({
|
|
320
|
-
op: z.literal("exec"),
|
|
321
|
-
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
322
|
-
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
323
|
-
...execRetrySchema.shape
|
|
324
|
-
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
325
|
-
z.strictObject({
|
|
326
|
-
op: z.literal("serve"),
|
|
327
|
-
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
328
|
-
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
329
|
-
}).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
|
|
330
|
-
z.strictObject({
|
|
331
|
-
op: z.literal("kill"),
|
|
332
|
-
handle: handleId
|
|
333
|
-
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
334
|
-
z.strictObject({
|
|
335
|
-
op: z.literal("capture"),
|
|
336
|
-
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
337
|
-
options: captureCallDataSchema.optional()
|
|
338
|
-
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
339
|
-
z.strictObject({
|
|
340
|
-
op: z.literal("readFile"),
|
|
341
|
-
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
342
|
-
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
343
|
-
z.strictObject({
|
|
344
|
-
op: z.literal("writeFile"),
|
|
345
|
-
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
346
|
-
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
347
|
-
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
348
|
-
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
349
|
-
z.strictObject({
|
|
350
|
-
op: z.literal("expect"),
|
|
351
|
-
subject: matcherSubjectSchema,
|
|
352
|
-
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
353
|
-
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
354
|
-
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
355
|
-
soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
|
|
356
|
-
label: z.string().optional().describe("`expect`'s second argument."),
|
|
357
|
-
at: callSiteSchema.optional(),
|
|
358
|
-
poll: z.strictObject({
|
|
359
|
-
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
360
|
-
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
361
|
-
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
362
|
-
}).superRefine((call, ctx) => {
|
|
363
|
-
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
364
|
-
code: "custom",
|
|
365
|
-
path: ["subject"],
|
|
366
|
-
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
367
|
-
});
|
|
368
|
-
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
369
|
-
code: "custom",
|
|
370
|
-
path: ["subject", "result"],
|
|
371
|
-
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
372
|
-
});
|
|
373
|
-
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
374
|
-
code: "custom",
|
|
375
|
-
path: ["args"],
|
|
376
|
-
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
377
|
-
});
|
|
378
|
-
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
379
|
-
z.strictObject({
|
|
380
|
-
op: z.literal("record"),
|
|
381
|
-
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
382
|
-
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
383
|
-
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
384
|
-
status: z.enum(["pass", "fail"]),
|
|
385
|
-
message: z.string().optional().describe("Why it failed."),
|
|
386
|
-
soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
|
|
387
|
-
at: callSiteSchema.optional()
|
|
388
|
-
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
389
|
-
z.strictObject({
|
|
390
|
-
op: z.literal("judge"),
|
|
391
|
-
question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
|
|
392
|
-
options: judgeCallDataSchema
|
|
393
|
-
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
394
|
-
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
395
|
-
const attemptErrorSchema = z.looseObject({
|
|
396
|
-
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
397
|
-
message: z.string()
|
|
398
|
-
}).describe("Why a call failed, in reeval's words.");
|
|
399
|
-
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
400
|
-
const attemptRequestSchema = z.strictObject({
|
|
401
|
-
id,
|
|
402
|
-
call: attemptCallSchema
|
|
403
|
-
}).describe("A call, numbered so its reply can find it.");
|
|
404
|
-
z.union([
|
|
405
|
-
z.discriminatedUnion("op", [
|
|
406
|
-
z.strictObject({
|
|
407
|
-
id,
|
|
408
|
-
ok: z.literal(true),
|
|
409
|
-
op: z.literal("send"),
|
|
410
|
-
turn: turnDataSchema
|
|
411
|
-
}),
|
|
412
|
-
z.strictObject({
|
|
413
|
-
id,
|
|
414
|
-
ok: z.literal(true),
|
|
415
|
-
op: z.literal("exec"),
|
|
416
|
-
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
417
|
-
}),
|
|
418
|
-
z.strictObject({
|
|
419
|
-
id,
|
|
420
|
-
ok: z.literal(true),
|
|
421
|
-
op: z.literal("serve"),
|
|
422
|
-
handle: handleId,
|
|
423
|
-
port: z.number().describe("The container port it listens on.")
|
|
424
|
-
}),
|
|
425
|
-
z.strictObject({
|
|
426
|
-
id,
|
|
427
|
-
ok: z.literal(true),
|
|
428
|
-
op: z.literal("kill")
|
|
429
|
-
}),
|
|
430
|
-
z.strictObject({
|
|
431
|
-
id,
|
|
432
|
-
ok: z.literal(true),
|
|
433
|
-
op: z.literal("capture"),
|
|
434
|
-
capture: captureSchema
|
|
435
|
-
}),
|
|
436
|
-
z.strictObject({
|
|
437
|
-
id,
|
|
438
|
-
ok: z.literal(true),
|
|
439
|
-
op: z.literal("readFile"),
|
|
440
|
-
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
441
|
-
}),
|
|
442
|
-
z.strictObject({
|
|
443
|
-
id,
|
|
444
|
-
ok: z.literal(true),
|
|
445
|
-
op: z.literal("writeFile")
|
|
446
|
-
}),
|
|
447
|
-
z.strictObject({
|
|
448
|
-
id,
|
|
449
|
-
ok: z.literal(true),
|
|
450
|
-
op: z.literal("expect"),
|
|
451
|
-
assertion: assertionRecordSchema
|
|
452
|
-
}),
|
|
453
|
-
z.strictObject({
|
|
454
|
-
id,
|
|
455
|
-
ok: z.literal(true),
|
|
456
|
-
op: z.literal("record"),
|
|
457
|
-
assertion: assertionRecordSchema
|
|
458
|
-
}),
|
|
459
|
-
z.strictObject({
|
|
460
|
-
id,
|
|
461
|
-
ok: z.literal(true),
|
|
462
|
-
op: z.literal("judge"),
|
|
463
|
-
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
464
|
-
})
|
|
465
|
-
]),
|
|
466
|
-
z.strictObject({
|
|
467
|
-
id,
|
|
468
|
-
ok: z.literal(true),
|
|
469
|
-
op: z.literal("expect"),
|
|
470
|
-
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
471
|
-
}),
|
|
472
|
-
z.strictObject({
|
|
473
|
-
id,
|
|
474
|
-
ok: z.literal(false),
|
|
475
|
-
op: z.enum([
|
|
476
|
-
"send",
|
|
477
|
-
"exec",
|
|
478
|
-
"readFile",
|
|
479
|
-
"writeFile",
|
|
480
|
-
"expect",
|
|
481
|
-
"record",
|
|
482
|
-
"judge",
|
|
483
|
-
"serve",
|
|
484
|
-
"kill",
|
|
485
|
-
"capture"
|
|
486
|
-
]),
|
|
487
|
-
error: attemptErrorSchema
|
|
488
|
-
})
|
|
489
|
-
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
490
|
-
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
491
|
-
const cellSchema = z.strictObject({
|
|
492
|
-
key: z.string(),
|
|
493
|
-
harness: z.string(),
|
|
494
|
-
environment: z.string(),
|
|
495
|
-
sandbox: z.string(),
|
|
496
|
-
role: cellRoleSchema,
|
|
497
|
-
variant: z.string().optional()
|
|
498
|
-
});
|
|
499
|
-
z.strictObject({
|
|
500
|
-
type: z.literal("start"),
|
|
501
|
-
attempt: z.strictObject({
|
|
502
|
-
id: nonEmpty,
|
|
503
|
-
cell: cellSchema,
|
|
504
|
-
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
505
|
-
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
506
|
-
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
507
|
-
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
508
|
-
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
509
|
-
z.strictObject({
|
|
510
|
-
type: z.literal("abort"),
|
|
511
|
-
reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
|
|
512
|
-
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
513
|
-
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
514
|
-
z.strictObject({
|
|
515
|
-
type: z.literal("done"),
|
|
516
|
-
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
|
|
517
|
-
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
518
|
-
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
519
|
-
const OTHER_KEYS = [
|
|
520
|
-
"id",
|
|
521
|
-
"call",
|
|
522
|
-
"turn",
|
|
523
|
-
"attempt",
|
|
524
|
-
"value",
|
|
525
|
-
"regexp",
|
|
526
|
-
"flags",
|
|
527
|
-
"matchedBy",
|
|
528
|
-
"answers",
|
|
529
|
-
...Object.keys(judgeCallDataSchema.shape),
|
|
530
|
-
...Object.keys(captureSchema.shape),
|
|
531
|
-
...Object.keys(captureCallDataSchema.shape),
|
|
532
|
-
"port",
|
|
533
|
-
"handle",
|
|
534
|
-
"exec",
|
|
535
|
-
"result",
|
|
536
|
-
"readFile",
|
|
537
|
-
"interval",
|
|
538
|
-
"timeout",
|
|
539
|
-
"ms"
|
|
540
|
-
];
|
|
541
|
-
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
542
|
-
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
543
31
|
// src/capture/fake-instructions.ts
|
|
544
32
|
const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
|
|
545
33
|
function isWaitStep(step) {
|
|
@@ -784,10 +272,13 @@ async function waitForPage(cdp, wait, timeoutMs, nav) {
|
|
|
784
272
|
await page.ready(nav, timeoutMs);
|
|
785
273
|
}
|
|
786
274
|
const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
|
|
787
|
-
/**
|
|
275
|
+
/**
|
|
276
|
+
* Map a full-page point onto the live window, scrolled to `scrollY`. A point already in the window
|
|
277
|
+
* is left where it is; any other is scrolled to the window's middle with the wheel.
|
|
278
|
+
*/
|
|
788
279
|
function mapPagePoint(x, y, window, page, scrollY) {
|
|
789
|
-
const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
|
|
790
|
-
const target = y >=
|
|
280
|
+
const maxY = Math.max(0, (page?.height ?? window.height) - window.height, scrollY);
|
|
281
|
+
const target = y >= scrollY && y < scrollY + window.height ? scrollY : clamp(y - window.height / 2, 0, maxY);
|
|
791
282
|
const nextY = clamp(target, 0, maxY);
|
|
792
283
|
return {
|
|
793
284
|
x: clamp(x, 0, window.width - 1),
|
|
@@ -858,7 +349,7 @@ async function pressKey(cdp, event) {
|
|
|
858
349
|
...up
|
|
859
350
|
});
|
|
860
351
|
}
|
|
861
|
-
async function dispatchInput(cdp, action, window, page, scrollY = 0) {
|
|
352
|
+
async function dispatchInput(cdp, action, window, page, scrollY = 0, keyed = false) {
|
|
862
353
|
if (action.type === "done") return scrollY;
|
|
863
354
|
if (action.type === "type") {
|
|
864
355
|
if (action.replace) {
|
|
@@ -871,7 +362,13 @@ async function dispatchInput(cdp, action, window, page, scrollY = 0) {
|
|
|
871
362
|
});
|
|
872
363
|
if (!action.text) await pressKey(cdp, KEYS.Backspace);
|
|
873
364
|
}
|
|
874
|
-
if (action.text)
|
|
365
|
+
if (action.text && keyed) for (const text of action.text) await pressKey(cdp, {
|
|
366
|
+
key: text,
|
|
367
|
+
code: "",
|
|
368
|
+
windowsVirtualKeyCode: 0,
|
|
369
|
+
text
|
|
370
|
+
});
|
|
371
|
+
else if (action.text) await cdp.send("Input.insertText", { text: action.text });
|
|
875
372
|
return scrollY;
|
|
876
373
|
}
|
|
877
374
|
if (action.type === "key") {
|
|
@@ -883,12 +380,13 @@ async function dispatchInput(cdp, action, window, page, scrollY = 0) {
|
|
|
883
380
|
if (action.type === "scroll") {
|
|
884
381
|
await cdp.send("Input.dispatchMouseEvent", {
|
|
885
382
|
type: "mouseWheel",
|
|
886
|
-
x: action.x,
|
|
887
|
-
y: action.y,
|
|
383
|
+
x: clamp(action.x, 0, window.width - 1),
|
|
384
|
+
y: clamp(action.y - scrollY, 0, window.height - 1),
|
|
888
385
|
deltaX: 0,
|
|
889
386
|
deltaY: action.dy
|
|
890
387
|
});
|
|
891
|
-
|
|
388
|
+
const maxY = Math.max(0, (page?.height ?? window.height) - window.height);
|
|
389
|
+
return clamp(scrollY + action.dy, 0, Math.max(maxY, scrollY));
|
|
892
390
|
}
|
|
893
391
|
const mapped = mapPagePoint(action.type === "drag" ? action.x : action.x, action.type === "drag" ? action.y : action.y, window, page, scrollY);
|
|
894
392
|
if (mapped.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
|
|
@@ -1024,10 +522,12 @@ function requestId(msg) {
|
|
|
1024
522
|
*/
|
|
1025
523
|
function trackNetwork(cdp, quietMs = 500) {
|
|
1026
524
|
const inFlight = /* @__PURE__ */ new Map();
|
|
525
|
+
let started = 0;
|
|
1027
526
|
let changed;
|
|
1028
527
|
cdp.on("Network.requestWillBeSent", (msg) => {
|
|
1029
528
|
const id = requestId(msg);
|
|
1030
529
|
if (id === void 0) return;
|
|
530
|
+
started++;
|
|
1031
531
|
inFlight.set(id, Date.now());
|
|
1032
532
|
changed?.();
|
|
1033
533
|
});
|
|
@@ -1039,6 +539,8 @@ function trackNetwork(cdp, quietMs = 500) {
|
|
|
1039
539
|
cdp.on("Network.loadingFinished", end);
|
|
1040
540
|
cdp.on("Network.loadingFailed", end);
|
|
1041
541
|
return {
|
|
542
|
+
started: () => started,
|
|
543
|
+
pending: () => inFlight.size > 0,
|
|
1042
544
|
/** A request started in the last `FRESH_REQUEST_MS` is still in flight; a long poll is not fresh. */
|
|
1043
545
|
fresh() {
|
|
1044
546
|
const since = Date.now() - FRESH_REQUEST_MS;
|
|
@@ -1269,7 +771,7 @@ function waitForPortFile(file, timeoutMs, signal) {
|
|
|
1269
771
|
});
|
|
1270
772
|
}
|
|
1271
773
|
function pickPageDebuggerUrl(targets) {
|
|
1272
|
-
return targets.find((t) => t.webSocketDebuggerUrl && t.type
|
|
774
|
+
return targets.find((t) => t.webSocketDebuggerUrl && t.type === "page")?.webSocketDebuggerUrl;
|
|
1273
775
|
}
|
|
1274
776
|
async function pageDebuggerUrl(port, timeoutMs) {
|
|
1275
777
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -1359,6 +861,70 @@ async function disposeChrome(child, userData) {
|
|
|
1359
861
|
} catch {}
|
|
1360
862
|
removeChromeProfile(userData);
|
|
1361
863
|
}
|
|
864
|
+
/** Clipped text the page controls, in the words the model reads. */
|
|
865
|
+
const TARGET_CAP = 40;
|
|
866
|
+
const targetAt = (x, y) => `(() => {
|
|
867
|
+
const clip = (s) => String(s ?? '').replace(/\\s+/g, ' ').trim().slice(0, ${TARGET_CAP})
|
|
868
|
+
let e = document.elementFromPoint(${x}, ${y})
|
|
869
|
+
while (e && e.shadowRoot && e.shadowRoot.elementFromPoint) {
|
|
870
|
+
const inner = e.shadowRoot.elementFromPoint(${x}, ${y})
|
|
871
|
+
if (!inner || inner === e) break
|
|
872
|
+
e = inner
|
|
873
|
+
}
|
|
874
|
+
if (!e || e === document.documentElement || e === document.body) return 'the page background'
|
|
875
|
+
const control = e.closest('a,button,input,select,textarea,label,summary,[role],[onclick],[tabindex]') || e
|
|
876
|
+
const tag = control.tagName.toLowerCase()
|
|
877
|
+
const type = tag === 'input' ? '[' + (control.type || 'text') + ']' : ''
|
|
878
|
+
const name = clip(control.getAttribute('aria-label') || control.getAttribute('title') || (control.labels && control.labels[0] && control.labels[0].innerText) || control.getAttribute('placeholder') || (tag === 'input' || tag === 'select' || tag === 'textarea' ? '' : control.innerText) || control.getAttribute('alt'))
|
|
879
|
+
return tag + type + (name ? ' ' + JSON.stringify(name) : '')
|
|
880
|
+
})()`;
|
|
881
|
+
const SEGMENTED_FOCUS = `(() => {
|
|
882
|
+
let a = document.activeElement
|
|
883
|
+
for (;;) {
|
|
884
|
+
if (a && a.shadowRoot && a.shadowRoot.activeElement) { a = a.shadowRoot.activeElement; continue }
|
|
885
|
+
let inner = null
|
|
886
|
+
try { inner = a && a.contentDocument ? a.contentDocument.activeElement : null } catch {}
|
|
887
|
+
if (inner && inner !== a.contentDocument.body) { a = inner; continue }
|
|
888
|
+
break
|
|
889
|
+
}
|
|
890
|
+
return !!a && a.tagName === 'INPUT' && ['date', 'time', 'datetime-local', 'month', 'week'].includes(a.type)
|
|
891
|
+
})()`;
|
|
892
|
+
/** The window's scroll offset and the page's size, from CDP: no script runs in the page. */
|
|
893
|
+
async function readLayout(cdp) {
|
|
894
|
+
const metrics = await cdp.send("Page.getLayoutMetrics").catch(() => void 0);
|
|
895
|
+
const scrollY = metrics?.cssVisualViewport?.pageY;
|
|
896
|
+
const { width, height } = metrics?.cssContentSize ?? {};
|
|
897
|
+
if (typeof scrollY !== "number" || typeof width !== "number" || typeof height !== "number") return;
|
|
898
|
+
return {
|
|
899
|
+
scrollY: Math.round(scrollY),
|
|
900
|
+
width: Math.ceil(width),
|
|
901
|
+
height: Math.ceil(height)
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
/** How long a wheel's scroll may stand still before it counts as stopped short. */
|
|
905
|
+
const SCROLL_STILL_MS = 200;
|
|
906
|
+
const SCROLL_CAP_MS = 1500;
|
|
907
|
+
/**
|
|
908
|
+
* The window's offset once a wheel sent from `from` toward `want` has scrolled: the compositor
|
|
909
|
+
* scrolls after `Input.dispatchMouseEvent` returns. A page that stops short (its end, a locked
|
|
910
|
+
* body) stands still, and its offset then is the answer.
|
|
911
|
+
*/
|
|
912
|
+
async function scrolledTo(cdp, from, want) {
|
|
913
|
+
const start = Date.now();
|
|
914
|
+
let last = from;
|
|
915
|
+
let still = start;
|
|
916
|
+
for (;;) {
|
|
917
|
+
const y = (await readLayout(cdp))?.scrollY;
|
|
918
|
+
if (y === void 0) return want;
|
|
919
|
+
if (y === want) return y;
|
|
920
|
+
const now = Date.now();
|
|
921
|
+
if (y !== last) {
|
|
922
|
+
last = y;
|
|
923
|
+
still = now;
|
|
924
|
+
} else if (now - still >= SCROLL_STILL_MS || now - start >= SCROLL_CAP_MS) return y;
|
|
925
|
+
await new Promise((resolve) => setTimeout(resolve, 16));
|
|
926
|
+
}
|
|
927
|
+
}
|
|
1362
928
|
/**
|
|
1363
929
|
* Chrome's flags for `proxy`: every request goes to it, loopback ones included, which Chrome
|
|
1364
930
|
* otherwise never proxies.
|
|
@@ -1441,11 +1007,12 @@ async function withChrome(bin, window, timeoutMs, options, fn, signal) {
|
|
|
1441
1007
|
if (wait === "idle" && !nav) return network.quiet(waitMs);
|
|
1442
1008
|
return waitForPage(cdp, wait, waitMs, nav);
|
|
1443
1009
|
},
|
|
1444
|
-
async screenshot(fullPage) {
|
|
1010
|
+
async screenshot(fullPage, clip) {
|
|
1445
1011
|
for (let tries = 1;; tries++) try {
|
|
1446
1012
|
const shot = await cdp.send("Page.captureScreenshot", {
|
|
1447
1013
|
format: "png",
|
|
1448
|
-
captureBeyondViewport: fullPage
|
|
1014
|
+
captureBeyondViewport: fullPage,
|
|
1015
|
+
...clip ? { clip } : {}
|
|
1449
1016
|
});
|
|
1450
1017
|
if (!shot.data) throw new Error("Page.captureScreenshot returned no data");
|
|
1451
1018
|
return Buffer.from(shot.data, "base64");
|
|
@@ -1454,12 +1021,56 @@ async function withChrome(bin, window, timeoutMs, options, fn, signal) {
|
|
|
1454
1021
|
await new Promise((resolve) => setTimeout(resolve, SCREENSHOT_RETRY_MS));
|
|
1455
1022
|
}
|
|
1456
1023
|
},
|
|
1024
|
+
async reveal(action, page) {
|
|
1025
|
+
if (action.type !== "click" && action.type !== "hover" && action.type !== "drag") return false;
|
|
1026
|
+
const from = (await readLayout(cdp))?.scrollY ?? scrollY;
|
|
1027
|
+
const at = mapPagePoint(action.x, action.y, window, page, from);
|
|
1028
|
+
if (at.dy === 0) return false;
|
|
1029
|
+
await cdp.send("Input.dispatchMouseEvent", {
|
|
1030
|
+
type: "mouseWheel",
|
|
1031
|
+
x: window.width / 2,
|
|
1032
|
+
y: window.height / 2,
|
|
1033
|
+
deltaX: 0,
|
|
1034
|
+
deltaY: at.dy
|
|
1035
|
+
});
|
|
1036
|
+
scrollY = await scrolledTo(cdp, from, at.scrollY);
|
|
1037
|
+
return scrollY !== from;
|
|
1038
|
+
},
|
|
1457
1039
|
async dispatch(action, page) {
|
|
1458
|
-
|
|
1040
|
+
const from = (await readLayout(cdp))?.scrollY ?? scrollY;
|
|
1041
|
+
if (action.type !== "click" && action.type !== "hover" && action.type !== "drag") {
|
|
1042
|
+
const keyed = action.type === "type" && await cdp.send("Runtime.evaluate", {
|
|
1043
|
+
expression: SEGMENTED_FOCUS,
|
|
1044
|
+
returnByValue: true
|
|
1045
|
+
}).then((r) => r.result?.value === true).catch(() => false);
|
|
1046
|
+
scrollY = await dispatchInput(cdp, action, window, page, from, keyed);
|
|
1047
|
+
return { scrolled: false };
|
|
1048
|
+
}
|
|
1049
|
+
const at = mapPagePoint(action.x, action.y, window, page, from);
|
|
1050
|
+
if (at.dy !== 0) await cdp.send("Input.dispatchMouseEvent", {
|
|
1051
|
+
type: "mouseWheel",
|
|
1052
|
+
x: window.width / 2,
|
|
1053
|
+
y: window.height / 2,
|
|
1054
|
+
deltaX: 0,
|
|
1055
|
+
deltaY: at.dy
|
|
1056
|
+
});
|
|
1057
|
+
const after = at.dy !== 0 ? await scrolledTo(cdp, from, at.scrollY) : from;
|
|
1058
|
+
const point = mapPagePoint(action.x, action.y, window, page, after);
|
|
1059
|
+
const target = (await cdp.send("Runtime.evaluate", {
|
|
1060
|
+
expression: targetAt(Math.round(point.x), Math.round(point.y)),
|
|
1061
|
+
returnByValue: true
|
|
1062
|
+
}).catch(() => void 0))?.result?.value;
|
|
1063
|
+
scrollY = await dispatchInput(cdp, action, window, page, after);
|
|
1064
|
+
return {
|
|
1065
|
+
...typeof target === "string" ? { target } : {},
|
|
1066
|
+
scrolled: after !== from
|
|
1067
|
+
};
|
|
1459
1068
|
},
|
|
1460
|
-
|
|
1461
|
-
|
|
1069
|
+
layout: () => readLayout(cdp),
|
|
1070
|
+
busy(any) {
|
|
1071
|
+
return loading.size > 0 || (any ? network.pending() : network.fresh());
|
|
1462
1072
|
},
|
|
1073
|
+
requests: () => network.started(),
|
|
1463
1074
|
dialogs() {
|
|
1464
1075
|
return opened.splice(0);
|
|
1465
1076
|
},
|
|
@@ -2011,7 +1622,13 @@ const PROBE = `(() => {
|
|
|
2011
1622
|
const hash = (s) => { let h = 2166136261; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 16777619) } return (h >>> 0).toString(36) }
|
|
2012
1623
|
const clip = (s) => String(s ?? '').replace(/\\s+/g, ' ').trim().slice(0, ${TEXT_CAP})
|
|
2013
1624
|
let a = document.activeElement
|
|
2014
|
-
|
|
1625
|
+
for (;;) {
|
|
1626
|
+
if (a && a.shadowRoot && a.shadowRoot.activeElement) { a = a.shadowRoot.activeElement; continue }
|
|
1627
|
+
let inner = null
|
|
1628
|
+
try { inner = a && a.contentDocument ? a.contentDocument.activeElement : null } catch {}
|
|
1629
|
+
if (inner && inner !== a.contentDocument.body) { a = inner; continue }
|
|
1630
|
+
break
|
|
1631
|
+
}
|
|
2015
1632
|
if (a === document.body || a === document.documentElement) a = null
|
|
2016
1633
|
const tag = a ? a.tagName.toLowerCase() : ''
|
|
2017
1634
|
const type = a && tag === 'input' ? (a.type || 'text') : ''
|
|
@@ -2025,7 +1642,9 @@ const PROBE = `(() => {
|
|
|
2025
1642
|
: a.isContentEditable ? clip(a.innerText)
|
|
2026
1643
|
: null
|
|
2027
1644
|
const buttons = ['button', 'submit', 'reset', 'image', 'file', 'range', 'color', 'hidden']
|
|
2028
|
-
|
|
1645
|
+
// Focus inside a cross-origin frame or a closed shadow root shows only its host, which can't tell.
|
|
1646
|
+
const sealed = !!a && (tag === 'iframe' || tag === 'frame' || (!!a.attachShadow && !a.shadowRoot && tag.includes('-')))
|
|
1647
|
+
const editable = sealed ? null : !!a && ((tag === 'input' && !toggles && !buttons.includes(type)) || tag === 'textarea' || a.isContentEditable)
|
|
2029
1648
|
const controls = Array.from(document.querySelectorAll('input,select,textarea'))
|
|
2030
1649
|
.map((e) => (e.type === 'checkbox' || e.type === 'radio' ? (e.checked ? '1' : '0') : e.value))
|
|
2031
1650
|
return {
|
|
@@ -2084,20 +1703,27 @@ function describeChange(after, fields, origin) {
|
|
|
2084
1703
|
/**
|
|
2085
1704
|
* Waits for the page to stop changing after a move: two reads `SETTLE_POLL_MS` apart that agree,
|
|
2086
1705
|
* with no navigation or fresh request in flight. At `SETTLE_CAP_MS` it gives up and adds the
|
|
2087
|
-
* fields
|
|
1706
|
+
* fields that changed `VOLATILE_CHANGES` times or more to `volatile`, so a clock or an animation
|
|
1707
|
+
* doesn't hold up every move. A field that changed once (content that loaded just as the cap came,
|
|
1708
|
+
* behind requests that kept the page busy) is not volatile. With `anyRequest`, a request of any
|
|
1709
|
+
* age holds it too, as a first look waits for data a page fetches after `load`.
|
|
2088
1710
|
*/
|
|
2089
|
-
async function settledState(session, volatile, left) {
|
|
1711
|
+
async function settledState(session, volatile, left, anyRequest = false) {
|
|
2090
1712
|
const start = Date.now();
|
|
2091
1713
|
const sleep = () => new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(100, left()))));
|
|
1714
|
+
const changes = /* @__PURE__ */ new Map();
|
|
2092
1715
|
let prev = await probePage(session);
|
|
2093
1716
|
for (;;) {
|
|
2094
1717
|
if (left() <= 0) return prev;
|
|
2095
1718
|
await sleep();
|
|
2096
1719
|
const next = await probePage(session);
|
|
2097
1720
|
const moving = differing(prev, next, volatile);
|
|
2098
|
-
|
|
1721
|
+
for (const field of moving) changes.set(field, (changes.get(field) ?? 0) + 1);
|
|
1722
|
+
if (!moving.length && !session.busy(anyRequest)) return next;
|
|
2099
1723
|
if (Date.now() - start >= 3e3 || left() <= 0) {
|
|
2100
|
-
|
|
1724
|
+
const marked = [...changes].filter(([field, n]) => n >= 2 && field !== "url" && field !== "focus").map(([field]) => field);
|
|
1725
|
+
for (const field of marked) volatile.add(field);
|
|
1726
|
+
if (marked.length) debugLog("check", `capture page keeps changing on its own: ${marked.join(", ")}`);
|
|
2101
1727
|
return next;
|
|
2102
1728
|
}
|
|
2103
1729
|
prev = next;
|
|
@@ -2106,12 +1732,12 @@ async function settledState(session, volatile, left) {
|
|
|
2106
1732
|
/** How long a settled page is watched for a change still to come after a move that ends its instruction. */
|
|
2107
1733
|
const LATE_CHANGE_MS = 1e3;
|
|
2108
1734
|
/**
|
|
2109
|
-
* The page once a change that starts within `
|
|
1735
|
+
* The page once a change that starts within `windowMs` of `settled` has settled too, or
|
|
2110
1736
|
* `settled`: an auth callback's redirect or a save's message often comes a beat after the
|
|
2111
1737
|
* request that made it, and the next instruction's first look must not be the page before it.
|
|
2112
1738
|
*/
|
|
2113
|
-
async function lateSettledState(session, settled, volatile, left) {
|
|
2114
|
-
const until = Date.now() + Math.min(
|
|
1739
|
+
async function lateSettledState(session, settled, volatile, left, windowMs = LATE_CHANGE_MS) {
|
|
1740
|
+
const until = Date.now() + Math.min(windowMs, left());
|
|
2115
1741
|
while (Date.now() < until) {
|
|
2116
1742
|
await new Promise((resolve) => setTimeout(resolve, Math.min(100, left())));
|
|
2117
1743
|
if (differing(settled, await probePage(session), volatile).length || session.busy()) return settledState(session, volatile, left);
|
|
@@ -2169,9 +1795,12 @@ function onClock(session, clock) {
|
|
|
2169
1795
|
},
|
|
2170
1796
|
navigate: (url) => race(clock, () => session.navigate(url)),
|
|
2171
1797
|
wait: (wait, ms, nav) => race(clock, () => session.wait(wait, ms, nav).catch(own)),
|
|
2172
|
-
screenshot: (fullPage) => race(clock, () => session.screenshot(fullPage)),
|
|
1798
|
+
screenshot: (fullPage, clip) => race(clock, () => session.screenshot(fullPage, clip)),
|
|
1799
|
+
reveal: (action, page) => race(clock, () => session.reveal(action, page)),
|
|
2173
1800
|
dispatch: (action, page) => race(clock, () => session.dispatch(action, page)),
|
|
2174
|
-
|
|
1801
|
+
layout: () => race(clock, () => session.layout()),
|
|
1802
|
+
busy: (any) => session.busy(any),
|
|
1803
|
+
requests: () => session.requests(),
|
|
2175
1804
|
dialogs: () => session.dialogs(),
|
|
2176
1805
|
page: (origin) => session.page(origin)
|
|
2177
1806
|
};
|
|
@@ -2269,29 +1898,107 @@ function moveFromTool(name, input) {
|
|
|
2269
1898
|
*/
|
|
2270
1899
|
const normalizesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
|
|
2271
1900
|
const NORMALIZED = 1e3;
|
|
2272
|
-
/**
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
1901
|
+
/**
|
|
1902
|
+
* The most pixels and the longest edge of the screenshot a model sees. Anthropic's API scales a
|
|
1903
|
+
* larger image down and its models then answer in the smaller image's pixels, so reeval scales it
|
|
1904
|
+
* itself and maps the points back.
|
|
1905
|
+
*/
|
|
1906
|
+
const SHOT_PIXELS = 115e4;
|
|
1907
|
+
const SHOT_EDGE = 1568;
|
|
1908
|
+
/** The scale that keeps a `width`×`height` image within `SHOT_PIXELS` and `SHOT_EDGE`. */
|
|
1909
|
+
function shotScale(width, height) {
|
|
1910
|
+
const fit = Math.min(1, Math.sqrt(SHOT_PIXELS / (width * height)), SHOT_EDGE / Math.max(width, height));
|
|
1911
|
+
return Math.floor(fit * 1e3) / 1e3;
|
|
1912
|
+
}
|
|
1913
|
+
/** What a model is shown of `layout`: the whole page, unless it is over `TALL_PAGE` windows tall. */
|
|
1914
|
+
function shotRegion(layout, window) {
|
|
1915
|
+
const height = Math.max(layout.height, window.height);
|
|
1916
|
+
if (height <= 2 * window.height) return {
|
|
1917
|
+
y: 0,
|
|
1918
|
+
width: window.width,
|
|
1919
|
+
height,
|
|
1920
|
+
partial: false
|
|
1921
|
+
};
|
|
1922
|
+
return {
|
|
1923
|
+
y: Math.min(Math.max(0, layout.scrollY), height - window.height),
|
|
1924
|
+
width: window.width,
|
|
1925
|
+
height: window.height,
|
|
1926
|
+
partial: true
|
|
1927
|
+
};
|
|
1928
|
+
}
|
|
1929
|
+
async function modelShot(session) {
|
|
1930
|
+
const layout = await session.layout();
|
|
1931
|
+
if (!layout) {
|
|
1932
|
+
const png = await session.screenshot(true);
|
|
1933
|
+
const size = pngSize(png);
|
|
1934
|
+
return {
|
|
1935
|
+
png,
|
|
1936
|
+
image: size,
|
|
1937
|
+
region: {
|
|
1938
|
+
y: 0,
|
|
1939
|
+
...size
|
|
1940
|
+
},
|
|
1941
|
+
page: size,
|
|
1942
|
+
partial: false
|
|
1943
|
+
};
|
|
1944
|
+
}
|
|
1945
|
+
const { partial, ...region } = shotRegion(layout, session.window);
|
|
1946
|
+
const scale = shotScale(region.width, region.height);
|
|
1947
|
+
const png = await session.screenshot(!partial, {
|
|
1948
|
+
x: 0,
|
|
1949
|
+
...region,
|
|
1950
|
+
scale
|
|
1951
|
+
});
|
|
1952
|
+
const page = {
|
|
1953
|
+
width: Math.max(layout.width, session.window.width),
|
|
1954
|
+
height: Math.max(layout.height, session.window.height)
|
|
1955
|
+
};
|
|
1956
|
+
return {
|
|
1957
|
+
png,
|
|
1958
|
+
image: pngSize(png),
|
|
1959
|
+
region,
|
|
1960
|
+
page,
|
|
1961
|
+
partial
|
|
1962
|
+
};
|
|
1963
|
+
}
|
|
1964
|
+
/**
|
|
1965
|
+
* An action's points, from the model's screenshot to page CSS pixels: 0–1000 across the image
|
|
1966
|
+
* for a model that normalizes, the image's pixels for any other. A point off the image is refused.
|
|
1967
|
+
*/
|
|
1968
|
+
function toPagePoints(action, shot, normalized) {
|
|
1969
|
+
const { image, region } = shot;
|
|
1970
|
+
const span = normalized ? {
|
|
1971
|
+
width: NORMALIZED,
|
|
1972
|
+
height: NORMALIZED
|
|
1973
|
+
} : image;
|
|
1974
|
+
const off = (px, py) => px < 0 || py < 0 || px > span.width || py > span.height;
|
|
1975
|
+
const x = (n) => Math.round(n * region.width / span.width);
|
|
1976
|
+
const y = (n) => Math.round(region.y + n * region.height / span.height);
|
|
1977
|
+
const refuse = (px, py) => ({ refused: `(${px}, ${py}) is outside the screenshot, which is ${normalized ? `0-${NORMALIZED} on both axes` : `${image.width}×${image.height}`}` });
|
|
2276
1978
|
switch (action.type) {
|
|
2277
1979
|
case "click":
|
|
2278
1980
|
case "hover":
|
|
2279
|
-
case "scroll":
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
y
|
|
2288
|
-
toX
|
|
2289
|
-
|
|
2290
|
-
|
|
1981
|
+
case "scroll":
|
|
1982
|
+
if (off(action.x, action.y)) return refuse(action.x, action.y);
|
|
1983
|
+
return {
|
|
1984
|
+
...action,
|
|
1985
|
+
x: x(action.x),
|
|
1986
|
+
y: y(action.y)
|
|
1987
|
+
};
|
|
1988
|
+
case "drag":
|
|
1989
|
+
if (off(action.x, action.y)) return refuse(action.x, action.y);
|
|
1990
|
+
if (off(action.toX, action.toY)) return refuse(action.toX, action.toY);
|
|
1991
|
+
return {
|
|
1992
|
+
...action,
|
|
1993
|
+
x: x(action.x),
|
|
1994
|
+
y: y(action.y),
|
|
1995
|
+
toX: x(action.toX),
|
|
1996
|
+
toY: y(action.toY)
|
|
1997
|
+
};
|
|
2291
1998
|
default: return action;
|
|
2292
1999
|
}
|
|
2293
2000
|
}
|
|
2294
|
-
const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the
|
|
2001
|
+
const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the screenshot: x and toX across its width, y and toY down its height.` : "Coordinates are pixels on the screenshot, whose size each turn gives."} A scroll's dy is in page pixels, positive to scroll down. Each turn you get the goal, what has focus, your earlier moves with what each did, and a fresh screenshot. Click a field before typing into it, and do not repeat a move that already worked. A dropdown (select) never opens in the screenshot: click it once, press ArrowDown or ArrowUp until it shows the option you want, then press Enter to choose it. On a click or key press after which nothing of the goal is left, such as the click that submits a form you have filled in, set completesGoal to true: reeval makes it and goes on to the next goal without asking you again. A goal with more than one step (sign in and allow access, fill in and confirm) is complete only on its last step's move, not on a submit that leads to the next step. Call done when the goal is met. Do not mention CDP.`;
|
|
2295
2002
|
const completesGoal = z.boolean().optional().describe("True only on the last move the goal needs: nothing of the goal is left after it.");
|
|
2296
2003
|
const useTools = {
|
|
2297
2004
|
click: tool({
|
|
@@ -2317,7 +2024,7 @@ const useTools = {
|
|
|
2317
2024
|
})
|
|
2318
2025
|
}),
|
|
2319
2026
|
scroll: tool({
|
|
2320
|
-
description: "Wheel-scroll
|
|
2027
|
+
description: "Wheel-scroll by dy page pixels with the pointer at (x, y): the page, or the scrolling box under the point.",
|
|
2321
2028
|
inputSchema: z.object({
|
|
2322
2029
|
x: z.number(),
|
|
2323
2030
|
y: z.number(),
|
|
@@ -2412,7 +2119,8 @@ function moveKey({ final: _, ...action }) {
|
|
|
2412
2119
|
/** Sets what `move` did, from its page before and `after`. */
|
|
2413
2120
|
function settleMove(move, after, origin, volatile) {
|
|
2414
2121
|
move.after = after;
|
|
2415
|
-
const
|
|
2122
|
+
const ignored = move.scrolled ? /* @__PURE__ */ new Set([...volatile, "view"]) : volatile;
|
|
2123
|
+
const fields = move.before ? differing(move.before, after, ignored) : [];
|
|
2416
2124
|
const notes = move.before ? describeChange(after, fields, origin) : [];
|
|
2417
2125
|
for (const dialog of move.dialogs) notes.push(`the page opened ${dialog}, which reeval accepted`);
|
|
2418
2126
|
move.changed = fields.length > 0 || move.dialogs.length > 0;
|
|
@@ -2429,6 +2137,23 @@ function stuckMoves(moves) {
|
|
|
2429
2137
|
}
|
|
2430
2138
|
return n;
|
|
2431
2139
|
}
|
|
2140
|
+
/**
|
|
2141
|
+
* A focused date or time input, as the probe names it. Only these get the no-separators hint,
|
|
2142
|
+
* which other fields would take as theirs too, dropping the spaces and slashes a card needs.
|
|
2143
|
+
*/
|
|
2144
|
+
const SEGMENTED = /^input\[(date|time|datetime-local|month|week)\]/;
|
|
2145
|
+
/**
|
|
2146
|
+
* How to type into a focused date or time input. Chrome's year segment takes up to six digits and
|
|
2147
|
+
* a month input's month segment waits for more, so a part followed by another needs ArrowRight.
|
|
2148
|
+
*/
|
|
2149
|
+
function segmentedHint(focus) {
|
|
2150
|
+
const kind = SEGMENTED.exec(focus)?.[1];
|
|
2151
|
+
if (!kind) return void 0;
|
|
2152
|
+
const first = "Click its left edge, its first part, before typing.";
|
|
2153
|
+
if (kind === "datetime-local") return `A date and time field takes the date's digits with no separators, then ArrowRight, then the time's: type 11142026, press ArrowRight, type 0730PM. ${first}`;
|
|
2154
|
+
if (kind === "month") return `A month field takes the month's number, then ArrowRight, then the year: type 11, press ArrowRight, type 2026. ${first}`;
|
|
2155
|
+
return `A date or time field takes its digits in the order it shows them, with no separators: 11142026 for mm/dd/yyyy, 0730PM for --:-- --, 462026 for Week --, ----. ${first}`;
|
|
2156
|
+
}
|
|
2432
2157
|
/** What the model reads about the page now: where it is and what has focus. */
|
|
2433
2158
|
function stateLines(now, origin, volatile) {
|
|
2434
2159
|
const lines = [];
|
|
@@ -2438,14 +2163,24 @@ function stateLines(now, origin, volatile) {
|
|
|
2438
2163
|
else if (now.focus) {
|
|
2439
2164
|
lines.push(`Focused: ${now.focus}${now.value ? `, holding ${JSON.stringify(now.value)}` : ""}.`);
|
|
2440
2165
|
if (now.focus.startsWith("select")) lines.push("A dropdown never opens in the screenshot: press ArrowDown or ArrowUp to the option, then Enter; it holds the new option only after Enter.");
|
|
2166
|
+
const hint = segmentedHint(now.focus);
|
|
2167
|
+
if (hint) lines.push(hint);
|
|
2441
2168
|
}
|
|
2442
2169
|
if (volatile.has("text") || volatile.has("nodes") || volatile.has("view")) lines.push("Parts of the page keep changing on their own, so the effects below leave them out.");
|
|
2443
2170
|
return lines;
|
|
2444
2171
|
}
|
|
2445
2172
|
/** The goal, where the journey is, the page now, and the moves so far with what each did. */
|
|
2446
|
-
function goalPrompt(step, earlier, now, moves, origin, volatile = /* @__PURE__ */ new Set()) {
|
|
2173
|
+
function goalPrompt(step, earlier, now, moves, origin, volatile = /* @__PURE__ */ new Set(), view = {}) {
|
|
2447
2174
|
const lines = [`Goal: ${step.goal}`];
|
|
2448
|
-
|
|
2175
|
+
const quoted = (goals) => goals.map((g) => JSON.stringify(g)).join(", ");
|
|
2176
|
+
if (step.total > 1) {
|
|
2177
|
+
const done = earlier.length ? ` Already done: ${quoted(earlier)}.` : "";
|
|
2178
|
+
const later = view.later?.length ? ` Still to come, not part of this goal: ${quoted(view.later)}.` : "";
|
|
2179
|
+
lines.push(`This is instruction ${step.index} of ${step.total}.${done}${later}`);
|
|
2180
|
+
}
|
|
2181
|
+
const { shot } = view;
|
|
2182
|
+
if (shot && !view.normalized) lines.push(`The screenshot is ${shot.image.width}×${shot.image.height} pixels.`);
|
|
2183
|
+
if (shot?.partial) lines.push(`The page is ${shot.page.height} px tall, more than the screenshot shows: it shows the window, scrolled ${shot.region.y} px down. Scroll to see the rest.`);
|
|
2449
2184
|
lines.push(...stateLines(now, origin, volatile));
|
|
2450
2185
|
if (!moves.length) {
|
|
2451
2186
|
lines.push("You have made no moves for this goal yet.");
|
|
@@ -2454,22 +2189,24 @@ function goalPrompt(step, earlier, now, moves, origin, volatile = /* @__PURE__ *
|
|
|
2454
2189
|
lines.push("Your moves for this goal so far, oldest first. The screenshot shows the page now.");
|
|
2455
2190
|
moves.forEach((m, i) => {
|
|
2456
2191
|
const effect = m.refused ? `not made: ${m.refused}` : m.effect ?? "";
|
|
2457
|
-
lines.push(`${i + 1}. ${m.said}${effect ? ` → ${effect}` : ""}`);
|
|
2192
|
+
lines.push(`${i + 1}. ${m.said}${m.target ? ` on ${m.target}` : ""}${effect ? ` → ${effect}` : ""}`);
|
|
2458
2193
|
});
|
|
2194
|
+
if (moves.at(-1)?.unproven) lines.push(`Move ${moves.length} was marked completesGoal, but it sent no request and opened no page or dialog, so the goal goes on. If the goal is met, call done; if that move only opened a step on the way, such as a confirmation, finish it.`);
|
|
2459
2195
|
if (stuckMoves(moves) >= 2) lines.push("Your last moves repeated the same action and nothing changed. Do not repeat it: if the goal is met, call done; otherwise try a different action.");
|
|
2460
2196
|
const from = pathOf(moves.find((m) => m.before)?.before?.url, origin);
|
|
2461
2197
|
const at = pathOf(now.url, origin);
|
|
2462
2198
|
if (from && at && from !== at) lines.push(`Your moves took the page from ${from} to ${at}. If they did what the goal asks, call done: a goal's "once", "if" or "when" says where it starts, so do not undo them to get back there.`);
|
|
2463
2199
|
return lines.join("\n");
|
|
2464
2200
|
}
|
|
2465
|
-
async function runModelGoal(session, { step, earlier, origin, modelName, clock, volatile, pristine, loadModel }) {
|
|
2201
|
+
async function runModelGoal(session, { step, earlier, origin, modelName, clock, volatile, pristine, later, loadModel }) {
|
|
2466
2202
|
const { goal } = step;
|
|
2467
2203
|
const model = await loadModel(modelName);
|
|
2468
2204
|
const normalized = normalizesPoints(modelName);
|
|
2469
2205
|
const made = [];
|
|
2470
2206
|
let dispatched = 0;
|
|
2471
2207
|
let usage = null;
|
|
2472
|
-
let now = await probePage(session);
|
|
2208
|
+
let now = pristine ? await settledState(session, volatile, clock.remaining, true) : await probePage(session);
|
|
2209
|
+
const quiet = pristine && !session.busy(true) ? session.requests() : void 0;
|
|
2473
2210
|
let stale = 0;
|
|
2474
2211
|
const trail = {
|
|
2475
2212
|
origin,
|
|
@@ -2486,18 +2223,19 @@ async function runModelGoal(session, { step, earlier, origin, modelName, clock,
|
|
|
2486
2223
|
check(clock);
|
|
2487
2224
|
const last = made.at(-1);
|
|
2488
2225
|
if (last?.after) {
|
|
2489
|
-
|
|
2226
|
+
let late = await probePage(session);
|
|
2227
|
+
if (differing(last.after, late, volatile).length || session.busy()) late = await settledState(session, volatile, clock.remaining);
|
|
2490
2228
|
last.dialogs.push(...session.dialogs());
|
|
2491
2229
|
if (differing(last.after, late, volatile).length || last.dialogs.length) settleMove(last, late, origin, volatile);
|
|
2492
2230
|
now = late;
|
|
2493
2231
|
}
|
|
2494
|
-
if (stuckMoves(made) >= 5) throw new Error(`c.capture() instruction ${step.index} of ${step.total} ${JSON.stringify(goal)} repeated ${last?.said} 5 times and nothing changed`);
|
|
2495
|
-
const
|
|
2496
|
-
const
|
|
2232
|
+
if (stuckMoves(made) >= 5) throw new Error(`c.capture() instruction ${step.index} of ${step.total} ${JSON.stringify(goal)} repeated ${last?.said}${last?.target ? ` on ${last.target}` : ""} 5 times and nothing changed`);
|
|
2233
|
+
const shot = await modelShot(session);
|
|
2234
|
+
const { png: ephemeral, ...seen } = shot;
|
|
2497
2235
|
debugTrace("check", "capture-shot", {
|
|
2498
2236
|
goal,
|
|
2499
2237
|
bytes: ephemeral.length,
|
|
2500
|
-
|
|
2238
|
+
...seen,
|
|
2501
2239
|
url: now.url
|
|
2502
2240
|
}, false);
|
|
2503
2241
|
const signal = AbortSignal.timeout(check(clock));
|
|
@@ -2510,7 +2248,11 @@ async function runModelGoal(session, { step, earlier, origin, modelName, clock,
|
|
|
2510
2248
|
role: "user",
|
|
2511
2249
|
content: [{
|
|
2512
2250
|
type: "text",
|
|
2513
|
-
text: goalPrompt(step, earlier, now, made, origin, volatile
|
|
2251
|
+
text: goalPrompt(step, earlier, now, made, origin, volatile, {
|
|
2252
|
+
shot: seen,
|
|
2253
|
+
normalized,
|
|
2254
|
+
later
|
|
2255
|
+
})
|
|
2514
2256
|
}, {
|
|
2515
2257
|
type: "file",
|
|
2516
2258
|
mediaType: "image/png",
|
|
@@ -2532,10 +2274,12 @@ async function runModelGoal(session, { step, earlier, origin, modelName, clock,
|
|
|
2532
2274
|
const said = `${call.toolName} ${JSON.stringify(input)}`;
|
|
2533
2275
|
const fresh = await probePage(session);
|
|
2534
2276
|
const opened = session.dialogs();
|
|
2535
|
-
|
|
2536
|
-
if (!last &&
|
|
2277
|
+
let moved = differing(now, fresh, volatile);
|
|
2278
|
+
if (!last && quiet !== void 0 && !opened.length && session.requests() === quiet) {
|
|
2537
2279
|
for (const f of moved) if (f !== "url" && f !== "focus") volatile.add(f);
|
|
2538
|
-
|
|
2280
|
+
moved = differing(now, fresh, volatile);
|
|
2281
|
+
}
|
|
2282
|
+
if ((moved.length || opened.length) && stale < STALE_ASKS) {
|
|
2539
2283
|
stale++;
|
|
2540
2284
|
if (last && !last.refused) {
|
|
2541
2285
|
last.dialogs.push(...opened);
|
|
@@ -2546,9 +2290,10 @@ async function runModelGoal(session, { step, earlier, origin, modelName, clock,
|
|
|
2546
2290
|
continue;
|
|
2547
2291
|
}
|
|
2548
2292
|
stale = 0;
|
|
2549
|
-
|
|
2293
|
+
const tool = actionFromTool(call.toolName, input);
|
|
2294
|
+
let asks = "refused" in tool ? tool : toPagePoints(tool, shot, normalized);
|
|
2550
2295
|
if (!("refused" in asks) && asks.type === "type" && now.editable === false) asks = { refused: "nothing that takes text is focused; click a field first" };
|
|
2551
|
-
const logged = !("refused" in asks) && asks.type === "type" && now.focus?.includes("[password]") ? `type ${JSON.stringify({
|
|
2296
|
+
const logged = !("refused" in asks) && asks.type === "type" && (now.focus?.includes("[password]") || now.editable === null) ? `type ${JSON.stringify({
|
|
2552
2297
|
...input,
|
|
2553
2298
|
text: "•".repeat(String(input.text ?? "").length)
|
|
2554
2299
|
})}` : said;
|
|
@@ -2563,13 +2308,13 @@ async function runModelGoal(session, { step, earlier, origin, modelName, clock,
|
|
|
2563
2308
|
});
|
|
2564
2309
|
continue;
|
|
2565
2310
|
}
|
|
2566
|
-
const
|
|
2311
|
+
const page = shot.page;
|
|
2567
2312
|
debugLog("check", `capture use ${logged} on ${JSON.stringify(goal)} · ${latency}`);
|
|
2568
2313
|
debugTrace("check", "capture-use", {
|
|
2569
2314
|
goal,
|
|
2570
|
-
action:
|
|
2315
|
+
action: asks,
|
|
2571
2316
|
page
|
|
2572
|
-
}, `capture use ${
|
|
2317
|
+
}, `capture use ${asks.type}`);
|
|
2573
2318
|
if (asks.type === "done") return {
|
|
2574
2319
|
moves: dispatched,
|
|
2575
2320
|
usage,
|
|
@@ -2582,18 +2327,30 @@ async function runModelGoal(session, { step, earlier, origin, modelName, clock,
|
|
|
2582
2327
|
dialogs: []
|
|
2583
2328
|
};
|
|
2584
2329
|
made.push(move);
|
|
2585
|
-
|
|
2330
|
+
let sent = session.requests();
|
|
2331
|
+
if (await session.reveal(asks, page)) {
|
|
2332
|
+
move.before = await settledState(session, volatile, clock.remaining);
|
|
2333
|
+
sent = session.requests();
|
|
2334
|
+
}
|
|
2335
|
+
const landed = await session.dispatch(asks, page);
|
|
2336
|
+
if (landed?.target) move.target = landed.target;
|
|
2337
|
+
if (landed?.scrolled) move.scrolled = true;
|
|
2586
2338
|
dispatched++;
|
|
2587
2339
|
now = await settledState(session, volatile, clock.remaining);
|
|
2340
|
+
if (asks.type === "type" || asks.type === "hover") now = await lateSettledState(session, now, volatile, clock.remaining, 700);
|
|
2588
2341
|
move.dialogs.push(...session.dialogs());
|
|
2589
2342
|
settleMove(move, now, origin, volatile);
|
|
2590
2343
|
if (asks.final && move.changed) {
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2344
|
+
const left = now.url !== move.before?.url;
|
|
2345
|
+
if (session.requests() > sent || left || move.dialogs.length) {
|
|
2346
|
+
await lateSettledState(session, now, volatile, clock.remaining);
|
|
2347
|
+
return {
|
|
2348
|
+
moves: dispatched,
|
|
2349
|
+
usage,
|
|
2350
|
+
ended: "final"
|
|
2351
|
+
};
|
|
2352
|
+
}
|
|
2353
|
+
move.unproven = true;
|
|
2597
2354
|
}
|
|
2598
2355
|
}
|
|
2599
2356
|
clock.step = {
|
|
@@ -2676,6 +2433,7 @@ async function runJourney(raw, url, options) {
|
|
|
2676
2433
|
clock,
|
|
2677
2434
|
volatile,
|
|
2678
2435
|
pristine: moves === 0,
|
|
2436
|
+
later: goals.slice(stringIndex + 1),
|
|
2679
2437
|
loadModel: options.loadModel
|
|
2680
2438
|
});
|
|
2681
2439
|
usage = addUsage(usage, goal.usage);
|
|
@@ -2747,30 +2505,54 @@ async function acceptIntoSnapshot(sandbox, path, signal) {
|
|
|
2747
2505
|
const r = await sandbox.execOwn(`${snapshotEnv(sandbox)} && export GIT_LITERAL_PATHSPECS=1 && ${GIT} add -f -- ${quoted} && ${next} ${GIT} read-tree HEAD && ${next} ${GIT} add -f -- ${quoted} && tree=$(${next} ${GIT} write-tree) && rm -f "$GIT_DIR/next-index" && ${RESET_HEAD}`, { signal });
|
|
2748
2506
|
if (r.exitCode !== 0) throw new InfraError(`Failed to keep ${path} out of the diff: ${r.stderr.trim()}`);
|
|
2749
2507
|
}
|
|
2750
|
-
async function
|
|
2751
|
-
const
|
|
2752
|
-
|
|
2753
|
-
const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames -z HEAD`, { signal });
|
|
2754
|
-
if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
|
|
2508
|
+
async function rawChanges(sandbox, signal) {
|
|
2509
|
+
const r = await sandbox.execOwn(`${snapshotEnv(sandbox)} && ${GIT} add -A && ${GIT} diff --cached --raw --no-renames --no-abbrev -z HEAD`, { signal });
|
|
2510
|
+
if (r.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${r.stderr}`);
|
|
2755
2511
|
const status = {
|
|
2756
2512
|
A: "added",
|
|
2757
2513
|
M: "modified",
|
|
2758
2514
|
D: "deleted"
|
|
2759
2515
|
};
|
|
2760
|
-
const fields =
|
|
2761
|
-
const
|
|
2516
|
+
const fields = r.stdout.split("\0");
|
|
2517
|
+
const changes = [];
|
|
2762
2518
|
for (let i = 0; i + 1 < fields.length; i += 2) {
|
|
2763
|
-
const code = fields[i];
|
|
2764
|
-
|
|
2519
|
+
const [, mode = "", , id = "", code = ""] = fields[i].split(" ");
|
|
2520
|
+
changes.push({
|
|
2765
2521
|
path: fields[i + 1],
|
|
2766
|
-
status: status[code[0]] ?? "modified"
|
|
2522
|
+
status: status[code[0]] ?? "modified",
|
|
2523
|
+
blob: `${mode} ${id}`
|
|
2767
2524
|
});
|
|
2768
2525
|
}
|
|
2526
|
+
return changes;
|
|
2527
|
+
}
|
|
2528
|
+
const blobsOf = (changes) => new Map(changes.map((c) => [c.path, c.blob]));
|
|
2529
|
+
async function diffSinceSnapshot(sandbox, signal) {
|
|
2530
|
+
const patch = await sandbox.execOwn(`${snapshotEnv(sandbox)} && ${GIT} add -A && ${GIT} diff --cached --binary HEAD`, { signal });
|
|
2531
|
+
if (patch.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr}`);
|
|
2532
|
+
const changes = await rawChanges(sandbox, signal);
|
|
2533
|
+
const files = changes.map(({ path, status }) => ({
|
|
2534
|
+
path,
|
|
2535
|
+
status
|
|
2536
|
+
}));
|
|
2769
2537
|
return {
|
|
2770
|
-
|
|
2771
|
-
|
|
2538
|
+
diff: {
|
|
2539
|
+
patch: patch.stdout,
|
|
2540
|
+
files
|
|
2541
|
+
},
|
|
2542
|
+
blobs: blobsOf(changes)
|
|
2772
2543
|
};
|
|
2773
2544
|
}
|
|
2545
|
+
/** The workspace's `Blobs` now, to tell later which files a turn changed. */
|
|
2546
|
+
async function workspaceBlobs(sandbox, signal) {
|
|
2547
|
+
return blobsOf(await rawChanges(sandbox, signal));
|
|
2548
|
+
}
|
|
2549
|
+
/**
|
|
2550
|
+
* Paths whose content or mode differs between two `Blobs`: a file changed, created or deleted
|
|
2551
|
+
* between them.
|
|
2552
|
+
*/
|
|
2553
|
+
function changedBetween(before, after) {
|
|
2554
|
+
return [.../* @__PURE__ */ new Set([...after.keys(), ...before.keys()])].filter((p) => before.get(p) !== after.get(p));
|
|
2555
|
+
}
|
|
2774
2556
|
/** Every file in the workspace as git sees it (its `.gitignore` applies), sorted. */
|
|
2775
2557
|
async function workspaceFiles(sandbox, signal) {
|
|
2776
2558
|
const r = await sandbox.execOwn(`${snapshotEnv(sandbox)} && ${GIT} add -A && ${GIT} ls-files -z`, { signal });
|
|
@@ -2847,7 +2629,7 @@ function createChecks(a) {
|
|
|
2847
2629
|
function exec(command, options) {
|
|
2848
2630
|
const problem = options === void 0 ? void 0 : execOptionsProblem(options);
|
|
2849
2631
|
if (problem) return Promise.reject(new UsageError(problem));
|
|
2850
|
-
if (options?.background) return import("./capture-
|
|
2632
|
+
if (options?.background) return import("./capture-YPwxp_iK.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
|
|
2851
2633
|
const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
|
|
2852
2634
|
return Object.defineProperty(run, SUBJECT, { value: command });
|
|
2853
2635
|
}
|
|
@@ -2957,12 +2739,13 @@ async function launch(a, command, who, extraEnv) {
|
|
|
2957
2739
|
* the one port that wasn't listening before it started, which it records.
|
|
2958
2740
|
*/
|
|
2959
2741
|
async function serveCommand(a, command, port) {
|
|
2960
|
-
const want = port ?? await allocatePort();
|
|
2961
2742
|
const box = await sandboxFor(a, "c.serve()");
|
|
2962
2743
|
const tables = a.spec.sandbox.kind !== "fake";
|
|
2963
2744
|
const look = (pid) => untraced(() => box.exec(probeScript(pid, tables), { signal: a.signal })).then((r) => parseProbe(r.stdout));
|
|
2964
2745
|
const first = await look();
|
|
2965
2746
|
const before = new Set(first.proc ? first.listeners.map((l) => l.port) : []);
|
|
2747
|
+
const taken = async (p) => first.proc ? before.has(p) : !tables && await hostListening(p) !== void 0;
|
|
2748
|
+
const want = await servePort(command, port, allocatePort, taken);
|
|
2966
2749
|
const { handle, pid, log } = await launch(a, command, "c.serve()", { PORT: String(want) });
|
|
2967
2750
|
const found = await awaitListening(command, {
|
|
2968
2751
|
want,
|
|
@@ -3205,6 +2988,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs, pages) {
|
|
|
3205
2988
|
for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
|
|
3206
2989
|
for (const [name, json] of pages) await writeFile(join(dir, `${name}.capture.json`), json);
|
|
3207
2990
|
}
|
|
3208
|
-
export { writeCaptureSidecars as a, sandboxFor as c,
|
|
2991
|
+
export { writeCaptureSidecars as a, sandboxFor as c, snapshot as d, workspaceBlobs as f, startBackground as i, changedBetween as l, fakeInstructions as m, msysTreeKill as n, asCheck as o, workspaceFiles as p, serveCommand as r, createChecks as s, capturePage as t, diffSinceSnapshot as u };
|
|
3209
2992
|
|
|
3210
|
-
//# sourceMappingURL=capture-
|
|
2993
|
+
//# sourceMappingURL=capture-C3RmjPph.js.map
|