reeval 0.15.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
- package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
- package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
- package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
- package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
- package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
- package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
- package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
- package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
- package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +63 -16
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
- package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
- package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
- package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
- package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
- package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
- package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
- package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
- package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
- package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +52 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
- package/dist/view/index.d.ts +16 -4
- package/dist/view/index.js +6 -6
- package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -1,29 +1,31 @@
|
|
|
1
|
-
import { n as isOneOf } from "./object-
|
|
1
|
+
import { n as isOneOf } from "./object-BkXGPKWY.js";
|
|
2
2
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
3
|
import { c as jsonValueSchema, h as parseDuration, i as durationSchema, m as TIMER_LIMIT, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
4
4
|
import { y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
|
|
5
5
|
import { l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
6
|
-
import {
|
|
6
|
+
import { It as sharedInputFields, Lt as turnsSchema, Nt as judgeFields, Ot as captureWindowSchema, Pt as judgeModelNameSchema, at as awaitListening, b as resolveModelGateway, jt as fakeJudgeDataSchema, ot as parseProbe, st as probeScript, v as gatewayScope } from "./config-CneBZB84.js";
|
|
7
7
|
import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
8
8
|
import { m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
9
|
-
import { i as formatDuration, n as clip } from "./text-DTigcA64.js";
|
|
9
|
+
import { a as plural, i as formatDuration, n as clip } from "./text-DTigcA64.js";
|
|
10
10
|
import { s as cacheDir } from "./root-Dlff-zp8.js";
|
|
11
|
-
import {
|
|
11
|
+
import { pt as shellQuote } from "./src-C0ySiej5.js";
|
|
12
12
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
13
|
-
import { r as getRunContext, s as SUBJECT } from "./runtime-
|
|
14
|
-
import { i as debugTrace, r as debugLog } from "./attempt-
|
|
15
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
16
|
-
import { i as hostModel } from "./model-
|
|
13
|
+
import { r as getRunContext, s as SUBJECT } from "./runtime-CZVpkr04.js";
|
|
14
|
+
import { i as debugTrace, r as debugLog } from "./attempt-CpbZ9arV.js";
|
|
15
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
|
|
16
|
+
import { i as hostModel } from "./model-2rBZrdZA.js";
|
|
17
17
|
import { t as addUsage } from "./usage-BNY1ABrU.js";
|
|
18
|
+
import { t as untraced } from "./docker-BihzJwbV.js";
|
|
18
19
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
19
20
|
import { homedir, tmpdir } from "node:os";
|
|
20
21
|
import { join, posix } from "node:path";
|
|
21
22
|
import { z } from "zod";
|
|
22
|
-
import { createServer } from "node:net";
|
|
23
|
+
import net, { createServer } from "node:net";
|
|
23
24
|
import { createHash, randomUUID } from "node:crypto";
|
|
24
25
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
25
26
|
import { spawn } from "node:child_process";
|
|
26
|
-
import
|
|
27
|
+
import http from "node:http";
|
|
28
|
+
import { crc32, deflateSync, inflateRawSync, inflateSync } from "node:zlib";
|
|
27
29
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
28
30
|
import { generateText, stepCountIs, tool } from "ai";
|
|
29
31
|
//#region src/contract/attempt.ts
|
|
@@ -170,6 +172,7 @@ const assertionRecordSchema = z.strictObject({
|
|
|
170
172
|
"error"
|
|
171
173
|
]),
|
|
172
174
|
soft: z.boolean(),
|
|
175
|
+
at: z.string().optional(),
|
|
173
176
|
message: z.string().optional(),
|
|
174
177
|
durationMs: z.number().optional()
|
|
175
178
|
});
|
|
@@ -269,9 +272,16 @@ const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObje
|
|
|
269
272
|
const judgeCallDataSchema = z.strictObject({
|
|
270
273
|
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
271
274
|
...judgeFields,
|
|
272
|
-
|
|
275
|
+
...sharedInputFields,
|
|
276
|
+
contents: z.array(nonEmpty).optional(),
|
|
277
|
+
turns: turnsSchema.optional(),
|
|
278
|
+
on: judgeOnSchema.optional().describe("The turn `task`, `output`, `transcript` and `diff` read instead of the latest: an earlier turn, or a value graded as the output."),
|
|
273
279
|
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
274
|
-
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score,
|
|
280
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, the inputs it shows the judge (at least one turned on) and `on`.");
|
|
281
|
+
const callSiteSchema = z.strictObject({
|
|
282
|
+
file: nonEmpty.describe("The eval file: absolute, or relative to the project root."),
|
|
283
|
+
line: z.number().int().min(1).describe("From 1.")
|
|
284
|
+
}).describe("Where the eval made the call, which the record names beside the check's outcome.");
|
|
275
285
|
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
276
286
|
z.strictObject({
|
|
277
287
|
op: z.literal("send"),
|
|
@@ -287,7 +297,7 @@ const attemptCallSchema = z.discriminatedUnion("op", [
|
|
|
287
297
|
op: z.literal("serve"),
|
|
288
298
|
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
289
299
|
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
290
|
-
}).describe("`c.serve(command, port?)`: a background process and its port."),
|
|
300
|
+
}).describe("`c.serve(command, port?)`: a background process, once it listens, and its port."),
|
|
291
301
|
z.strictObject({
|
|
292
302
|
op: z.literal("kill"),
|
|
293
303
|
handle: handleId
|
|
@@ -313,8 +323,9 @@ const attemptCallSchema = z.discriminatedUnion("op", [
|
|
|
313
323
|
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
314
324
|
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
315
325
|
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
316
|
-
soft: z.boolean().describe("Called as `expect
|
|
326
|
+
soft: z.boolean().describe("Called as `expect`: a failure fails the attempt at its end and the body carries on. False for `expect.require`, whose failure the runner raises, ending the body."),
|
|
317
327
|
label: z.string().optional().describe("`expect`'s second argument."),
|
|
328
|
+
at: callSiteSchema.optional(),
|
|
318
329
|
poll: z.strictObject({
|
|
319
330
|
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
320
331
|
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
@@ -343,12 +354,13 @@ const attemptCallSchema = z.discriminatedUnion("op", [
|
|
|
343
354
|
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
344
355
|
status: z.enum(["pass", "fail"]),
|
|
345
356
|
message: z.string().optional().describe("Why it failed."),
|
|
346
|
-
soft: z.boolean().optional().describe("
|
|
357
|
+
soft: z.boolean().optional().describe("The body carried on after a failure, as after an `expect`. Default false: the check raised, as Python's `assert` does."),
|
|
358
|
+
at: callSiteSchema.optional()
|
|
347
359
|
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
348
360
|
z.strictObject({
|
|
349
361
|
op: z.literal("judge"),
|
|
350
|
-
question: nonEmpty.describe("A yes/no question about the
|
|
351
|
-
options: judgeCallDataSchema
|
|
362
|
+
question: nonEmpty.describe("A yes/no question about the inputs `options` turns on."),
|
|
363
|
+
options: judgeCallDataSchema
|
|
352
364
|
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
353
365
|
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
354
366
|
const attemptErrorSchema = z.looseObject({
|
|
@@ -379,7 +391,7 @@ z.union([
|
|
|
379
391
|
ok: z.literal(true),
|
|
380
392
|
op: z.literal("serve"),
|
|
381
393
|
handle: handleId,
|
|
382
|
-
port: z.number().describe("The container port it
|
|
394
|
+
port: z.number().describe("The container port it listens on.")
|
|
383
395
|
}),
|
|
384
396
|
z.strictObject({
|
|
385
397
|
id,
|
|
@@ -472,7 +484,7 @@ z.strictObject({
|
|
|
472
484
|
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
473
485
|
z.strictObject({
|
|
474
486
|
type: z.literal("done"),
|
|
475
|
-
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect`
|
|
487
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect.require` is one.")
|
|
476
488
|
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
477
489
|
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
478
490
|
const OTHER_KEYS = [
|
|
@@ -486,7 +498,6 @@ const OTHER_KEYS = [
|
|
|
486
498
|
"matchedBy",
|
|
487
499
|
"answers",
|
|
488
500
|
...Object.keys(judgeCallDataSchema.shape),
|
|
489
|
-
...Object.keys(judgeStateSchema.shape),
|
|
490
501
|
...Object.keys(captureSchema.shape),
|
|
491
502
|
...Object.keys(captureCallDataSchema.shape),
|
|
492
503
|
"port",
|
|
@@ -595,27 +606,88 @@ function mapPagePoint(x, y, window, page, scrollY) {
|
|
|
595
606
|
dy: nextY - scrollY
|
|
596
607
|
};
|
|
597
608
|
}
|
|
609
|
+
const KEYS = {
|
|
610
|
+
Tab: {
|
|
611
|
+
key: "Tab",
|
|
612
|
+
code: "Tab",
|
|
613
|
+
windowsVirtualKeyCode: 9
|
|
614
|
+
},
|
|
615
|
+
Enter: {
|
|
616
|
+
key: "Enter",
|
|
617
|
+
code: "Enter",
|
|
618
|
+
windowsVirtualKeyCode: 13,
|
|
619
|
+
text: "\r"
|
|
620
|
+
},
|
|
621
|
+
Escape: {
|
|
622
|
+
key: "Escape",
|
|
623
|
+
code: "Escape",
|
|
624
|
+
windowsVirtualKeyCode: 27
|
|
625
|
+
},
|
|
626
|
+
Backspace: {
|
|
627
|
+
key: "Backspace",
|
|
628
|
+
code: "Backspace",
|
|
629
|
+
windowsVirtualKeyCode: 8
|
|
630
|
+
},
|
|
631
|
+
Space: {
|
|
632
|
+
key: " ",
|
|
633
|
+
code: "Space",
|
|
634
|
+
windowsVirtualKeyCode: 32,
|
|
635
|
+
text: " "
|
|
636
|
+
},
|
|
637
|
+
ArrowUp: {
|
|
638
|
+
key: "ArrowUp",
|
|
639
|
+
code: "ArrowUp",
|
|
640
|
+
windowsVirtualKeyCode: 38
|
|
641
|
+
},
|
|
642
|
+
ArrowDown: {
|
|
643
|
+
key: "ArrowDown",
|
|
644
|
+
code: "ArrowDown",
|
|
645
|
+
windowsVirtualKeyCode: 40
|
|
646
|
+
},
|
|
647
|
+
ArrowLeft: {
|
|
648
|
+
key: "ArrowLeft",
|
|
649
|
+
code: "ArrowLeft",
|
|
650
|
+
windowsVirtualKeyCode: 37
|
|
651
|
+
},
|
|
652
|
+
ArrowRight: {
|
|
653
|
+
key: "ArrowRight",
|
|
654
|
+
code: "ArrowRight",
|
|
655
|
+
windowsVirtualKeyCode: 39
|
|
656
|
+
}
|
|
657
|
+
};
|
|
658
|
+
/** `CaptureKey` names, for the driver's tool schema. */
|
|
659
|
+
const CAPTURE_KEYS = Object.keys(KEYS);
|
|
660
|
+
async function pressKey(cdp, event) {
|
|
661
|
+
await cdp.send("Input.dispatchKeyEvent", {
|
|
662
|
+
type: "keyDown",
|
|
663
|
+
...event
|
|
664
|
+
});
|
|
665
|
+
const { text: _text, commands: _commands, ...up } = event;
|
|
666
|
+
await cdp.send("Input.dispatchKeyEvent", {
|
|
667
|
+
type: "keyUp",
|
|
668
|
+
...up
|
|
669
|
+
});
|
|
670
|
+
}
|
|
598
671
|
async function dispatchInput(cdp, action, window, page, scrollY = 0) {
|
|
599
672
|
if (action.type === "done") return scrollY;
|
|
600
673
|
if (action.type === "type") {
|
|
601
|
-
|
|
674
|
+
if (action.replace) {
|
|
675
|
+
await pressKey(cdp, {
|
|
676
|
+
key: "a",
|
|
677
|
+
code: "KeyA",
|
|
678
|
+
windowsVirtualKeyCode: 65,
|
|
679
|
+
modifiers: 2,
|
|
680
|
+
commands: ["selectAll"]
|
|
681
|
+
});
|
|
682
|
+
if (!action.text) await pressKey(cdp, KEYS.Backspace);
|
|
683
|
+
}
|
|
684
|
+
if (action.text) await cdp.send("Input.insertText", { text: action.text });
|
|
602
685
|
return scrollY;
|
|
603
686
|
}
|
|
604
687
|
if (action.type === "key") {
|
|
605
|
-
const
|
|
606
|
-
|
|
607
|
-
await cdp
|
|
608
|
-
type: "keyDown",
|
|
609
|
-
key,
|
|
610
|
-
code: key,
|
|
611
|
-
windowsVirtualKeyCode: vk
|
|
612
|
-
});
|
|
613
|
-
await cdp.send("Input.dispatchKeyEvent", {
|
|
614
|
-
type: "keyUp",
|
|
615
|
-
key,
|
|
616
|
-
code: key,
|
|
617
|
-
windowsVirtualKeyCode: vk
|
|
618
|
-
});
|
|
688
|
+
const event = KEYS[action.key];
|
|
689
|
+
if (!event) throw new Error(`c.capture() can't press ${JSON.stringify(action.key)}: use one of ${CAPTURE_KEYS.join(", ")}`);
|
|
690
|
+
await pressKey(cdp, event);
|
|
619
691
|
return scrollY;
|
|
620
692
|
}
|
|
621
693
|
if (action.type === "scroll") {
|
|
@@ -747,6 +819,10 @@ function listenForCaptureWait(cdp, wait) {
|
|
|
747
819
|
};
|
|
748
820
|
}
|
|
749
821
|
const QUIET_IN_FLIGHT = 2;
|
|
822
|
+
const FRESH_REQUEST_MS = 2e3;
|
|
823
|
+
/** How long a screenshot waits, in tries, for a page that has no layout yet. */
|
|
824
|
+
const SCREENSHOT_TRIES = 20;
|
|
825
|
+
const SCREENSHOT_RETRY_MS = 100;
|
|
750
826
|
function requestId(msg) {
|
|
751
827
|
const body = msg;
|
|
752
828
|
return (body.params ?? body).requestId;
|
|
@@ -757,12 +833,12 @@ function requestId(msg) {
|
|
|
757
833
|
* never comes again. As in Chromium, every request that starts or ends restarts the quiet window.
|
|
758
834
|
*/
|
|
759
835
|
function trackNetwork(cdp, quietMs = 500) {
|
|
760
|
-
const inFlight = /* @__PURE__ */ new
|
|
836
|
+
const inFlight = /* @__PURE__ */ new Map();
|
|
761
837
|
let changed;
|
|
762
838
|
cdp.on("Network.requestWillBeSent", (msg) => {
|
|
763
839
|
const id = requestId(msg);
|
|
764
840
|
if (id === void 0) return;
|
|
765
|
-
inFlight.
|
|
841
|
+
inFlight.set(id, Date.now());
|
|
766
842
|
changed?.();
|
|
767
843
|
});
|
|
768
844
|
const end = (msg) => {
|
|
@@ -772,29 +848,37 @@ function trackNetwork(cdp, quietMs = 500) {
|
|
|
772
848
|
};
|
|
773
849
|
cdp.on("Network.loadingFinished", end);
|
|
774
850
|
cdp.on("Network.loadingFailed", end);
|
|
775
|
-
return {
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
const
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
851
|
+
return {
|
|
852
|
+
/** A request started in the last `FRESH_REQUEST_MS` is still in flight; a long poll is not fresh. */
|
|
853
|
+
fresh() {
|
|
854
|
+
const since = Date.now() - FRESH_REQUEST_MS;
|
|
855
|
+
for (const started of inFlight.values()) if (started >= since) return true;
|
|
856
|
+
return false;
|
|
857
|
+
},
|
|
858
|
+
quiet(timeoutMs) {
|
|
859
|
+
return new Promise((resolve, reject) => {
|
|
860
|
+
let settle;
|
|
861
|
+
const stop = () => {
|
|
862
|
+
clearTimeout(timer);
|
|
863
|
+
clearTimeout(settle);
|
|
864
|
+
changed = void 0;
|
|
865
|
+
};
|
|
866
|
+
const timer = setTimeout(() => {
|
|
791
867
|
stop();
|
|
792
|
-
|
|
793
|
-
},
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
868
|
+
reject(captureWaitTimeout("idle"));
|
|
869
|
+
}, timeoutMs);
|
|
870
|
+
changed = () => {
|
|
871
|
+
clearTimeout(settle);
|
|
872
|
+
if (inFlight.size > QUIET_IN_FLIGHT) return;
|
|
873
|
+
settle = setTimeout(() => {
|
|
874
|
+
stop();
|
|
875
|
+
resolve();
|
|
876
|
+
}, quietMs);
|
|
877
|
+
};
|
|
878
|
+
changed();
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
};
|
|
798
882
|
}
|
|
799
883
|
/** Pinned Chrome for Testing `chrome-headless-shell` build. */
|
|
800
884
|
const CHROME_VERSION = "140.0.7339.185";
|
|
@@ -1049,8 +1133,15 @@ async function disposeChrome(child, userData) {
|
|
|
1049
1133
|
} catch {}
|
|
1050
1134
|
removeChromeProfile(userData);
|
|
1051
1135
|
}
|
|
1136
|
+
/**
|
|
1137
|
+
* Chrome's flags for `proxy`: every request goes to it, loopback ones included, which Chrome
|
|
1138
|
+
* otherwise never proxies.
|
|
1139
|
+
*/
|
|
1140
|
+
function proxyArgs(proxy) {
|
|
1141
|
+
return proxy ? [`--proxy-server=http://${proxy}`, "--proxy-bypass-list=<-loopback>"] : [];
|
|
1142
|
+
}
|
|
1052
1143
|
/** One Chrome process, one tab. `fn` must not keep the session after it returns. */
|
|
1053
|
-
async function withChrome(bin, window, timeoutMs, fn) {
|
|
1144
|
+
async function withChrome(bin, window, timeoutMs, options, fn) {
|
|
1054
1145
|
const userData = mkdtempSync(join(tmpdir(), `reeval-chrome-${process.pid}-`));
|
|
1055
1146
|
const child = spawn(bin, [
|
|
1056
1147
|
"--headless",
|
|
@@ -1062,6 +1153,7 @@ async function withChrome(bin, window, timeoutMs, fn) {
|
|
|
1062
1153
|
`--window-size=${window.width},${window.height}`,
|
|
1063
1154
|
`--user-data-dir=${userData}`,
|
|
1064
1155
|
"--remote-debugging-port=0",
|
|
1156
|
+
...proxyArgs(options.proxy),
|
|
1065
1157
|
"about:blank"
|
|
1066
1158
|
], { stdio: [
|
|
1067
1159
|
"ignore",
|
|
@@ -1074,6 +1166,24 @@ async function withChrome(bin, window, timeoutMs, fn) {
|
|
|
1074
1166
|
await cdp.send("Page.enable");
|
|
1075
1167
|
await cdp.send("Page.setLifecycleEventsEnabled", { enabled: true });
|
|
1076
1168
|
const network = trackNetwork(cdp);
|
|
1169
|
+
const loading = /* @__PURE__ */ new Set();
|
|
1170
|
+
const frame = (msg) => {
|
|
1171
|
+
const body = msg;
|
|
1172
|
+
return (body.params ?? body).frameId ?? "";
|
|
1173
|
+
};
|
|
1174
|
+
cdp.on("Page.frameStartedLoading", (msg) => loading.add(frame(msg)));
|
|
1175
|
+
cdp.on("Page.frameStoppedLoading", (msg) => loading.delete(frame(msg)));
|
|
1176
|
+
cdp.on("Page.frameDetached", (msg) => loading.delete(frame(msg)));
|
|
1177
|
+
const opened = [];
|
|
1178
|
+
cdp.on("Page.javascriptDialogOpening", (msg) => {
|
|
1179
|
+
const body = msg;
|
|
1180
|
+
const { type = "dialog", message = "", defaultPrompt = "" } = body.params ?? body;
|
|
1181
|
+
opened.push(`${type} ${JSON.stringify(message.slice(0, 60))}`);
|
|
1182
|
+
cdp.send("Page.handleJavaScriptDialog", {
|
|
1183
|
+
accept: true,
|
|
1184
|
+
promptText: defaultPrompt
|
|
1185
|
+
}).catch(() => {});
|
|
1186
|
+
});
|
|
1077
1187
|
await cdp.send("Network.enable");
|
|
1078
1188
|
await cdp.send("Emulation.setDeviceMetricsOverride", {
|
|
1079
1189
|
width: window.width,
|
|
@@ -1096,15 +1206,26 @@ async function withChrome(bin, window, timeoutMs, fn) {
|
|
|
1096
1206
|
return waitForPage(cdp, wait, waitMs, nav);
|
|
1097
1207
|
},
|
|
1098
1208
|
async screenshot(fullPage) {
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1209
|
+
for (let tries = 1;; tries++) try {
|
|
1210
|
+
const shot = await cdp.send("Page.captureScreenshot", {
|
|
1211
|
+
format: "png",
|
|
1212
|
+
captureBeyondViewport: fullPage
|
|
1213
|
+
});
|
|
1214
|
+
if (!shot.data) throw new Error("Page.captureScreenshot returned no data");
|
|
1215
|
+
return Buffer.from(shot.data, "base64");
|
|
1216
|
+
} catch (error) {
|
|
1217
|
+
if (!/Cannot take screenshot with 0 (width|height)/.test(String(error)) || tries >= SCREENSHOT_TRIES) throw error;
|
|
1218
|
+
await new Promise((resolve) => setTimeout(resolve, SCREENSHOT_RETRY_MS));
|
|
1219
|
+
}
|
|
1105
1220
|
},
|
|
1106
1221
|
async dispatch(action, page) {
|
|
1107
1222
|
scrollY = await dispatchInput(cdp, action, window, page, scrollY);
|
|
1223
|
+
},
|
|
1224
|
+
busy() {
|
|
1225
|
+
return loading.size > 0 || network.fresh();
|
|
1226
|
+
},
|
|
1227
|
+
dialogs() {
|
|
1228
|
+
return opened.splice(0);
|
|
1108
1229
|
}
|
|
1109
1230
|
});
|
|
1110
1231
|
} finally {
|
|
@@ -1128,22 +1249,35 @@ async function navigateAndWait(session, url, wait, timeoutMs) {
|
|
|
1128
1249
|
}
|
|
1129
1250
|
//#endregion
|
|
1130
1251
|
//#region src/capture/poll.ts
|
|
1131
|
-
/**
|
|
1252
|
+
/** A dev server still starting (Nuxt's loading page) or a proxy whose upstream isn't up yet. */
|
|
1253
|
+
const STARTING = /* @__PURE__ */ new Set([
|
|
1254
|
+
502,
|
|
1255
|
+
503,
|
|
1256
|
+
504
|
|
1257
|
+
]);
|
|
1258
|
+
/**
|
|
1259
|
+
* Any HTTP response, including 404, means something accepted the connection. A 502, 503 or 504
|
|
1260
|
+
* is polled past until the deadline, and then taken as it is.
|
|
1261
|
+
*/
|
|
1132
1262
|
async function pollHttp(url, timeoutMs, signal) {
|
|
1133
1263
|
const deadline = Date.now() + timeoutMs;
|
|
1134
1264
|
let last;
|
|
1265
|
+
let answered = false;
|
|
1135
1266
|
while (Date.now() < deadline) {
|
|
1136
1267
|
signal?.throwIfAborted();
|
|
1137
1268
|
const left = Math.max(1, deadline - Date.now());
|
|
1138
1269
|
try {
|
|
1139
|
-
await fetch(url, { signal: AbortSignal.any([AbortSignal.timeout(Math.min(1e3, left)), ...signal ? [signal] : []]) });
|
|
1140
|
-
|
|
1270
|
+
const res = await fetch(url, { signal: AbortSignal.any([AbortSignal.timeout(Math.min(1e3, left)), ...signal ? [signal] : []]) });
|
|
1271
|
+
await res.body?.cancel();
|
|
1272
|
+
if (!STARTING.has(res.status)) return;
|
|
1273
|
+
answered = true;
|
|
1141
1274
|
} catch (error) {
|
|
1142
1275
|
last = error;
|
|
1143
1276
|
if (signal?.aborted) throw error;
|
|
1144
|
-
await new Promise((r) => setTimeout(r, 100));
|
|
1145
1277
|
}
|
|
1278
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
1146
1279
|
}
|
|
1280
|
+
if (answered) return;
|
|
1147
1281
|
throw last instanceof Error ? last : /* @__PURE__ */ new Error("poll timed out");
|
|
1148
1282
|
}
|
|
1149
1283
|
//#endregion
|
|
@@ -1330,15 +1464,437 @@ function pageUrl(origin, path) {
|
|
|
1330
1464
|
const rel = path.startsWith("/") ? path.slice(1) : path;
|
|
1331
1465
|
return new URL(rel, base).href;
|
|
1332
1466
|
}
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1467
|
+
//#endregion
|
|
1468
|
+
//#region src/capture/loopback.ts
|
|
1469
|
+
/**
|
|
1470
|
+
* `localhost`, `*.localhost`, `127.0.0.0/8`, `::1`, their IPv4-mapped IPv6 form and the
|
|
1471
|
+
* unspecified addresses, with or without a trailing dot: the names a page means the sandbox by.
|
|
1472
|
+
*/
|
|
1473
|
+
function isLoopbackHost(host) {
|
|
1474
|
+
const h = host.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
|
1475
|
+
if (h === "localhost" || h.endsWith(".localhost")) return true;
|
|
1476
|
+
if (net.isIPv4(h)) return h.startsWith("127.") || h === "0.0.0.0";
|
|
1477
|
+
if (!net.isIPv6(h)) return false;
|
|
1478
|
+
if (h === "::1" || h === "::") return true;
|
|
1479
|
+
const mapped = /^::ffff:(?:(\d+\.\d+\.\d+\.\d+)|([0-9a-f]{1,4}):[0-9a-f]{1,4})$/.exec(h);
|
|
1480
|
+
if (!mapped) return false;
|
|
1481
|
+
return mapped[1] ? mapped[1].startsWith("127.") : Number.parseInt(mapped[2] ?? "", 16) >> 8 === 127;
|
|
1482
|
+
}
|
|
1483
|
+
function authority(value) {
|
|
1484
|
+
if (!URL.canParse(`http://${value}`)) return void 0;
|
|
1485
|
+
const url = new URL(`http://${value}`);
|
|
1486
|
+
return {
|
|
1487
|
+
host: url.hostname.replace(/^\[|\]$/g, ""),
|
|
1488
|
+
port: Number(url.port) || 80
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
/** `rawHeaders` without the ones meant for the proxy, in the flat form `http.request` takes. */
|
|
1492
|
+
function forwarded(raw) {
|
|
1493
|
+
const out = [];
|
|
1494
|
+
for (let i = 0; i < raw.length; i += 2) {
|
|
1495
|
+
const name = raw[i] ?? "";
|
|
1496
|
+
if (!/^proxy-/i.test(name)) out.push(name, raw[i + 1] ?? "");
|
|
1497
|
+
}
|
|
1498
|
+
return out;
|
|
1499
|
+
}
|
|
1500
|
+
/**
|
|
1501
|
+
* The capture browser's HTTP proxy, on the host's loopback. Chrome sends it every request
|
|
1502
|
+
* (`--proxy-bypass-list=<-loopback>`): one for a loopback address goes into the sandbox, to that
|
|
1503
|
+
* port through its tunnel, opened on first use; everything else goes out as it would without a
|
|
1504
|
+
* proxy, so the page reaches the internet as before.
|
|
1505
|
+
*
|
|
1506
|
+
* A plain-HTTP request that gets no response (nothing listens on that sandbox port) drops the
|
|
1507
|
+
* page's connection rather than answering, so the page's fetch fails as a refused one does. A
|
|
1508
|
+
* CONNECT to such a port is answered and then closed: the tunnel accepts before the sandbox side
|
|
1509
|
+
* connects, and Chrome fails that request too.
|
|
1510
|
+
*/
|
|
1511
|
+
async function startLoopbackProxy(options) {
|
|
1512
|
+
const direct = new Set(options.direct);
|
|
1513
|
+
const reached = [];
|
|
1514
|
+
const tunnels = /* @__PURE__ */ new Map();
|
|
1515
|
+
const live = /* @__PURE__ */ new Set();
|
|
1516
|
+
let closed = false;
|
|
1517
|
+
const track = (s) => {
|
|
1518
|
+
live.add(s);
|
|
1519
|
+
s.once("close", () => live.delete(s));
|
|
1520
|
+
return s;
|
|
1521
|
+
};
|
|
1522
|
+
const answered = (route) => {
|
|
1523
|
+
if (route.sandbox !== void 0 && !reached.includes(route.sandbox)) reached.push(route.sandbox);
|
|
1524
|
+
};
|
|
1525
|
+
const resolve = async (to) => {
|
|
1526
|
+
if (!isLoopbackHost(to.host) || direct.has(to.port)) return to;
|
|
1527
|
+
let open = tunnels.get(to.port);
|
|
1528
|
+
if (!open) {
|
|
1529
|
+
open = options.tunnel(to.port).then((url) => {
|
|
1530
|
+
const at = new URL(url);
|
|
1531
|
+
direct.add(Number(at.port));
|
|
1532
|
+
return {
|
|
1533
|
+
host: at.hostname,
|
|
1534
|
+
port: Number(at.port)
|
|
1535
|
+
};
|
|
1536
|
+
});
|
|
1537
|
+
open.catch(() => tunnels.delete(to.port));
|
|
1538
|
+
tunnels.set(to.port, open);
|
|
1539
|
+
}
|
|
1540
|
+
return {
|
|
1541
|
+
...await open,
|
|
1542
|
+
sandbox: to.port
|
|
1543
|
+
};
|
|
1544
|
+
};
|
|
1545
|
+
const connect = async (to) => {
|
|
1546
|
+
const route = await resolve(to);
|
|
1547
|
+
if (closed) throw new Error("the capture proxy closed");
|
|
1548
|
+
return new Promise((ok, fail) => {
|
|
1549
|
+
const socket = track(net.connect({
|
|
1550
|
+
host: route.host,
|
|
1551
|
+
port: route.port,
|
|
1552
|
+
allowHalfOpen: true
|
|
1553
|
+
}));
|
|
1554
|
+
socket.once("connect", () => {
|
|
1555
|
+
socket.off("error", fail);
|
|
1556
|
+
ok({
|
|
1557
|
+
socket,
|
|
1558
|
+
route
|
|
1559
|
+
});
|
|
1560
|
+
});
|
|
1561
|
+
socket.once("error", fail);
|
|
1562
|
+
});
|
|
1563
|
+
};
|
|
1564
|
+
/** Joins the page's connection to the upstream one, unless the page left while it opened. */
|
|
1565
|
+
const join = (client, socket, route) => {
|
|
1566
|
+
if (closed || client.destroyed) {
|
|
1567
|
+
socket.destroy();
|
|
1568
|
+
return false;
|
|
1569
|
+
}
|
|
1570
|
+
socket.once("data", () => answered(route));
|
|
1571
|
+
socket.on("error", () => client.destroy());
|
|
1572
|
+
client.on("error", () => socket.destroy());
|
|
1573
|
+
client.once("close", () => socket.destroy());
|
|
1574
|
+
socket.once("close", () => client.destroy());
|
|
1575
|
+
client.pipe(socket);
|
|
1576
|
+
socket.pipe(client);
|
|
1577
|
+
return true;
|
|
1578
|
+
};
|
|
1579
|
+
const server = http.createServer((req, res) => {
|
|
1580
|
+
const url = URL.canParse(req.url ?? "") ? new URL(req.url ?? "") : void 0;
|
|
1581
|
+
const to = url && authority(url.host);
|
|
1582
|
+
if (!url || !to || url.protocol !== "http:") {
|
|
1583
|
+
res.writeHead(400).end();
|
|
1584
|
+
return;
|
|
1585
|
+
}
|
|
1586
|
+
const drop = () => req.socket.destroy();
|
|
1587
|
+
resolve(to).then((route) => {
|
|
1588
|
+
if (closed || res.destroyed) return drop();
|
|
1589
|
+
const upstream = http.request({
|
|
1590
|
+
host: route.host,
|
|
1591
|
+
port: route.port,
|
|
1592
|
+
method: req.method,
|
|
1593
|
+
path: `${url.pathname}${url.search}`,
|
|
1594
|
+
headers: forwarded(req.rawHeaders),
|
|
1595
|
+
agent: false
|
|
1596
|
+
}, (up) => {
|
|
1597
|
+
answered(route);
|
|
1598
|
+
res.writeHead(up.statusCode ?? 502, up.statusMessage, up.rawHeaders);
|
|
1599
|
+
up.pipe(res);
|
|
1600
|
+
});
|
|
1601
|
+
track(upstream);
|
|
1602
|
+
res.once("close", () => upstream.destroy());
|
|
1603
|
+
upstream.on("error", drop);
|
|
1604
|
+
req.pipe(upstream);
|
|
1605
|
+
}).catch(drop);
|
|
1606
|
+
});
|
|
1607
|
+
server.on("connection", track);
|
|
1608
|
+
server.on("connect", (req, client, early) => {
|
|
1609
|
+
client.on("error", () => client.destroy());
|
|
1610
|
+
const to = authority(req.url ?? "");
|
|
1611
|
+
if (!to) {
|
|
1612
|
+
client.end("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
1613
|
+
return;
|
|
1614
|
+
}
|
|
1615
|
+
connect(to).then(({ socket, route }) => {
|
|
1616
|
+
if (!join(client, socket, route)) return;
|
|
1617
|
+
client.write("HTTP/1.1 200 Connection Established\r\n\r\n");
|
|
1618
|
+
if (early.length) socket.write(early);
|
|
1619
|
+
}).catch(() => {
|
|
1620
|
+
if (!client.destroyed) client.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
|
|
1621
|
+
});
|
|
1622
|
+
});
|
|
1623
|
+
server.on("upgrade", (req, client, early) => {
|
|
1624
|
+
client.on("error", () => client.destroy());
|
|
1625
|
+
const url = URL.canParse(req.url ?? "") ? new URL(req.url ?? "") : void 0;
|
|
1626
|
+
const to = url && authority(url.host);
|
|
1627
|
+
if (!url || !to) {
|
|
1628
|
+
client.end("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
1629
|
+
return;
|
|
1630
|
+
}
|
|
1631
|
+
const headers = forwarded(req.rawHeaders);
|
|
1632
|
+
const lines = [`${req.method} ${url.pathname}${url.search} HTTP/${req.httpVersion}`];
|
|
1633
|
+
for (let i = 0; i < headers.length; i += 2) lines.push(`${headers[i]}: ${headers[i + 1]}`);
|
|
1634
|
+
connect(to).then(({ socket, route }) => {
|
|
1635
|
+
if (!join(client, socket, route)) return;
|
|
1636
|
+
socket.write(`${lines.join("\r\n")}\r\n\r\n`);
|
|
1637
|
+
if (early.length) socket.write(early);
|
|
1638
|
+
}).catch(() => {
|
|
1639
|
+
if (!client.destroyed) client.end("HTTP/1.1 502 Bad Gateway\r\n\r\n");
|
|
1640
|
+
});
|
|
1641
|
+
});
|
|
1642
|
+
await new Promise((ok, fail) => {
|
|
1643
|
+
server.once("error", fail);
|
|
1644
|
+
server.listen(0, "127.0.0.1", () => ok());
|
|
1645
|
+
});
|
|
1646
|
+
const { port } = server.address();
|
|
1647
|
+
return {
|
|
1648
|
+
address: `127.0.0.1:${port}`,
|
|
1649
|
+
reached,
|
|
1650
|
+
close: () => new Promise((ok) => {
|
|
1651
|
+
closed = true;
|
|
1652
|
+
for (const s of live) s.destroy();
|
|
1653
|
+
server.close(() => ok());
|
|
1654
|
+
})
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
//#endregion
|
|
1658
|
+
//#region src/capture/png.ts
|
|
1659
|
+
const CHANNELS = {
|
|
1660
|
+
0: 1,
|
|
1661
|
+
2: 3,
|
|
1662
|
+
4: 2,
|
|
1663
|
+
6: 4
|
|
1664
|
+
};
|
|
1665
|
+
/** An 8-bit, non-interlaced PNG's unfiltered pixels, or undefined for any other. */
|
|
1666
|
+
function pixels(png) {
|
|
1667
|
+
if (png.length < 33 || png.readUInt32BE(12) !== 1229472850) return void 0;
|
|
1668
|
+
const width = png.readUInt32BE(16);
|
|
1669
|
+
const height = png.readUInt32BE(20);
|
|
1670
|
+
const channels = CHANNELS[png[25] ?? -1];
|
|
1671
|
+
if (png[24] !== 8 || png[28] !== 0 || !channels) return void 0;
|
|
1672
|
+
const idat = [];
|
|
1673
|
+
for (let at = 8; at + 8 <= png.length;) {
|
|
1674
|
+
const length = png.readUInt32BE(at);
|
|
1675
|
+
const type = png.toString("latin1", at + 4, at + 8);
|
|
1676
|
+
if (type === "IDAT") idat.push(png.subarray(at + 8, at + 8 + length));
|
|
1677
|
+
if (type === "IEND") break;
|
|
1678
|
+
at += 12 + length;
|
|
1679
|
+
}
|
|
1680
|
+
const raw = inflateSync(Buffer.concat(idat));
|
|
1681
|
+
const stride = width * channels;
|
|
1682
|
+
const data = Buffer.alloc(stride * height);
|
|
1683
|
+
for (let y = 0; y < height; y++) {
|
|
1684
|
+
const filter = raw[y * (stride + 1)];
|
|
1685
|
+
const line = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));
|
|
1686
|
+
const row = y * stride;
|
|
1687
|
+
for (let x = 0; x < stride; x++) {
|
|
1688
|
+
const a = x >= channels ? data[row + x - channels] ?? 0 : 0;
|
|
1689
|
+
const b = y > 0 ? data[row - stride + x] ?? 0 : 0;
|
|
1690
|
+
const c = x >= channels && y > 0 ? data[row - stride + x - channels] ?? 0 : 0;
|
|
1691
|
+
let value = line[x] ?? 0;
|
|
1692
|
+
if (filter === 1) value += a;
|
|
1693
|
+
else if (filter === 2) value += b;
|
|
1694
|
+
else if (filter === 3) value += a + b >> 1;
|
|
1695
|
+
else if (filter === 4) {
|
|
1696
|
+
const p = a + b - c;
|
|
1697
|
+
const pa = Math.abs(p - a);
|
|
1698
|
+
const pb = Math.abs(p - b);
|
|
1699
|
+
const pc = Math.abs(p - c);
|
|
1700
|
+
value += pa <= pb && pa <= pc ? a : pb <= pc ? b : c;
|
|
1701
|
+
}
|
|
1702
|
+
data[row + x] = value & 255;
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
return {
|
|
1706
|
+
width,
|
|
1707
|
+
height,
|
|
1708
|
+
channels,
|
|
1709
|
+
data
|
|
1710
|
+
};
|
|
1711
|
+
}
|
|
1712
|
+
/** The diff samples every `SAMPLE`th pixel across and down. */
|
|
1713
|
+
const SAMPLE = 4;
|
|
1714
|
+
/** A sampled pixel differs when a channel moves more than this, so antialiasing doesn't count. */
|
|
1715
|
+
const CHANNEL_DELTA = 32;
|
|
1716
|
+
/** The share of sampled pixels that must differ: a caret at any font size is below it. */
|
|
1717
|
+
const VIEW_CHANGE = .002;
|
|
1718
|
+
/**
|
|
1719
|
+
* Whether `after` looks different from `before`, two viewport shots: more than `VIEW_CHANGE` of
|
|
1720
|
+
* the sampled pixels differ, or the size does. Bytes decide when either PNG is one this can't read.
|
|
1721
|
+
*/
|
|
1722
|
+
function pageChanged(before, after) {
|
|
1723
|
+
if (before.equals(after)) return false;
|
|
1724
|
+
let a;
|
|
1725
|
+
let b;
|
|
1726
|
+
try {
|
|
1727
|
+
a = pixels(before);
|
|
1728
|
+
b = pixels(after);
|
|
1729
|
+
} catch {
|
|
1730
|
+
return true;
|
|
1731
|
+
}
|
|
1732
|
+
if (!a || !b) return true;
|
|
1733
|
+
if (a.width !== b.width || a.height !== b.height || a.channels !== b.channels) return true;
|
|
1734
|
+
const stride = a.width * a.channels;
|
|
1735
|
+
let sampled = 0;
|
|
1736
|
+
let differ = 0;
|
|
1737
|
+
for (let y = 0; y < a.height; y += SAMPLE) for (let x = 0; x < a.width; x += SAMPLE) {
|
|
1738
|
+
sampled++;
|
|
1739
|
+
const at = y * stride + x * a.channels;
|
|
1740
|
+
for (let c = 0; c < a.channels; c++) if (Math.abs((a.data[at + c] ?? 0) - (b.data[at + c] ?? 0)) > CHANNEL_DELTA) {
|
|
1741
|
+
differ++;
|
|
1742
|
+
break;
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
return differ > sampled * VIEW_CHANGE;
|
|
1746
|
+
}
|
|
1747
|
+
//#endregion
|
|
1748
|
+
//#region src/capture/page-state.ts
|
|
1749
|
+
const FIELDS = [
|
|
1750
|
+
"url",
|
|
1751
|
+
"focus",
|
|
1752
|
+
"value",
|
|
1753
|
+
"form",
|
|
1754
|
+
"text",
|
|
1755
|
+
"nodes",
|
|
1756
|
+
"view"
|
|
1757
|
+
];
|
|
1758
|
+
/** Page text in the prompt is capped: the page controls it. */
|
|
1759
|
+
const TEXT_CAP = 60;
|
|
1760
|
+
const PROBE = `(() => {
|
|
1761
|
+
const hash = (s) => { let h = 2166136261; for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 16777619) } return (h >>> 0).toString(36) }
|
|
1762
|
+
const clip = (s) => String(s ?? '').replace(/\\s+/g, ' ').trim().slice(0, ${TEXT_CAP})
|
|
1763
|
+
let a = document.activeElement
|
|
1764
|
+
while (a && a.shadowRoot && a.shadowRoot.activeElement) a = a.shadowRoot.activeElement
|
|
1765
|
+
if (a === document.body || a === document.documentElement) a = null
|
|
1766
|
+
const tag = a ? a.tagName.toLowerCase() : ''
|
|
1767
|
+
const type = a && tag === 'input' ? (a.type || 'text') : ''
|
|
1768
|
+
const label = a ? clip(a.getAttribute('aria-label') || (a.labels && a.labels[0] && a.labels[0].innerText) || a.getAttribute('placeholder') || a.getAttribute('name') || a.id || (tag === 'button' || tag === 'a' ? a.innerText : '')) : ''
|
|
1769
|
+
const toggles = type === 'checkbox' || type === 'radio'
|
|
1770
|
+
const value = !a ? null
|
|
1771
|
+
: toggles ? (a.checked ? 'checked' : 'unchecked')
|
|
1772
|
+
: tag === 'select' ? clip(a.selectedOptions && a.selectedOptions[0] ? a.selectedOptions[0].text : '')
|
|
1773
|
+
: type === 'password' ? a.value.length + ' characters'
|
|
1774
|
+
: tag === 'input' || tag === 'textarea' ? clip(a.value)
|
|
1775
|
+
: a.isContentEditable ? clip(a.innerText)
|
|
1776
|
+
: null
|
|
1777
|
+
const buttons = ['button', 'submit', 'reset', 'image', 'file', 'range', 'color', 'hidden']
|
|
1778
|
+
const editable = !!a && ((tag === 'input' && !toggles && !buttons.includes(type)) || tag === 'textarea' || a.isContentEditable)
|
|
1779
|
+
const controls = Array.from(document.querySelectorAll('input,select,textarea'))
|
|
1780
|
+
.map((e) => (e.type === 'checkbox' || e.type === 'radio' ? (e.checked ? '1' : '0') : e.value))
|
|
1781
|
+
return {
|
|
1782
|
+
url: location.href,
|
|
1783
|
+
focus: a ? tag + (type ? '[' + type + ']' : '') + (label ? ' ' + JSON.stringify(label) : '') : null,
|
|
1784
|
+
value,
|
|
1785
|
+
editable,
|
|
1786
|
+
form: hash(controls.join('\\u0001')),
|
|
1787
|
+
text: hash(document.body ? document.body.innerText : ''),
|
|
1788
|
+
nodes: document.getElementsByTagName('*').length,
|
|
1789
|
+
}
|
|
1790
|
+
})()`;
|
|
1791
|
+
/** Reads the page. A field it can't read is left out. */
|
|
1792
|
+
async function probePage(session) {
|
|
1793
|
+
const read = await session.cdp.send("Runtime.evaluate", {
|
|
1794
|
+
expression: PROBE,
|
|
1795
|
+
returnByValue: true
|
|
1796
|
+
}).catch(() => void 0);
|
|
1797
|
+
const view = await session.screenshot(false).catch(() => void 0);
|
|
1798
|
+
return {
|
|
1799
|
+
...read?.result?.value ?? {},
|
|
1800
|
+
...view ? { view } : {}
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
/** The fields that differ between two states, ignoring those `volatile` names. */
|
|
1804
|
+
function differing(before, after, volatile = /* @__PURE__ */ new Set()) {
|
|
1805
|
+
return FIELDS.filter((field) => {
|
|
1806
|
+
if (volatile.has(field)) return false;
|
|
1807
|
+
const a = before[field];
|
|
1808
|
+
const b = after[field];
|
|
1809
|
+
if (a === void 0 || b === void 0) return a !== b;
|
|
1810
|
+
if (field === "view") return pageChanged(a, b);
|
|
1811
|
+
return a !== b;
|
|
1812
|
+
});
|
|
1813
|
+
}
|
|
1814
|
+
/** The page's path, without its query or fragment, and capped. */
|
|
1815
|
+
function pathOf(url, origin) {
|
|
1816
|
+
if (!url) return void 0;
|
|
1817
|
+
const local = url.startsWith(`${origin}/`) ? url.slice(origin.length) : url;
|
|
1818
|
+
const cut = local.search(/[?#]/);
|
|
1819
|
+
const path = cut === -1 ? local : `${local.slice(0, cut)}${local[cut] === "?" ? "?…" : ""}`;
|
|
1820
|
+
return path.length > TEXT_CAP ? `${path.slice(0, TEXT_CAP)}…` : path;
|
|
1821
|
+
}
|
|
1822
|
+
/** What changed between two states, in the words the model reads. */
|
|
1823
|
+
function describeChange(after, fields, origin) {
|
|
1824
|
+
const notes = [];
|
|
1825
|
+
const has = (f) => fields.includes(f);
|
|
1826
|
+
if (has("url")) notes.push(`now at ${pathOf(after.url, origin) ?? "a page reeval could not read"}`);
|
|
1827
|
+
if (has("focus")) notes.push(after.focus ? `focus moved to ${after.focus}` : "nothing is focused now");
|
|
1828
|
+
if (after.focus && (has("value") || has("focus") && after.value)) notes.push(`${after.focus} holds ${JSON.stringify(after.value ?? "")}`);
|
|
1829
|
+
else if (has("form")) notes.push("a form control changed");
|
|
1830
|
+
if (has("text") || has("nodes")) notes.push("the page's content changed");
|
|
1831
|
+
else if (has("view") && !notes.length) notes.push("the page looks different");
|
|
1832
|
+
return notes;
|
|
1833
|
+
}
|
|
1834
|
+
/**
|
|
1835
|
+
* Waits for the page to stop changing after a move: two reads `SETTLE_POLL_MS` apart that agree,
|
|
1836
|
+
* with no navigation or fresh request in flight. At `SETTLE_CAP_MS` it gives up and adds the
|
|
1837
|
+
* fields still changing to `volatile`, so a clock or an animation doesn't hold up every move.
|
|
1838
|
+
*/
|
|
1839
|
+
async function settledState(session, volatile, left) {
|
|
1840
|
+
const start = Date.now();
|
|
1841
|
+
const sleep = () => new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.min(100, left()))));
|
|
1842
|
+
let prev = await probePage(session);
|
|
1843
|
+
for (;;) {
|
|
1844
|
+
if (left() <= 0) return prev;
|
|
1845
|
+
await sleep();
|
|
1846
|
+
const next = await probePage(session);
|
|
1847
|
+
const moving = differing(prev, next, volatile);
|
|
1848
|
+
if (!moving.length && !session.busy()) return next;
|
|
1849
|
+
if (Date.now() - start >= 3e3 || left() <= 0) {
|
|
1850
|
+
for (const field of moving) if (field !== "url" && field !== "focus") volatile.add(field);
|
|
1851
|
+
return next;
|
|
1852
|
+
}
|
|
1853
|
+
prev = next;
|
|
1854
|
+
}
|
|
1855
|
+
}
|
|
1856
|
+
/** Points within this many CSS pixels of each other are the same move. */
|
|
1857
|
+
const SAME_POINT = 8;
|
|
1858
|
+
/** Model turns in a row thrown away because the page changed while the model chose. */
|
|
1859
|
+
const STALE_ASKS = 2;
|
|
1860
|
+
function timedOut(clock) {
|
|
1861
|
+
const { step } = clock;
|
|
1862
|
+
const where = step ? ` (instruction ${step.index} of ${step.total}, after ${plural(step.moves, "move")})` : "";
|
|
1863
|
+
return /* @__PURE__ */ new Error(`c.capture() timed out on ${clock.detail}${where}: it used all of its ${formatDuration(clock.budgetMs)} timeout. Raise timeout on the call or capture.timeout.`);
|
|
1864
|
+
}
|
|
1865
|
+
function check(clock) {
|
|
1866
|
+
const left = clock.remaining();
|
|
1867
|
+
if (left <= 0) throw timedOut(clock);
|
|
1338
1868
|
return left;
|
|
1339
1869
|
}
|
|
1340
|
-
|
|
1341
|
-
|
|
1870
|
+
/** `work()`, or the timeout once the budget runs out first: a page can hold any CDP call open. */
|
|
1871
|
+
function race(clock, work) {
|
|
1872
|
+
const left = clock.remaining();
|
|
1873
|
+
if (left <= 0) return Promise.reject(timedOut(clock));
|
|
1874
|
+
let timer;
|
|
1875
|
+
const out = new Promise((_, reject) => {
|
|
1876
|
+
timer = setTimeout(() => reject(timedOut(clock)), left);
|
|
1877
|
+
});
|
|
1878
|
+
return Promise.race([work(), out]).finally(() => clearTimeout(timer));
|
|
1879
|
+
}
|
|
1880
|
+
/** `session` with every call raced against the clock, and its own waits' timeouts named as the budget's. */
|
|
1881
|
+
function onClock(session, clock) {
|
|
1882
|
+
const own = (error) => {
|
|
1883
|
+
throw /c\.capture\(\) timed out/.test(String(error)) ? timedOut(clock) : error;
|
|
1884
|
+
};
|
|
1885
|
+
return {
|
|
1886
|
+
window: session.window,
|
|
1887
|
+
cdp: {
|
|
1888
|
+
on: (method, fn) => session.cdp.on(method, fn),
|
|
1889
|
+
send: (method, params) => race(clock, () => session.cdp.send(method, params))
|
|
1890
|
+
},
|
|
1891
|
+
navigate: (url) => race(clock, () => session.navigate(url)),
|
|
1892
|
+
wait: (wait, ms, nav) => race(clock, () => session.wait(wait, ms, nav).catch(own)),
|
|
1893
|
+
screenshot: (fullPage) => race(clock, () => session.screenshot(fullPage)),
|
|
1894
|
+
dispatch: (action, page) => race(clock, () => session.dispatch(action, page)),
|
|
1895
|
+
busy: () => session.busy(),
|
|
1896
|
+
dialogs: () => session.dialogs()
|
|
1897
|
+
};
|
|
1342
1898
|
}
|
|
1343
1899
|
function asUsage(value) {
|
|
1344
1900
|
return {
|
|
@@ -1349,38 +1905,68 @@ function asUsage(value) {
|
|
|
1349
1905
|
reasoning: null
|
|
1350
1906
|
};
|
|
1351
1907
|
}
|
|
1908
|
+
const finite = (value) => {
|
|
1909
|
+
const n = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN;
|
|
1910
|
+
return Number.isFinite(n) ? n : void 0;
|
|
1911
|
+
};
|
|
1912
|
+
/** The point at `x`/`y`, or both packed into one as `"217,163"`, which models sometimes send. */
|
|
1913
|
+
function point(input, x, y) {
|
|
1914
|
+
const packed = typeof input[x] === "string" ? input[x].split(",") : [];
|
|
1915
|
+
const px = finite(packed.length === 2 ? packed[0] : input[x]);
|
|
1916
|
+
const py = finite(packed.length === 2 ? packed[1] : input[y]);
|
|
1917
|
+
return px === void 0 || py === void 0 ? void 0 : [px, py];
|
|
1918
|
+
}
|
|
1919
|
+
/** The action a tool call asks for, or why it can't be made. */
|
|
1352
1920
|
function actionFromTool(name, input) {
|
|
1353
1921
|
if (name === "done") return { type: "done" };
|
|
1354
|
-
if (name === "type")
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
}
|
|
1362
|
-
if (name === "
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1922
|
+
if (name === "type") {
|
|
1923
|
+
if (typeof input.text !== "string") return { refused: "it had no text" };
|
|
1924
|
+
return {
|
|
1925
|
+
type: "type",
|
|
1926
|
+
text: input.text,
|
|
1927
|
+
...input.replace === true ? { replace: true } : {}
|
|
1928
|
+
};
|
|
1929
|
+
}
|
|
1930
|
+
if (name === "key") {
|
|
1931
|
+
const key = CAPTURE_KEYS.find((k) => k === input.key);
|
|
1932
|
+
if (!key) return { refused: `the key is not one of ${CAPTURE_KEYS.join(", ")}` };
|
|
1933
|
+
return {
|
|
1934
|
+
type: "key",
|
|
1935
|
+
key
|
|
1936
|
+
};
|
|
1937
|
+
}
|
|
1938
|
+
const at = point(input, "x", "y");
|
|
1939
|
+
if (!at) return { refused: "it had no numeric x and y" };
|
|
1940
|
+
const [x, y] = at;
|
|
1941
|
+
if (name === "scroll") {
|
|
1942
|
+
const dy = finite(input.dy);
|
|
1943
|
+
return dy === void 0 ? { refused: "it had no numeric dy" } : {
|
|
1944
|
+
type: "scroll",
|
|
1945
|
+
x,
|
|
1946
|
+
y,
|
|
1947
|
+
dy
|
|
1948
|
+
};
|
|
1949
|
+
}
|
|
1950
|
+
if (name === "drag") {
|
|
1951
|
+
const to = point(input, "toX", "toY");
|
|
1952
|
+
if (!to) return { refused: "it had no numeric toX and toY" };
|
|
1953
|
+
return {
|
|
1954
|
+
type: "drag",
|
|
1955
|
+
x,
|
|
1956
|
+
y,
|
|
1957
|
+
toX: to[0],
|
|
1958
|
+
toY: to[1]
|
|
1959
|
+
};
|
|
1960
|
+
}
|
|
1375
1961
|
if (name === "hover") return {
|
|
1376
1962
|
type: "hover",
|
|
1377
|
-
x
|
|
1378
|
-
y
|
|
1963
|
+
x,
|
|
1964
|
+
y
|
|
1379
1965
|
};
|
|
1380
1966
|
return {
|
|
1381
1967
|
type: "click",
|
|
1382
|
-
x
|
|
1383
|
-
y
|
|
1968
|
+
x,
|
|
1969
|
+
y
|
|
1384
1970
|
};
|
|
1385
1971
|
}
|
|
1386
1972
|
/**
|
|
@@ -1411,7 +1997,7 @@ function scalePoints(action, page) {
|
|
|
1411
1997
|
default: return action;
|
|
1412
1998
|
}
|
|
1413
1999
|
}
|
|
1414
|
-
const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
|
|
2000
|
+
const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Each turn you get the goal, what has focus, your earlier moves with what each did, and a fresh screenshot. Click a field before typing into it, and do not repeat a move that already worked. A dropdown (select) never opens in the screenshot: click it once, press ArrowDown or ArrowUp until it shows the option you want, then press Enter to choose it. Call done when the goal is met. Do not mention CDP.`;
|
|
1415
2001
|
const useTools = {
|
|
1416
2002
|
click: tool({
|
|
1417
2003
|
description: "Click at (x, y) on the screenshot.",
|
|
@@ -1421,12 +2007,15 @@ const useTools = {
|
|
|
1421
2007
|
})
|
|
1422
2008
|
}),
|
|
1423
2009
|
type: tool({
|
|
1424
|
-
description: "Insert text into the focused field. Click the field first.",
|
|
1425
|
-
inputSchema: z.object({
|
|
2010
|
+
description: "Insert text into the focused field. Click the field first. Set replace to true to replace what the field already holds instead of adding to it.",
|
|
2011
|
+
inputSchema: z.object({
|
|
2012
|
+
text: z.string(),
|
|
2013
|
+
replace: z.boolean().optional()
|
|
2014
|
+
})
|
|
1426
2015
|
}),
|
|
1427
2016
|
key: tool({
|
|
1428
|
-
description: "Press
|
|
1429
|
-
inputSchema: z.object({ key: z.enum(
|
|
2017
|
+
description: "Press a key in the focused element. In a dropdown (select), click it, press ArrowDown or ArrowUp to the option, then Enter to choose it. Space toggles a focused checkbox.",
|
|
2018
|
+
inputSchema: z.object({ key: z.enum(CAPTURE_KEYS) })
|
|
1430
2019
|
}),
|
|
1431
2020
|
scroll: tool({
|
|
1432
2021
|
description: "Wheel-scroll a constrained scroller at (x, y).",
|
|
@@ -1457,12 +2046,17 @@ const useTools = {
|
|
|
1457
2046
|
inputSchema: z.object({})
|
|
1458
2047
|
})
|
|
1459
2048
|
};
|
|
1460
|
-
async function runFakeScript(session, script,
|
|
2049
|
+
async function runFakeScript(session, script, step, clock, volatile) {
|
|
2050
|
+
const goal = step.goal;
|
|
1461
2051
|
let moves = 0;
|
|
1462
2052
|
for (const action of script) {
|
|
1463
|
-
|
|
2053
|
+
clock.step = {
|
|
2054
|
+
...step,
|
|
2055
|
+
moves
|
|
2056
|
+
};
|
|
2057
|
+
check(clock);
|
|
1464
2058
|
if (action.type === "done") return moves;
|
|
1465
|
-
if (moves >= 32) throw
|
|
2059
|
+
if (moves >= 32) throw unfinished(step);
|
|
1466
2060
|
const page = pngSize(await session.screenshot(true));
|
|
1467
2061
|
debugTrace("check", "capture-use", {
|
|
1468
2062
|
goal,
|
|
@@ -1471,97 +2065,274 @@ async function runFakeScript(session, script, goal, remaining) {
|
|
|
1471
2065
|
}, `capture use ${action.type}`);
|
|
1472
2066
|
await session.dispatch(action, page);
|
|
1473
2067
|
moves++;
|
|
1474
|
-
await
|
|
2068
|
+
await settledState(session, volatile, clock.remaining);
|
|
2069
|
+
}
|
|
2070
|
+
clock.step = {
|
|
2071
|
+
...step,
|
|
2072
|
+
moves
|
|
2073
|
+
};
|
|
2074
|
+
while (clock.remaining() > 0) await new Promise((resolve) => setTimeout(resolve, Math.min(100, clock.remaining())));
|
|
2075
|
+
throw timedOut(clock);
|
|
2076
|
+
}
|
|
2077
|
+
const unfinished = (step) => /* @__PURE__ */ new Error(`c.capture() instruction ${step.index} of ${step.total} ${JSON.stringify(step.goal)} did not finish in 32 moves`);
|
|
2078
|
+
const grid = (n) => Math.round(n / SAME_POINT);
|
|
2079
|
+
function moveKey(action) {
|
|
2080
|
+
switch (action.type) {
|
|
2081
|
+
case "click":
|
|
2082
|
+
case "hover": return JSON.stringify([
|
|
2083
|
+
action.type,
|
|
2084
|
+
grid(action.x),
|
|
2085
|
+
grid(action.y)
|
|
2086
|
+
]);
|
|
2087
|
+
case "scroll": return JSON.stringify([
|
|
2088
|
+
action.type,
|
|
2089
|
+
grid(action.x),
|
|
2090
|
+
grid(action.y),
|
|
2091
|
+
action.dy
|
|
2092
|
+
]);
|
|
2093
|
+
case "drag": return JSON.stringify([
|
|
2094
|
+
action.type,
|
|
2095
|
+
grid(action.x),
|
|
2096
|
+
grid(action.y),
|
|
2097
|
+
grid(action.toX),
|
|
2098
|
+
grid(action.toY)
|
|
2099
|
+
]);
|
|
2100
|
+
default: return JSON.stringify(action);
|
|
1475
2101
|
}
|
|
1476
|
-
while (remaining() > 0) await settle();
|
|
1477
|
-
throw new Error(`c.capture() timed out on ${JSON.stringify(goal)}`);
|
|
1478
2102
|
}
|
|
1479
|
-
|
|
2103
|
+
/** Sets what `move` did, from its page before and `after`. */
|
|
2104
|
+
function settleMove(move, after, origin, volatile) {
|
|
2105
|
+
move.after = after;
|
|
2106
|
+
const fields = move.before ? differing(move.before, after, volatile) : [];
|
|
2107
|
+
const notes = move.before ? describeChange(after, fields, origin) : [];
|
|
2108
|
+
for (const dialog of move.dialogs) notes.push(`the page opened ${dialog}, which reeval accepted`);
|
|
2109
|
+
move.changed = fields.length > 0 || move.dialogs.length > 0;
|
|
2110
|
+
move.effect = notes.length ? notes.join("; ") : "nothing changed";
|
|
2111
|
+
}
|
|
2112
|
+
/** The same move made, unchanged, at the end of `moves`: a refused move ends the run. */
|
|
2113
|
+
function stuckMoves(moves) {
|
|
2114
|
+
const last = moves.at(-1);
|
|
2115
|
+
let n = 0;
|
|
2116
|
+
for (let i = moves.length - 1; i >= 0; i--) {
|
|
2117
|
+
const move = moves[i];
|
|
2118
|
+
if (!move || move.refused || move.key !== last?.key || move.changed !== false) break;
|
|
2119
|
+
n++;
|
|
2120
|
+
}
|
|
2121
|
+
return n;
|
|
2122
|
+
}
|
|
2123
|
+
/** What the model reads about the page now: where it is and what has focus. */
|
|
2124
|
+
function stateLines(now, origin, volatile) {
|
|
2125
|
+
const lines = [];
|
|
2126
|
+
const path = pathOf(now.url, origin);
|
|
2127
|
+
if (path) lines.push(`The page is at ${path}.`);
|
|
2128
|
+
if (now.focus === null) lines.push("Nothing is focused.");
|
|
2129
|
+
else if (now.focus) {
|
|
2130
|
+
lines.push(`Focused: ${now.focus}${now.value ? `, holding ${JSON.stringify(now.value)}` : ""}.`);
|
|
2131
|
+
if (now.focus.startsWith("select")) lines.push("A dropdown never opens in the screenshot: press ArrowDown or ArrowUp to the option, then Enter; it holds the new option only after Enter.");
|
|
2132
|
+
}
|
|
2133
|
+
if (volatile.has("text") || volatile.has("nodes") || volatile.has("view")) lines.push("Parts of the page keep changing on their own, so the effects below leave them out.");
|
|
2134
|
+
return lines;
|
|
2135
|
+
}
|
|
2136
|
+
/** The goal, where the journey is, the page now, and the moves so far with what each did. */
|
|
2137
|
+
function goalPrompt(step, earlier, now, moves, origin, volatile = /* @__PURE__ */ new Set()) {
|
|
2138
|
+
const lines = [`Goal: ${step.goal}`];
|
|
2139
|
+
if (step.total > 1) lines.push(`This is instruction ${step.index} of ${step.total}.${earlier.length ? ` Already done: ${earlier.map((g) => JSON.stringify(g)).join(", ")}.` : ""}`);
|
|
2140
|
+
lines.push(...stateLines(now, origin, volatile));
|
|
2141
|
+
if (!moves.length) {
|
|
2142
|
+
lines.push("You have made no moves for this goal yet.");
|
|
2143
|
+
return lines.join("\n");
|
|
2144
|
+
}
|
|
2145
|
+
lines.push("Your moves for this goal so far, oldest first. The screenshot shows the page now.");
|
|
2146
|
+
moves.forEach((m, i) => {
|
|
2147
|
+
const effect = m.refused ? `not made: ${m.refused}` : m.effect ?? "";
|
|
2148
|
+
lines.push(`${i + 1}. ${m.said}${effect ? ` → ${effect}` : ""}`);
|
|
2149
|
+
});
|
|
2150
|
+
if (stuckMoves(moves) >= 2) lines.push("Your last moves repeated the same action and nothing changed. Do not repeat it: if the goal is met, call done; otherwise try a different action.");
|
|
2151
|
+
return lines.join("\n");
|
|
2152
|
+
}
|
|
2153
|
+
async function runModelGoal(session, { step, earlier, origin, modelName, clock, volatile, pristine, loadModel }) {
|
|
2154
|
+
const { goal } = step;
|
|
1480
2155
|
const model = await loadModel(modelName);
|
|
1481
2156
|
const normalized = normalizesPoints(modelName);
|
|
1482
|
-
|
|
2157
|
+
const made = [];
|
|
2158
|
+
let dispatched = 0;
|
|
1483
2159
|
let usage = null;
|
|
1484
|
-
|
|
1485
|
-
|
|
2160
|
+
let now = await probePage(session);
|
|
2161
|
+
let stale = 0;
|
|
2162
|
+
while (made.length < 32) {
|
|
2163
|
+
clock.step = {
|
|
2164
|
+
...step,
|
|
2165
|
+
moves: dispatched
|
|
2166
|
+
};
|
|
2167
|
+
check(clock);
|
|
2168
|
+
const last = made.at(-1);
|
|
2169
|
+
if (last?.after) {
|
|
2170
|
+
const late = await probePage(session);
|
|
2171
|
+
last.dialogs.push(...session.dialogs());
|
|
2172
|
+
if (differing(last.after, late, volatile).length || last.dialogs.length) settleMove(last, late, origin, volatile);
|
|
2173
|
+
now = late;
|
|
2174
|
+
}
|
|
2175
|
+
if (stuckMoves(made) >= 5) throw new Error(`c.capture() instruction ${step.index} of ${step.total} ${JSON.stringify(goal)} repeated ${last?.said} 5 times and nothing changed`);
|
|
1486
2176
|
const ephemeral = await session.screenshot(true);
|
|
1487
2177
|
const page = pngSize(ephemeral);
|
|
1488
2178
|
debugTrace("check", "capture-shot", {
|
|
1489
2179
|
goal,
|
|
1490
2180
|
bytes: ephemeral.length,
|
|
1491
|
-
page
|
|
2181
|
+
page,
|
|
2182
|
+
url: now.url
|
|
1492
2183
|
}, false);
|
|
1493
|
-
|
|
1494
|
-
const
|
|
2184
|
+
const signal = AbortSignal.timeout(check(clock));
|
|
2185
|
+
const asked = Date.now();
|
|
2186
|
+
const result = await race(clock, () => generateText({
|
|
1495
2187
|
model,
|
|
1496
2188
|
system: systemPrompt(normalized),
|
|
1497
2189
|
messages: [{
|
|
1498
2190
|
role: "user",
|
|
1499
2191
|
content: [{
|
|
1500
2192
|
type: "text",
|
|
1501
|
-
text:
|
|
2193
|
+
text: goalPrompt(step, earlier, now, made, origin, volatile)
|
|
1502
2194
|
}, {
|
|
1503
|
-
type: "
|
|
1504
|
-
|
|
2195
|
+
type: "file",
|
|
2196
|
+
mediaType: "image/png",
|
|
2197
|
+
data: ephemeral
|
|
1505
2198
|
}]
|
|
1506
2199
|
}],
|
|
1507
2200
|
tools: useTools,
|
|
1508
2201
|
toolChoice: "required",
|
|
1509
2202
|
stopWhen: stepCountIs(1),
|
|
1510
|
-
abortSignal:
|
|
2203
|
+
abortSignal: signal
|
|
2204
|
+
})).catch((error) => {
|
|
2205
|
+
throw signal.aborted ? timedOut(clock) : error;
|
|
1511
2206
|
});
|
|
1512
2207
|
usage = addUsage(usage, asUsage(result.usage ?? {}));
|
|
1513
2208
|
const call = result.toolCalls?.[0];
|
|
1514
|
-
if (call)
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
2209
|
+
if (!call) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
|
|
2210
|
+
const input = call.input ?? {};
|
|
2211
|
+
const said = `${call.toolName} ${JSON.stringify(input)}`;
|
|
2212
|
+
const fresh = await probePage(session);
|
|
2213
|
+
const opened = session.dialogs();
|
|
2214
|
+
const moved = differing(now, fresh, volatile);
|
|
2215
|
+
if (!last) {
|
|
2216
|
+
if (pristine) {
|
|
2217
|
+
for (const f of moved) if (f !== "url" && f !== "focus") volatile.add(f);
|
|
2218
|
+
}
|
|
2219
|
+
} else if ((moved.length || opened.length) && stale < STALE_ASKS) {
|
|
2220
|
+
stale++;
|
|
2221
|
+
if (!last.refused) {
|
|
2222
|
+
last.dialogs.push(...opened);
|
|
2223
|
+
settleMove(last, fresh, origin, volatile);
|
|
2224
|
+
}
|
|
2225
|
+
now = fresh;
|
|
2226
|
+
debugLog("check", `capture use ${said} on ${JSON.stringify(goal)} not made: the page changed while the model chose`);
|
|
2227
|
+
continue;
|
|
2228
|
+
}
|
|
2229
|
+
stale = 0;
|
|
2230
|
+
let asks = actionFromTool(call.toolName, input);
|
|
2231
|
+
if (!("refused" in asks) && asks.type === "type" && now.editable === false) asks = { refused: "nothing that takes text is focused; click a field first" };
|
|
2232
|
+
const logged = !("refused" in asks) && asks.type === "type" && now.focus?.includes("[password]") ? `type ${JSON.stringify({
|
|
2233
|
+
...input,
|
|
2234
|
+
text: "•".repeat(String(input.text ?? "").length)
|
|
2235
|
+
})}` : said;
|
|
2236
|
+
const latency = formatDuration(Date.now() - asked);
|
|
2237
|
+
if ("refused" in asks) {
|
|
2238
|
+
debugLog("check", `capture use ${logged} on ${JSON.stringify(goal)} not made: ${asks.refused} · ${latency}`);
|
|
2239
|
+
made.push({
|
|
2240
|
+
said,
|
|
2241
|
+
key: said,
|
|
2242
|
+
refused: asks.refused,
|
|
2243
|
+
dialogs: []
|
|
2244
|
+
});
|
|
2245
|
+
continue;
|
|
2246
|
+
}
|
|
2247
|
+
const chosen = normalized ? scalePoints(asks, page) : asks;
|
|
2248
|
+
debugLog("check", `capture use ${logged} on ${JSON.stringify(goal)} · ${latency}`);
|
|
1518
2249
|
debugTrace("check", "capture-use", {
|
|
1519
2250
|
goal,
|
|
1520
2251
|
action: chosen,
|
|
1521
2252
|
page
|
|
1522
2253
|
}, `capture use ${chosen.type}`);
|
|
1523
2254
|
if (chosen.type === "done") return {
|
|
1524
|
-
moves,
|
|
2255
|
+
moves: dispatched,
|
|
1525
2256
|
usage
|
|
1526
2257
|
};
|
|
2258
|
+
const move = {
|
|
2259
|
+
said,
|
|
2260
|
+
key: moveKey(asks),
|
|
2261
|
+
before: now,
|
|
2262
|
+
dialogs: []
|
|
2263
|
+
};
|
|
2264
|
+
made.push(move);
|
|
1527
2265
|
await session.dispatch(chosen, page);
|
|
1528
|
-
|
|
1529
|
-
await
|
|
2266
|
+
dispatched++;
|
|
2267
|
+
now = await settledState(session, volatile, clock.remaining);
|
|
2268
|
+
move.dialogs.push(...session.dialogs());
|
|
2269
|
+
settleMove(move, now, origin, volatile);
|
|
1530
2270
|
}
|
|
1531
|
-
throw
|
|
2271
|
+
throw unfinished(step);
|
|
1532
2272
|
}
|
|
1533
2273
|
/**
|
|
1534
2274
|
* Top-level wait, then the instruction loop, then the saved PNG.
|
|
1535
2275
|
* Throws before `screenshot` when the budget or move cap is hit.
|
|
1536
2276
|
*/
|
|
1537
|
-
async function runJourney(
|
|
2277
|
+
async function runJourney(raw, url, options) {
|
|
1538
2278
|
const { wait, timeoutMs, fullPage, instructions, use, remaining } = options;
|
|
1539
|
-
|
|
1540
|
-
|
|
2279
|
+
const clock = {
|
|
2280
|
+
budgetMs: options.budgetMs ?? timeoutMs,
|
|
2281
|
+
remaining,
|
|
2282
|
+
detail: wait === "idle" ? "loading the page with wait: \"idle\"" : "loading the page"
|
|
2283
|
+
};
|
|
2284
|
+
const session = onClock(raw, clock);
|
|
2285
|
+
const origin = new URL(url).origin;
|
|
2286
|
+
const volatile = /* @__PURE__ */ new Set();
|
|
2287
|
+
check(clock);
|
|
2288
|
+
await navigateAndWait(session, url, wait, Math.min(timeoutMs, remaining())).catch((error) => {
|
|
2289
|
+
throw /c\.capture\(\) timed out/.test(String(error)) ? timedOut(clock) : error;
|
|
2290
|
+
});
|
|
2291
|
+
const goals = instructions.filter((s) => typeof s === "string");
|
|
1541
2292
|
let moves = 0;
|
|
1542
2293
|
let usage = null;
|
|
1543
2294
|
let stringIndex = 0;
|
|
1544
|
-
for (const
|
|
1545
|
-
if (typeof
|
|
1546
|
-
|
|
1547
|
-
|
|
2295
|
+
for (const instruction of instructions) {
|
|
2296
|
+
if (typeof instruction !== "string") {
|
|
2297
|
+
clock.detail = `{ wait: ${JSON.stringify(instruction.wait)} }`;
|
|
2298
|
+
clock.step = void 0;
|
|
2299
|
+
check(clock);
|
|
2300
|
+
await session.wait(instruction.wait, remaining(), void 0);
|
|
1548
2301
|
continue;
|
|
1549
2302
|
}
|
|
1550
|
-
|
|
1551
|
-
|
|
2303
|
+
const step = {
|
|
2304
|
+
goal: instruction,
|
|
2305
|
+
index: stringIndex + 1,
|
|
2306
|
+
total: goals.length,
|
|
2307
|
+
moves: 0
|
|
2308
|
+
};
|
|
2309
|
+
clock.detail = JSON.stringify(instruction);
|
|
2310
|
+
clock.step = step;
|
|
2311
|
+
check(clock);
|
|
2312
|
+
debugLog("check", `capture instruction ${step.index} of ${step.total} ${JSON.stringify(instruction)}`);
|
|
1552
2313
|
if (use && "fake" in use) {
|
|
1553
2314
|
const script = use.fake[stringIndex] ?? [];
|
|
1554
|
-
moves += await runFakeScript(session, script, step,
|
|
2315
|
+
moves += await runFakeScript(session, script, step, clock, volatile);
|
|
1555
2316
|
} else if (use && "model" in use) {
|
|
1556
2317
|
if (!options.loadModel) throw new Error("c.capture() instructions need a model loader");
|
|
1557
|
-
const ran = await runModelGoal(session,
|
|
2318
|
+
const ran = await runModelGoal(session, {
|
|
2319
|
+
step,
|
|
2320
|
+
earlier: goals.slice(0, stringIndex),
|
|
2321
|
+
origin,
|
|
2322
|
+
modelName: use.model,
|
|
2323
|
+
clock,
|
|
2324
|
+
volatile,
|
|
2325
|
+
pristine: moves === 0,
|
|
2326
|
+
loadModel: options.loadModel
|
|
2327
|
+
});
|
|
1558
2328
|
moves += ran.moves;
|
|
1559
2329
|
usage = addUsage(usage, ran.usage);
|
|
1560
2330
|
}
|
|
1561
2331
|
stringIndex++;
|
|
1562
|
-
await settle();
|
|
1563
2332
|
}
|
|
1564
|
-
|
|
2333
|
+
clock.detail = "the final screenshot";
|
|
2334
|
+
clock.step = void 0;
|
|
2335
|
+
check(clock);
|
|
1565
2336
|
return {
|
|
1566
2337
|
png: await session.screenshot(fullPage),
|
|
1567
2338
|
moves,
|
|
@@ -1700,12 +2471,12 @@ function createChecks(a) {
|
|
|
1700
2471
|
tries: retries + 1
|
|
1701
2472
|
} : {}
|
|
1702
2473
|
});
|
|
1703
|
-
return r;
|
|
2474
|
+
return Object.defineProperty(r, SUBJECT, { value: command });
|
|
1704
2475
|
};
|
|
1705
2476
|
function exec(command, options) {
|
|
1706
2477
|
const problem = options === void 0 ? void 0 : execOptionsProblem(options);
|
|
1707
2478
|
if (problem) return Promise.reject(new UsageError(problem));
|
|
1708
|
-
if (options?.background) return import("./capture-
|
|
2479
|
+
if (options?.background) return import("./capture-px7Ey95_.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
|
|
1709
2480
|
const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
|
|
1710
2481
|
return Object.defineProperty(run, SUBJECT, { value: command });
|
|
1711
2482
|
}
|
|
@@ -1729,7 +2500,7 @@ function createChecks(a) {
|
|
|
1729
2500
|
}
|
|
1730
2501
|
//#endregion
|
|
1731
2502
|
//#region src/attempt/capture.ts
|
|
1732
|
-
const BG_LOG = "/tmp/reeval-bg
|
|
2503
|
+
const BG_LOG = "/tmp/reeval-bg";
|
|
1733
2504
|
const BG_TAG = "REEVAL_BACKGROUND";
|
|
1734
2505
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
1735
2506
|
const languageModel = (model) => typeof model === "string" && !isEvaluationModel(model) ? model : void 0;
|
|
@@ -1779,10 +2550,14 @@ kill -9 -${group} 2>/dev/null`;
|
|
|
1779
2550
|
}
|
|
1780
2551
|
/** Starts `command` without waiting for exit. Teardown kills it if the eval didn't. */
|
|
1781
2552
|
async function startBackground(a, command, who, extraEnv = {}) {
|
|
2553
|
+
return (await launch(a, command, who, extraEnv)).handle;
|
|
2554
|
+
}
|
|
2555
|
+
async function launch(a, command, who, extraEnv) {
|
|
1782
2556
|
const box = await sandboxFor(a, who);
|
|
1783
2557
|
const docker = a.spec.sandbox.kind === "docker";
|
|
1784
|
-
const wrapped = `${docker ? "bash" : "sh"} -c ${shellQuote(command)} >${BG_LOG} 2>&1 & echo $$ $!`;
|
|
1785
2558
|
const tag = randomUUID();
|
|
2559
|
+
const log = `${BG_LOG}-${tag.slice(0, 8)}.log`;
|
|
2560
|
+
const wrapped = `${docker ? "bash" : "sh"} -c ${shellQuote(command)} >${log} 2>&1 & echo $$ $!`;
|
|
1786
2561
|
const r = await asCheck(a, () => box.exec(wrapped, {
|
|
1787
2562
|
signal: a.signal,
|
|
1788
2563
|
env: {
|
|
@@ -1798,16 +2573,53 @@ async function startBackground(a, command, who, extraEnv = {}) {
|
|
|
1798
2573
|
const handle = { async kill() {
|
|
1799
2574
|
if (killed) return;
|
|
1800
2575
|
killed = true;
|
|
1801
|
-
await box.exec(`${tree}; kill -9 ${pid} 2>/dev/null; true`).catch(() => {});
|
|
2576
|
+
await box.exec(`${tree}; kill -9 ${pid} 2>/dev/null; rm -f ${log}; true`).catch(() => {});
|
|
1802
2577
|
} };
|
|
1803
2578
|
a.backgrounds.push(handle);
|
|
1804
|
-
return
|
|
2579
|
+
return {
|
|
2580
|
+
handle,
|
|
2581
|
+
pid,
|
|
2582
|
+
log
|
|
2583
|
+
};
|
|
1805
2584
|
}
|
|
2585
|
+
/**
|
|
2586
|
+
* Starts `command` with `PORT` set and returns once it listens: on `PORT` (or `port`), else on
|
|
2587
|
+
* the one port that wasn't listening before it started, which it records.
|
|
2588
|
+
*/
|
|
1806
2589
|
async function serveCommand(a, command, port) {
|
|
1807
|
-
const
|
|
2590
|
+
const want = port ?? await allocatePort();
|
|
2591
|
+
const box = await sandboxFor(a, "c.serve()");
|
|
2592
|
+
const tables = a.spec.sandbox.kind !== "fake";
|
|
2593
|
+
const look = (pid) => untraced(() => box.exec(probeScript(pid, tables), { signal: a.signal })).then((r) => parseProbe(r.stdout));
|
|
2594
|
+
const first = await look();
|
|
2595
|
+
const before = new Set(first.proc ? first.listeners.map((l) => l.port) : []);
|
|
2596
|
+
const { handle, pid, log } = await launch(a, command, "c.serve()", { PORT: String(want) });
|
|
2597
|
+
const found = await awaitListening(command, {
|
|
2598
|
+
want,
|
|
2599
|
+
given: port !== void 0,
|
|
2600
|
+
before,
|
|
2601
|
+
signal: a.signal,
|
|
2602
|
+
probe: () => look(pid),
|
|
2603
|
+
output: async () => (await box.exec(`tail -c 65536 ${log} 2>/dev/null`)).stdout,
|
|
2604
|
+
...tables ? { hostListening: async () => "" } : {}
|
|
2605
|
+
}).catch(async (error) => {
|
|
2606
|
+
await handle.kill();
|
|
2607
|
+
throw error;
|
|
2608
|
+
});
|
|
2609
|
+
const serve = {
|
|
2610
|
+
command,
|
|
2611
|
+
port: found.port,
|
|
2612
|
+
portEnv: want,
|
|
2613
|
+
detected: found.detected,
|
|
2614
|
+
address: found.address,
|
|
2615
|
+
readyMs: found.readyMs
|
|
2616
|
+
};
|
|
2617
|
+
a.record.serves = [...a.record.serves ?? [], serve];
|
|
2618
|
+
const at = found.address ? `${found.address} ` : "";
|
|
2619
|
+
debugLog("check", found.detected ? `c.serve ${command}: listening on ${at}port ${found.port}, not PORT ${want}; using ${found.port}` : `c.serve ${command}: listening on ${at}port ${found.port} after ${found.readyMs}ms`);
|
|
1808
2620
|
return {
|
|
1809
|
-
...
|
|
1810
|
-
port:
|
|
2621
|
+
...handle,
|
|
2622
|
+
port: found.port
|
|
1811
2623
|
};
|
|
1812
2624
|
}
|
|
1813
2625
|
async function capturePage(a, target, call) {
|
|
@@ -1816,8 +2628,9 @@ async function capturePage(a, target, call) {
|
|
|
1816
2628
|
const resolved = resolveCaptureCall(target, call, a.spec.capture);
|
|
1817
2629
|
if (a.captureNames.has(resolved.name)) throw new Error(`c.capture() reused the name "${resolved.name}"`);
|
|
1818
2630
|
const box = await sandboxFor(a, "c.capture()");
|
|
1819
|
-
|
|
1820
|
-
|
|
2631
|
+
const tunnel = box.getPortEndpoint?.bind(box);
|
|
2632
|
+
if (!tunnel) throw new Error("c.capture() needs a sandbox that can tunnel a port");
|
|
2633
|
+
const { url: origin } = await tunnel({
|
|
1821
2634
|
port: resolved.port,
|
|
1822
2635
|
protocol: "http"
|
|
1823
2636
|
});
|
|
@@ -1837,7 +2650,10 @@ async function capturePage(a, target, call) {
|
|
|
1837
2650
|
use
|
|
1838
2651
|
}, remaining, (n) => {
|
|
1839
2652
|
moves = n;
|
|
1840
|
-
})
|
|
2653
|
+
}, (port) => tunnel({
|
|
2654
|
+
port,
|
|
2655
|
+
protocol: "http"
|
|
2656
|
+
}).then((t) => t.url));
|
|
1841
2657
|
const digest = sha256(png);
|
|
1842
2658
|
const file = {
|
|
1843
2659
|
path: `artifacts/${resolved.name}.png`,
|
|
@@ -1873,19 +2689,26 @@ async function capturePage(a, target, call) {
|
|
|
1873
2689
|
file
|
|
1874
2690
|
};
|
|
1875
2691
|
}
|
|
1876
|
-
async function shot(a, url, resolved, remaining, onMoves) {
|
|
2692
|
+
async function shot(a, url, resolved, remaining, onMoves, tunnel) {
|
|
1877
2693
|
const cache = cacheDir();
|
|
1878
|
-
|
|
2694
|
+
const bin = await ensureChrome(cache, {
|
|
1879
2695
|
cacheLabel: cacheLabel(cache),
|
|
1880
2696
|
signal: a.signal,
|
|
1881
2697
|
onProgress: (p) => a.report({
|
|
1882
2698
|
kind: "browser-download",
|
|
1883
2699
|
...p
|
|
1884
2700
|
})
|
|
1885
|
-
})
|
|
2701
|
+
});
|
|
2702
|
+
const proxy = await startLoopbackProxy({
|
|
2703
|
+
tunnel,
|
|
2704
|
+
direct: [Number(new URL(url).port)]
|
|
2705
|
+
});
|
|
2706
|
+
const browser = { proxy: proxy.address };
|
|
2707
|
+
return withChrome(bin, resolved.window, remaining(), browser, async (session) => {
|
|
1886
2708
|
const result = await runJourney(session, url, {
|
|
1887
2709
|
wait: resolved.wait,
|
|
1888
2710
|
timeoutMs: remaining(),
|
|
2711
|
+
budgetMs: resolved.timeoutMs,
|
|
1889
2712
|
fullPage: resolved.fullPage,
|
|
1890
2713
|
instructions: resolved.instructions,
|
|
1891
2714
|
use: resolved.use,
|
|
@@ -1911,6 +2734,9 @@ async function shot(a, url, resolved, remaining, onMoves) {
|
|
|
1911
2734
|
} : spend;
|
|
1912
2735
|
}
|
|
1913
2736
|
return result.png;
|
|
2737
|
+
}).finally(async () => {
|
|
2738
|
+
await proxy.close();
|
|
2739
|
+
if (proxy.reached.length) debugLog("check", `c.capture ${resolved.name}: sandbox port ${proxy.reached.join(", ")} answered the page through its tunnel`);
|
|
1914
2740
|
});
|
|
1915
2741
|
}
|
|
1916
2742
|
/** Writes PNG sidecars next to the staging eval JSON so redact never sees the bytes. */
|
|
@@ -1923,4 +2749,4 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
|
|
|
1923
2749
|
//#endregion
|
|
1924
2750
|
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, attemptRequestProblem as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
|
|
1925
2751
|
|
|
1926
|
-
//# sourceMappingURL=capture-
|
|
2752
|
+
//# sourceMappingURL=capture-iQ6kYJV0.js.map
|