reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,25 +1,506 @@
|
|
|
1
1
|
import { n as isOneOf } from "./object-CLf2-tNq.js";
|
|
2
|
-
import {
|
|
2
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
|
+
import { c as jsonValueSchema, h as parseDuration, i as durationSchema, m as TIMER_LIMIT, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
4
|
+
import { y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
|
|
3
5
|
import { l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
6
|
+
import { Ct as captureWindowSchema, Et as judgeModelNameSchema, Ot as judgeStateSchema, Tt as judgeFields, b as resolveModelGateway, v as gatewayScope, wt as fakeJudgeDataSchema } from "./config-k6m7XbAz.js";
|
|
7
|
+
import { y as vertexSettingsIn } from "./vertex-DypYjx71.js";
|
|
8
|
+
import { m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
9
|
+
import { i as formatDuration, n as clip } from "./text-DTigcA64.js";
|
|
10
|
+
import { s as cacheDir } from "./root-Dlff-zp8.js";
|
|
11
|
+
import { ft as shellQuote } from "./src-D91Hd6hT.js";
|
|
7
12
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
8
|
-
import {
|
|
9
|
-
import { i as debugTrace, r as debugLog } from "./attempt-
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { t as addUsage } from "./usage-
|
|
13
|
+
import { r as getRunContext, s as SUBJECT } from "./runtime-d56Gg3J5.js";
|
|
14
|
+
import { i as debugTrace, r as debugLog } from "./attempt-Ba2QerOK.js";
|
|
15
|
+
import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
|
|
16
|
+
import { i as hostModel } from "./model-Bq3G0OG1.js";
|
|
17
|
+
import { t as addUsage } from "./usage-BNY1ABrU.js";
|
|
13
18
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
14
19
|
import { homedir, tmpdir } from "node:os";
|
|
15
20
|
import { join, posix } from "node:path";
|
|
16
21
|
import { z } from "zod";
|
|
17
22
|
import { createServer } from "node:net";
|
|
18
23
|
import { createHash, randomUUID } from "node:crypto";
|
|
24
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
19
25
|
import { spawn } from "node:child_process";
|
|
20
26
|
import { crc32, deflateSync, inflateRawSync } from "node:zlib";
|
|
21
27
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
22
28
|
import { generateText, stepCountIs, tool } from "ai";
|
|
29
|
+
//#region src/contract/attempt.ts
|
|
30
|
+
const nullableNumber = z.number().nullable();
|
|
31
|
+
const tokenUsageSchema = z.strictObject({
|
|
32
|
+
input: z.number(),
|
|
33
|
+
output: z.number(),
|
|
34
|
+
cacheRead: z.number(),
|
|
35
|
+
cacheWrite: z.number(),
|
|
36
|
+
reasoning: nullableNumber
|
|
37
|
+
});
|
|
38
|
+
const fileChangeSchema = z.strictObject({
|
|
39
|
+
path: z.string(),
|
|
40
|
+
status: z.enum([
|
|
41
|
+
"added",
|
|
42
|
+
"modified",
|
|
43
|
+
"deleted"
|
|
44
|
+
])
|
|
45
|
+
});
|
|
46
|
+
const diffSchema = z.strictObject({
|
|
47
|
+
patch: z.string(),
|
|
48
|
+
files: z.array(fileChangeSchema)
|
|
49
|
+
});
|
|
50
|
+
const turnStatusSchema = z.enum([
|
|
51
|
+
"completed",
|
|
52
|
+
"aborted",
|
|
53
|
+
"error"
|
|
54
|
+
]);
|
|
55
|
+
const webEventTypeSchema = z.enum(["web-search", "web-fetch"]);
|
|
56
|
+
const at = {
|
|
57
|
+
ts: z.string(),
|
|
58
|
+
atMs: z.number()
|
|
59
|
+
};
|
|
60
|
+
const transcriptEventSchema = z.discriminatedUnion("type", [
|
|
61
|
+
z.strictObject({
|
|
62
|
+
...at,
|
|
63
|
+
type: z.enum(["text", "reasoning"]),
|
|
64
|
+
text: z.string(),
|
|
65
|
+
durationMs: nullableNumber
|
|
66
|
+
}),
|
|
67
|
+
z.strictObject({
|
|
68
|
+
...at,
|
|
69
|
+
type: z.literal("file-edit"),
|
|
70
|
+
toolCallId: z.string(),
|
|
71
|
+
path: z.string(),
|
|
72
|
+
kind: z.enum([
|
|
73
|
+
"create",
|
|
74
|
+
"modify",
|
|
75
|
+
"write",
|
|
76
|
+
"delete"
|
|
77
|
+
]),
|
|
78
|
+
added: nullableNumber,
|
|
79
|
+
removed: nullableNumber,
|
|
80
|
+
durationMs: nullableNumber,
|
|
81
|
+
stalled: z.boolean().optional()
|
|
82
|
+
}),
|
|
83
|
+
z.strictObject({
|
|
84
|
+
...at,
|
|
85
|
+
type: z.literal("web-search"),
|
|
86
|
+
toolCallId: z.string(),
|
|
87
|
+
query: z.string(),
|
|
88
|
+
references: nullableNumber,
|
|
89
|
+
ok: z.boolean().optional(),
|
|
90
|
+
durationMs: nullableNumber,
|
|
91
|
+
stalled: z.boolean().optional()
|
|
92
|
+
}),
|
|
93
|
+
z.strictObject({
|
|
94
|
+
...at,
|
|
95
|
+
type: z.literal("web-fetch"),
|
|
96
|
+
toolCallId: z.string(),
|
|
97
|
+
url: z.string(),
|
|
98
|
+
ok: z.boolean(),
|
|
99
|
+
durationMs: nullableNumber,
|
|
100
|
+
stalled: z.boolean().optional()
|
|
101
|
+
}),
|
|
102
|
+
z.strictObject({
|
|
103
|
+
...at,
|
|
104
|
+
type: z.literal("tool"),
|
|
105
|
+
toolCallId: z.string(),
|
|
106
|
+
tool: z.string(),
|
|
107
|
+
input: z.unknown(),
|
|
108
|
+
output: z.unknown(),
|
|
109
|
+
ok: z.boolean(),
|
|
110
|
+
durationMs: nullableNumber,
|
|
111
|
+
stalled: z.boolean().optional()
|
|
112
|
+
}),
|
|
113
|
+
z.strictObject({
|
|
114
|
+
...at,
|
|
115
|
+
type: z.literal("error"),
|
|
116
|
+
message: z.string()
|
|
117
|
+
}),
|
|
118
|
+
z.strictObject({
|
|
119
|
+
...at,
|
|
120
|
+
type: z.literal("retry"),
|
|
121
|
+
message: z.string()
|
|
122
|
+
})
|
|
123
|
+
]);
|
|
124
|
+
const turnDataSchema = z.strictObject({
|
|
125
|
+
prompt: z.string(),
|
|
126
|
+
text: z.string(),
|
|
127
|
+
status: turnStatusSchema,
|
|
128
|
+
startedAt: z.string(),
|
|
129
|
+
durationMs: z.number(),
|
|
130
|
+
transcript: z.array(transcriptEventSchema),
|
|
131
|
+
usage: tokenUsageSchema.nullable(),
|
|
132
|
+
model: z.string().nullable(),
|
|
133
|
+
unreported: z.array(webEventTypeSchema),
|
|
134
|
+
diff: diffSchema
|
|
135
|
+
});
|
|
136
|
+
const execResultSchema = z.strictObject({
|
|
137
|
+
exitCode: z.number().int(),
|
|
138
|
+
stdout: z.string(),
|
|
139
|
+
stderr: z.string()
|
|
140
|
+
});
|
|
141
|
+
const execRetrySchema = z.strictObject({
|
|
142
|
+
retries: z.number().int("must be a whole number, from 0").min(0, "must be a whole number, from 0").optional(),
|
|
143
|
+
interval: z.number().min(0, "must be 0 or more milliseconds").max(MAX_TIMER_MS, `must be ${TIMER_LIMIT}`).optional()
|
|
144
|
+
});
|
|
145
|
+
/** A background command has no exit for `retries` to wait on. */
|
|
146
|
+
const noBackgroundRetry = (options, ctx) => {
|
|
147
|
+
if (!options.background) return;
|
|
148
|
+
for (const key of ["retries", "interval"]) {
|
|
149
|
+
if (options[key] === void 0) continue;
|
|
150
|
+
ctx.addIssue({
|
|
151
|
+
code: "custom",
|
|
152
|
+
path: [key],
|
|
153
|
+
message: "can't be used with background: true, whose exit reeval doesn't wait for: drop one of them"
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
const execOptionsSchema = z.strictObject({
|
|
158
|
+
background: z.boolean().optional(),
|
|
159
|
+
...execRetrySchema.shape
|
|
160
|
+
}).superRefine(noBackgroundRetry);
|
|
161
|
+
/** What's wrong with `c.exec()`'s options, or undefined. */
|
|
162
|
+
const execOptionsProblem = (value) => dataProblem(execOptionsSchema, value, "c.exec()", () => Object.keys(execOptionsSchema.shape));
|
|
163
|
+
const assertionRecordSchema = z.strictObject({
|
|
164
|
+
label: z.string(),
|
|
165
|
+
labelled: z.boolean(),
|
|
166
|
+
matcher: z.string(),
|
|
167
|
+
status: z.enum([
|
|
168
|
+
"pass",
|
|
169
|
+
"fail",
|
|
170
|
+
"error"
|
|
171
|
+
]),
|
|
172
|
+
soft: z.boolean(),
|
|
173
|
+
message: z.string().optional(),
|
|
174
|
+
durationMs: z.number().optional()
|
|
175
|
+
});
|
|
176
|
+
const turnIndex = z.number().int("must be a turn index, from 0").min(0, "must be a turn index, from 0").describe("The turn's index, from 0.");
|
|
177
|
+
const captureSchema = z.strictObject({
|
|
178
|
+
name: z.string(),
|
|
179
|
+
port: z.number(),
|
|
180
|
+
path: z.string(),
|
|
181
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]),
|
|
182
|
+
file: z.strictObject({
|
|
183
|
+
path: z.string(),
|
|
184
|
+
sha256: z.string(),
|
|
185
|
+
bytes: z.number()
|
|
186
|
+
})
|
|
187
|
+
});
|
|
188
|
+
const judgeArtifactRefSchema = z.strictObject({
|
|
189
|
+
name: z.string(),
|
|
190
|
+
path: z.string(),
|
|
191
|
+
sha256: z.string(),
|
|
192
|
+
bytes: z.number(),
|
|
193
|
+
mediaType: z.literal("image/png")
|
|
194
|
+
});
|
|
195
|
+
const verdictFields = {
|
|
196
|
+
threshold: z.number(),
|
|
197
|
+
question: z.string(),
|
|
198
|
+
model: z.string(),
|
|
199
|
+
gateway: gatewayNameSchema.optional(),
|
|
200
|
+
artifacts: z.array(judgeArtifactRefSchema).optional()
|
|
201
|
+
};
|
|
202
|
+
const verdictSchema = z.strictObject({
|
|
203
|
+
verdict: z.enum(["pass", "fail"]),
|
|
204
|
+
score: z.number(),
|
|
205
|
+
...verdictFields,
|
|
206
|
+
resolvedModel: z.string().optional(),
|
|
207
|
+
upstream: z.string().optional(),
|
|
208
|
+
costUsd: z.number().optional()
|
|
209
|
+
});
|
|
210
|
+
const judgeFailureSchema = z.strictObject({
|
|
211
|
+
verdict: z.literal("error"),
|
|
212
|
+
score: z.null(),
|
|
213
|
+
...verdictFields,
|
|
214
|
+
error: z.string()
|
|
215
|
+
});
|
|
216
|
+
/** A pattern, for the matchers that take a string or a `RegExp`. */
|
|
217
|
+
const patternSchema = z.strictObject({
|
|
218
|
+
regexp: z.string().describe("The pattern's source, as `RegExp.prototype.source` gives it."),
|
|
219
|
+
flags: z.string().regex(/^[dgimsuvy]*$/, "must use only the flags d, g, i, m, s, u, v and y").optional().describe("Its flags, e.g. `i`.")
|
|
220
|
+
}).describe("A regular expression, where a matcher takes a string or a `RegExp`.");
|
|
221
|
+
/** A pattern in the runner's own language, which reeval never compiles. */
|
|
222
|
+
function hasAnsweredPattern(value) {
|
|
223
|
+
if (Array.isArray(value)) return value.some(hasAnsweredPattern);
|
|
224
|
+
if (typeof value !== "object" || value === null) return false;
|
|
225
|
+
return "matchedBy" in value || Object.values(value).some(hasAnsweredPattern);
|
|
226
|
+
}
|
|
227
|
+
const answeredPatternSchema = z.strictObject({
|
|
228
|
+
matchedBy: z.string().regex(/\S/).describe("The pattern as the runner's language writes it, e.g. `re.compile('npm t')`."),
|
|
229
|
+
answers: z.record(z.string(), z.boolean()).describe("Whether it matches each string the runner has been asked about.")
|
|
230
|
+
}).describe("A pattern the runner matches in its own language. reeval runs the matcher with it, and when the matcher tests a string not in `answers` replies with those strings (`untested`) instead of a record; the runner matches them and makes the call again with them in `answers`.");
|
|
231
|
+
const isPatternLike = (value) => typeof value === "object" && value !== null && !Array.isArray(value) && "regexp" in value;
|
|
232
|
+
const matcherArgSchema = z.union([
|
|
233
|
+
patternSchema,
|
|
234
|
+
answeredPatternSchema,
|
|
235
|
+
jsonValueSchema
|
|
236
|
+
]).superRefine((value, ctx) => {
|
|
237
|
+
if (!isPatternLike(value)) return;
|
|
238
|
+
for (const issue of patternSchema.safeParse(value).error?.issues ?? []) ctx.addIssue(issue);
|
|
239
|
+
}).describe("One argument to a matcher: a JSON value, a regular expression as `{ regexp, flags? }`, or a pattern in the runner's own language as `{ matchedBy, answers }`.");
|
|
240
|
+
const matcherSubjectSchema = z.union([
|
|
241
|
+
z.strictObject({ turn: turnIndex }),
|
|
242
|
+
z.strictObject({ attempt: z.literal(true).describe("The attempt itself, `c`.") }),
|
|
243
|
+
z.strictObject({ value: jsonValueSchema.describe("A value the eval passes as it is.") }),
|
|
244
|
+
z.strictObject({
|
|
245
|
+
exec: z.string().regex(/\S/).describe("A command reeval runs, for its `ExecResult`; again on each try when polled."),
|
|
246
|
+
result: execResultSchema.optional().describe("The command's `ExecResult` when the runner already ran it: reeval checks that instead of running it again, and the check is still named by the command.")
|
|
247
|
+
}),
|
|
248
|
+
z.strictObject({ readFile: z.string().regex(/\S/).describe("A file reeval reads, for its contents (null when there's none).") })
|
|
249
|
+
]).describe("What `expect` was given: a turn `c.send` returned, the attempt (for its files and latest turn), a value, such as an `ExecResult`, or a command or file reeval reads for it.");
|
|
250
|
+
const nonEmpty = z.string().regex(/\S/, "must not be empty");
|
|
251
|
+
const CAPTURE_NAME = new RegExp(`^[a-z0-9][a-z0-9_-]{0,99}$`);
|
|
252
|
+
const CAPTURE_NAME_RULE = `must be 1 to 100 of a-z, 0-9, _ and -, starting with a letter or digit`;
|
|
253
|
+
const portSchema = z.number().int().min(1, "a port from 1 to 65535").max(65535, "a port from 1 to 65535");
|
|
254
|
+
const millisSchema = z.number().positive("more than zero milliseconds").max(MAX_TIMER_MS, TIMER_LIMIT);
|
|
255
|
+
const captureWaitSchema = z.enum(["load", "idle"]);
|
|
256
|
+
const waitSchema = z.union([captureWaitSchema, z.strictObject({ ms: millisSchema })]);
|
|
257
|
+
const captureInstructionSchema = z.union([nonEmpty, z.strictObject({ wait: waitSchema })]);
|
|
258
|
+
const captureCallDataSchema = z.strictObject({
|
|
259
|
+
name: z.string().regex(CAPTURE_NAME, CAPTURE_NAME_RULE).optional(),
|
|
260
|
+
path: z.string().optional(),
|
|
261
|
+
viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional(),
|
|
262
|
+
timeout: durationSchema.optional(),
|
|
263
|
+
wait: waitSchema.optional(),
|
|
264
|
+
instructions: z.array(captureInstructionSchema).optional().describe("Journey: goal strings and `{ wait }` beats, run by the vision model."),
|
|
265
|
+
model: z.string().optional()
|
|
266
|
+
}).describe("`c.capture()`'s camera and path as data. Default `path` is `/`; default `name` is derived from the path. A test's `fakeInstructions` journey is TypeScript-only.");
|
|
267
|
+
const handleId = nonEmpty.describe("A background process, as an exec or serve reply named it.");
|
|
268
|
+
const judgeOnSchema = z.union([z.strictObject({ turn: turnIndex }), z.strictObject({ value: jsonValueSchema.describe("Graded as the output.") })]);
|
|
269
|
+
const judgeCallDataSchema = z.strictObject({
|
|
270
|
+
model: z.union([judgeModelNameSchema, fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge with a numeric score."),
|
|
271
|
+
...judgeFields,
|
|
272
|
+
on: judgeOnSchema.optional().describe("What to grade instead of the latest turn: an earlier turn, or a value."),
|
|
273
|
+
artifacts: z.union([z.literal(true), z.array(z.union([nonEmpty, captureSchema]))]).optional().describe("Captures this call sends the judge as image file parts: a capture `c.capture` returned, or its name, or `true` for every capture so far. The provider that cannot take files fails the call.")
|
|
274
|
+
}).describe("`c.judge()`'s options as data: the judge options, with the model by name or a fake judge's numeric score, plus the call-only `on` and `artifacts`.");
|
|
275
|
+
const attemptCallSchema = z.discriminatedUnion("op", [
|
|
276
|
+
z.strictObject({
|
|
277
|
+
op: z.literal("send"),
|
|
278
|
+
prompt: z.string().describe("The turn's prompt.")
|
|
279
|
+
}).describe("`c.send(prompt)`: the next turn of the agent session. Replies with a turn."),
|
|
280
|
+
z.strictObject({
|
|
281
|
+
op: z.literal("exec"),
|
|
282
|
+
command: nonEmpty.describe("A shell command, run in the workspace."),
|
|
283
|
+
background: z.boolean().optional().describe("Leave it running and reply with its handle, not its output."),
|
|
284
|
+
...execRetrySchema.shape
|
|
285
|
+
}).superRefine(noBackgroundRetry).describe("`c.exec(command, { background?, retries?, interval? })`: a command in the sandbox. Replies with its exit code and output (the last try, with `retries`), or with a handle when it runs in the background."),
|
|
286
|
+
z.strictObject({
|
|
287
|
+
op: z.literal("serve"),
|
|
288
|
+
command: nonEmpty.describe("A shell command; `PORT` is set to the port."),
|
|
289
|
+
port: portSchema.optional().describe("The container port. Omit it and reeval allocates one.")
|
|
290
|
+
}).describe("`c.serve(command, port?)`: a background process and its port."),
|
|
291
|
+
z.strictObject({
|
|
292
|
+
op: z.literal("kill"),
|
|
293
|
+
handle: handleId
|
|
294
|
+
}).describe("`handle.kill()`: stops a background process. Killing one that's gone is fine."),
|
|
295
|
+
z.strictObject({
|
|
296
|
+
op: z.literal("capture"),
|
|
297
|
+
target: z.union([z.strictObject({ port: portSchema }), z.strictObject({ handle: handleId })]).describe("A container port, or a serve handle by its id."),
|
|
298
|
+
options: captureCallDataSchema.optional()
|
|
299
|
+
}).describe("`c.capture(target, options)`: screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`. Replies with the capture."),
|
|
300
|
+
z.strictObject({
|
|
301
|
+
op: z.literal("readFile"),
|
|
302
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory.")
|
|
303
|
+
}).describe("`c.readFile(path)`. Replies with its contents, or null when there is no such file."),
|
|
304
|
+
z.strictObject({
|
|
305
|
+
op: z.literal("writeFile"),
|
|
306
|
+
path: nonEmpty.describe("Workspace-relative, or `~/` for the home directory."),
|
|
307
|
+
contents: z.string().describe("The file's contents, in `encoding`."),
|
|
308
|
+
encoding: z.enum(["utf8", "base64"]).optional().describe("How `contents` is encoded; base64 carries bytes. Default `utf8`.")
|
|
309
|
+
}).describe("`c.writeFile(path, contents)`, creating its directories."),
|
|
310
|
+
z.strictObject({
|
|
311
|
+
op: z.literal("expect"),
|
|
312
|
+
subject: matcherSubjectSchema,
|
|
313
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a matcher name such as toHaveFile, without \"not.\" (set not: true instead)").describe("The matcher's name, e.g. `toHaveFile` or `toBe`, without `not.`."),
|
|
314
|
+
args: z.array(matcherArgSchema).describe("The matcher's arguments."),
|
|
315
|
+
not: z.boolean().describe("Called as `expect(…).not`."),
|
|
316
|
+
soft: z.boolean().describe("Called as `expect.soft`: a failure fails the attempt at its end."),
|
|
317
|
+
label: z.string().optional().describe("`expect`'s second argument."),
|
|
318
|
+
poll: z.strictObject({
|
|
319
|
+
interval: millisSchema.optional().describe("Milliseconds between tries. Default 50."),
|
|
320
|
+
timeout: millisSchema.optional().describe("Milliseconds before it fails with the last error. Default 1000.")
|
|
321
|
+
}).optional().describe("`expect.poll`: reeval re-reads the subject and retries the matcher until it passes, recording only the final outcome. The subject must be one reeval can re-read: the attempt, a command or a file.")
|
|
322
|
+
}).superRefine((call, ctx) => {
|
|
323
|
+
if (call.poll && "value" in call.subject) ctx.addIssue({
|
|
324
|
+
code: "custom",
|
|
325
|
+
path: ["subject"],
|
|
326
|
+
message: "can't be polled: reeval can't re-read a value. Poll the attempt, an exec or a readFile"
|
|
327
|
+
});
|
|
328
|
+
if (call.poll && "exec" in call.subject && call.subject.result) ctx.addIssue({
|
|
329
|
+
code: "custom",
|
|
330
|
+
path: ["subject", "result"],
|
|
331
|
+
message: "can't be polled: drop result so reeval runs the command on each try"
|
|
332
|
+
});
|
|
333
|
+
if (call.poll && call.args.some(hasAnsweredPattern)) ctx.addIssue({
|
|
334
|
+
code: "custom",
|
|
335
|
+
path: ["args"],
|
|
336
|
+
message: "can't be polled with a { matchedBy } pattern, whose answers can't follow the subject from try to try: poll with a string, or check once"
|
|
337
|
+
});
|
|
338
|
+
}).describe("A matcher call: `expect(subject, label).not?.matcher(...args)`. Replies with its record."),
|
|
339
|
+
z.strictObject({
|
|
340
|
+
op: z.literal("record"),
|
|
341
|
+
matcher: z.string().regex(/^[a-z][A-Za-z]*$/, "must be a name such as assert").describe("What made the check in the eval's language, e.g. `assert`."),
|
|
342
|
+
subject: z.string().optional().describe("What it checked, as the eval wrote it (`result.ok`): names the check when it has no label."),
|
|
343
|
+
label: z.string().optional().describe("The check's own name, the same whether it passes or fails."),
|
|
344
|
+
status: z.enum(["pass", "fail"]),
|
|
345
|
+
message: z.string().optional().describe("Why it failed."),
|
|
346
|
+
soft: z.boolean().optional().describe("A failure fails the attempt at its end, as `expect.soft`. Default false.")
|
|
347
|
+
}).describe("A check the eval's own language made, such as Python's `assert`: recorded as a matcher call is, so it's named, counted and compared the same. Replies with its record."),
|
|
348
|
+
z.strictObject({
|
|
349
|
+
op: z.literal("judge"),
|
|
350
|
+
question: nonEmpty.describe("A yes/no question about the graded turn or value."),
|
|
351
|
+
options: judgeCallDataSchema.optional()
|
|
352
|
+
}).describe("`c.judge(question, options)`. Replies with its verdict, or a judge failure when the provider failed.")
|
|
353
|
+
]).describe("One call an eval's body makes on its attempt. The sandbox and the agent session start on the first call that needs them; there's no call to start them.");
|
|
354
|
+
const attemptErrorSchema = z.looseObject({
|
|
355
|
+
name: z.string().describe("The error's class, e.g. `UsageError` or `AssertionError`."),
|
|
356
|
+
message: z.string()
|
|
357
|
+
}).describe("Why a call failed, in reeval's words.");
|
|
358
|
+
const id = nonEmpty.describe("Pairs a reply with its request.");
|
|
359
|
+
const attemptRequestSchema = z.strictObject({
|
|
360
|
+
id,
|
|
361
|
+
call: attemptCallSchema
|
|
362
|
+
}).describe("A call, numbered so its reply can find it.");
|
|
363
|
+
z.union([
|
|
364
|
+
z.discriminatedUnion("op", [
|
|
365
|
+
z.strictObject({
|
|
366
|
+
id,
|
|
367
|
+
ok: z.literal(true),
|
|
368
|
+
op: z.literal("send"),
|
|
369
|
+
turn: turnDataSchema
|
|
370
|
+
}),
|
|
371
|
+
z.strictObject({
|
|
372
|
+
id,
|
|
373
|
+
ok: z.literal(true),
|
|
374
|
+
op: z.literal("exec"),
|
|
375
|
+
result: z.union([execResultSchema, z.strictObject({ handle: handleId }).describe("A background process, still running.")])
|
|
376
|
+
}),
|
|
377
|
+
z.strictObject({
|
|
378
|
+
id,
|
|
379
|
+
ok: z.literal(true),
|
|
380
|
+
op: z.literal("serve"),
|
|
381
|
+
handle: handleId,
|
|
382
|
+
port: z.number().describe("The container port it was given.")
|
|
383
|
+
}),
|
|
384
|
+
z.strictObject({
|
|
385
|
+
id,
|
|
386
|
+
ok: z.literal(true),
|
|
387
|
+
op: z.literal("kill")
|
|
388
|
+
}),
|
|
389
|
+
z.strictObject({
|
|
390
|
+
id,
|
|
391
|
+
ok: z.literal(true),
|
|
392
|
+
op: z.literal("capture"),
|
|
393
|
+
capture: captureSchema
|
|
394
|
+
}),
|
|
395
|
+
z.strictObject({
|
|
396
|
+
id,
|
|
397
|
+
ok: z.literal(true),
|
|
398
|
+
op: z.literal("readFile"),
|
|
399
|
+
contents: z.string().nullable().describe("Null when there is no such file.")
|
|
400
|
+
}),
|
|
401
|
+
z.strictObject({
|
|
402
|
+
id,
|
|
403
|
+
ok: z.literal(true),
|
|
404
|
+
op: z.literal("writeFile")
|
|
405
|
+
}),
|
|
406
|
+
z.strictObject({
|
|
407
|
+
id,
|
|
408
|
+
ok: z.literal(true),
|
|
409
|
+
op: z.literal("expect"),
|
|
410
|
+
assertion: assertionRecordSchema
|
|
411
|
+
}),
|
|
412
|
+
z.strictObject({
|
|
413
|
+
id,
|
|
414
|
+
ok: z.literal(true),
|
|
415
|
+
op: z.literal("record"),
|
|
416
|
+
assertion: assertionRecordSchema
|
|
417
|
+
}),
|
|
418
|
+
z.strictObject({
|
|
419
|
+
id,
|
|
420
|
+
ok: z.literal(true),
|
|
421
|
+
op: z.literal("judge"),
|
|
422
|
+
verdict: z.union([verdictSchema, judgeFailureSchema])
|
|
423
|
+
})
|
|
424
|
+
]),
|
|
425
|
+
z.strictObject({
|
|
426
|
+
id,
|
|
427
|
+
ok: z.literal(true),
|
|
428
|
+
op: z.literal("expect"),
|
|
429
|
+
untested: z.record(z.string(), z.array(z.string())).describe("The strings the matcher tested that a `{ matchedBy, answers }` argument has no answer for, by its `matchedBy`. Nothing was recorded: answer them and make the call again.")
|
|
430
|
+
}),
|
|
431
|
+
z.strictObject({
|
|
432
|
+
id,
|
|
433
|
+
ok: z.literal(false),
|
|
434
|
+
op: z.enum([
|
|
435
|
+
"send",
|
|
436
|
+
"exec",
|
|
437
|
+
"readFile",
|
|
438
|
+
"writeFile",
|
|
439
|
+
"expect",
|
|
440
|
+
"record",
|
|
441
|
+
"judge",
|
|
442
|
+
"serve",
|
|
443
|
+
"kill",
|
|
444
|
+
"capture"
|
|
445
|
+
]),
|
|
446
|
+
error: attemptErrorSchema
|
|
447
|
+
})
|
|
448
|
+
]).describe("reeval's reply to a call: its result, or the error it threw. A failed matcher is a result (its record says `fail`), not an error, unless the matcher couldn't run.");
|
|
449
|
+
const cellRoleSchema = z.enum(["control", "variant"]);
|
|
450
|
+
const cellSchema = z.strictObject({
|
|
451
|
+
key: z.string(),
|
|
452
|
+
harness: z.string(),
|
|
453
|
+
environment: z.string(),
|
|
454
|
+
sandbox: z.string(),
|
|
455
|
+
role: cellRoleSchema,
|
|
456
|
+
variant: z.string().optional()
|
|
457
|
+
});
|
|
458
|
+
z.strictObject({
|
|
459
|
+
type: z.literal("start"),
|
|
460
|
+
attempt: z.strictObject({
|
|
461
|
+
id: nonEmpty,
|
|
462
|
+
cell: cellSchema,
|
|
463
|
+
n: z.number().int().min(1, "a run number, from 1").describe("The run number, from 1.")
|
|
464
|
+
}).describe("`c.attempt`: which cell of the matrix this attempt is, and its run number."),
|
|
465
|
+
file: nonEmpty.describe("The eval file, relative to the project."),
|
|
466
|
+
body: nonEmpty.describe("The eval's `body`, as its declaration named it.")
|
|
467
|
+
}).describe("reeval to the runner: run this eval's body for this attempt. Nothing has started yet: the sandbox and the agent session start on the body's first call that needs them, so its own code runs first. The runner sends its calls, then `done`.");
|
|
468
|
+
z.strictObject({
|
|
469
|
+
type: z.literal("abort"),
|
|
470
|
+
reason: z.enum(["timeout", "canceled"]).describe("`timeout`: the eval's `timeout` ran out. `canceled`: the run was stopped."),
|
|
471
|
+
message: z.string().describe("reeval's sentence, e.g. `Eval timed out after 600s`.")
|
|
472
|
+
}).describe("reeval to the runner: the attempt was aborted, as `c.signal` aborts. Calls in flight reply with errors; the runner should stop the body and send `done`.");
|
|
473
|
+
z.strictObject({
|
|
474
|
+
type: z.literal("done"),
|
|
475
|
+
error: attemptErrorSchema.optional().describe("What the body threw, if it threw. A failed `expect` (not soft) is one.")
|
|
476
|
+
}).describe("The runner to reeval: the eval's body has finished. The attempt's checks and record follow.");
|
|
477
|
+
const CALL_KEYS = Object.fromEntries(attemptCallSchema.options.map((o) => [o.shape.op.value, Object.keys(o.shape)]));
|
|
478
|
+
const OTHER_KEYS = [
|
|
479
|
+
"id",
|
|
480
|
+
"call",
|
|
481
|
+
"turn",
|
|
482
|
+
"attempt",
|
|
483
|
+
"value",
|
|
484
|
+
"regexp",
|
|
485
|
+
"flags",
|
|
486
|
+
"matchedBy",
|
|
487
|
+
"answers",
|
|
488
|
+
...Object.keys(judgeCallDataSchema.shape),
|
|
489
|
+
...Object.keys(judgeStateSchema.shape),
|
|
490
|
+
...Object.keys(captureSchema.shape),
|
|
491
|
+
...Object.keys(captureCallDataSchema.shape),
|
|
492
|
+
"port",
|
|
493
|
+
"handle",
|
|
494
|
+
"exec",
|
|
495
|
+
"result",
|
|
496
|
+
"readFile",
|
|
497
|
+
"interval",
|
|
498
|
+
"timeout",
|
|
499
|
+
"ms"
|
|
500
|
+
];
|
|
501
|
+
/** What's wrong with an attempt request a runner sent, by the field's path, or undefined. */
|
|
502
|
+
const attemptRequestProblem = (value) => dataProblem(attemptRequestSchema, value, "The attempt request", (object) => typeof object.op === "string" ? CALL_KEYS[object.op] ?? [] : OTHER_KEYS);
|
|
503
|
+
//#endregion
|
|
23
504
|
//#region src/capture/fake-instructions.ts
|
|
24
505
|
const FAKE_INSTRUCTIONS = Symbol.for("reeval.capture.fakeInstructions");
|
|
25
506
|
function isWaitStep(step) {
|
|
@@ -725,7 +1206,6 @@ const DEFAULT_VIEWPORT = {
|
|
|
725
1206
|
width: 1280,
|
|
726
1207
|
height: 720
|
|
727
1208
|
};
|
|
728
|
-
const CAPTURE_NAME = /^[a-z0-9_-]+$/;
|
|
729
1209
|
const CAPTURE_WAITS = ["load", "idle"];
|
|
730
1210
|
/** `'load'` when the call omits `wait`. `'load'`, `'idle'` or `{ ms }`. */
|
|
731
1211
|
function resolveWait(wait) {
|
|
@@ -784,9 +1264,12 @@ function resolveCaptureTarget(target) {
|
|
|
784
1264
|
if (target !== null && typeof target === "object" && typeof target.port === "number" && typeof target.kill === "function") return resolveCaptureTarget(target.port);
|
|
785
1265
|
throw new Error("c.capture() needs a port or a ServeHandle as the first argument");
|
|
786
1266
|
}
|
|
787
|
-
/**
|
|
1267
|
+
/**
|
|
1268
|
+
* Artifact name from a path: `/login` → `login`, `/foo/bar` → `foo-bar`, `/` → `root`. Always
|
|
1269
|
+
* a valid capture name: leading `_` and `-` go, and it's cut at the name's length limit.
|
|
1270
|
+
*/
|
|
788
1271
|
function nameFromPath(path) {
|
|
789
|
-
return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(
|
|
1272
|
+
return (path.split(/[?#]/, 1)[0] ?? "").replaceAll("\\", "/").split("/").filter(Boolean).join("-").toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^[_-]+/, "").slice(0, 100).replace(/-+$/, "") || "root";
|
|
790
1273
|
}
|
|
791
1274
|
function resolveCaptureCall(target, call, cascade) {
|
|
792
1275
|
if (call && Object.hasOwn(call, "port")) throw new Error("c.capture() takes the port as the first argument, not in the options");
|
|
@@ -827,7 +1310,7 @@ function checkCallModel(model) {
|
|
|
827
1310
|
if (model.startsWith("typesafe-ai/")) throw new Error(`c.capture() model "${model}" is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
|
|
828
1311
|
}
|
|
829
1312
|
function checkCaptureName(name) {
|
|
830
|
-
if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name
|
|
1313
|
+
if (!CAPTURE_NAME.test(name)) throw new Error(`c.capture() name ${CAPTURE_NAME_RULE} (got ${JSON.stringify(name)})`);
|
|
831
1314
|
}
|
|
832
1315
|
/** A free host port. Closed before the promise resolves, so the caller can bind it. */
|
|
833
1316
|
function allocatePort() {
|
|
@@ -900,9 +1383,38 @@ function actionFromTool(name, input) {
|
|
|
900
1383
|
y: Number(input.y)
|
|
901
1384
|
};
|
|
902
1385
|
}
|
|
1386
|
+
/**
|
|
1387
|
+
* Gemini models answer a point normalized to 0–1000 across the image they were shown, whatever
|
|
1388
|
+
* the prompt asks, so their journeys are prompted in that scale and their points scaled back.
|
|
1389
|
+
*/
|
|
1390
|
+
const normalizesPoints = (model) => model.slice(model.lastIndexOf("/") + 1).toLowerCase().startsWith("gemini-");
|
|
1391
|
+
const NORMALIZED = 1e3;
|
|
1392
|
+
/** Scales an action's 0–1000 points to CSS pixels on a `page`-sized screenshot. */
|
|
1393
|
+
function scalePoints(action, page) {
|
|
1394
|
+
const x = (n) => Math.round(n * page.width / NORMALIZED);
|
|
1395
|
+
const y = (n) => Math.round(n * page.height / NORMALIZED);
|
|
1396
|
+
switch (action.type) {
|
|
1397
|
+
case "click":
|
|
1398
|
+
case "hover":
|
|
1399
|
+
case "scroll": return {
|
|
1400
|
+
...action,
|
|
1401
|
+
x: x(action.x),
|
|
1402
|
+
y: y(action.y)
|
|
1403
|
+
};
|
|
1404
|
+
case "drag": return {
|
|
1405
|
+
...action,
|
|
1406
|
+
x: x(action.x),
|
|
1407
|
+
y: y(action.y),
|
|
1408
|
+
toX: x(action.toX),
|
|
1409
|
+
toY: y(action.toY)
|
|
1410
|
+
};
|
|
1411
|
+
default: return action;
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
const systemPrompt = (normalized) => `You drive a web page from screenshots. Call one tool. ${normalized ? `Coordinates are normalized to 0-${NORMALIZED} on the full-page image: x and toX across its width, y and toY down its height. A scroll's dy is in CSS pixels.` : "Coordinates are CSS pixels on the full-page image."} Call done when the goal is met. Do not mention CDP.`;
|
|
903
1415
|
const useTools = {
|
|
904
1416
|
click: tool({
|
|
905
|
-
description: "Click at
|
|
1417
|
+
description: "Click at (x, y) on the screenshot.",
|
|
906
1418
|
inputSchema: z.object({
|
|
907
1419
|
x: z.number(),
|
|
908
1420
|
y: z.number()
|
|
@@ -966,6 +1478,7 @@ async function runFakeScript(session, script, goal, remaining) {
|
|
|
966
1478
|
}
|
|
967
1479
|
async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
968
1480
|
const model = await loadModel(modelName);
|
|
1481
|
+
const normalized = normalizesPoints(modelName);
|
|
969
1482
|
let moves = 0;
|
|
970
1483
|
let usage = null;
|
|
971
1484
|
while (moves < 32) {
|
|
@@ -980,7 +1493,7 @@ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
|
980
1493
|
let chosen;
|
|
981
1494
|
const result = await generateText({
|
|
982
1495
|
model,
|
|
983
|
-
system:
|
|
1496
|
+
system: systemPrompt(normalized),
|
|
984
1497
|
messages: [{
|
|
985
1498
|
role: "user",
|
|
986
1499
|
content: [{
|
|
@@ -999,6 +1512,7 @@ async function runModelGoal(session, goal, modelName, remaining, loadModel) {
|
|
|
999
1512
|
usage = addUsage(usage, asUsage(result.usage ?? {}));
|
|
1000
1513
|
const call = result.toolCalls?.[0];
|
|
1001
1514
|
if (call) chosen = actionFromTool(call.toolName, call.input ?? {});
|
|
1515
|
+
if (chosen && normalized) chosen = scalePoints(chosen, page);
|
|
1002
1516
|
if (!chosen) throw new Error(`c.capture() instruction ${JSON.stringify(goal)} returned no action`);
|
|
1003
1517
|
debugLog("check", `capture use ${chosen.type} on ${JSON.stringify(goal)}`);
|
|
1004
1518
|
debugTrace("check", "capture-use", {
|
|
@@ -1065,7 +1579,7 @@ function captureUseSpend(result) {
|
|
|
1065
1579
|
}
|
|
1066
1580
|
//#endregion
|
|
1067
1581
|
//#region src/diff.ts
|
|
1068
|
-
const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0";
|
|
1582
|
+
const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null -c core.logAllRefUpdates=false -c gc.auto=0 -c core.quotePath=false";
|
|
1069
1583
|
const snapshotEnv = (sandbox) => `export GIT_DIR=${shellQuote(`${sandbox.stateDir}/workspace.git`)} GIT_WORK_TREE=${shellQuote(sandbox.workdir)} GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null`;
|
|
1070
1584
|
/** Points HEAD at a commit of `$tree` with no parent, then drops every object nothing reaches. */
|
|
1071
1585
|
const RESET_HEAD = `c=$(${GIT} commit-tree -m base "$tree") && ${GIT} update-ref HEAD "$c" && ${GIT} prune --expire=now`;
|
|
@@ -1093,20 +1607,22 @@ async function acceptIntoSnapshot(sandbox, path, signal) {
|
|
|
1093
1607
|
async function diffSinceSnapshot(sandbox, signal) {
|
|
1094
1608
|
const env = `${snapshotEnv(sandbox)} && ${GIT} add -A && `;
|
|
1095
1609
|
const patch = await sandbox.execOwn(`${env}${GIT} diff --cached --binary HEAD`, { signal });
|
|
1096
|
-
const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
|
|
1610
|
+
const names = await sandbox.execOwn(`${env}${GIT} diff --cached --name-status --no-renames -z HEAD`, { signal });
|
|
1097
1611
|
if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
|
|
1098
1612
|
const status = {
|
|
1099
1613
|
A: "added",
|
|
1100
1614
|
M: "modified",
|
|
1101
1615
|
D: "deleted"
|
|
1102
1616
|
};
|
|
1103
|
-
const
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1617
|
+
const fields = names.stdout.split("\0");
|
|
1618
|
+
const files = [];
|
|
1619
|
+
for (let i = 0; i + 1 < fields.length; i += 2) {
|
|
1620
|
+
const code = fields[i];
|
|
1621
|
+
files.push({
|
|
1622
|
+
path: fields[i + 1],
|
|
1107
1623
|
status: status[code[0]] ?? "modified"
|
|
1108
|
-
};
|
|
1109
|
-
}
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1110
1626
|
return {
|
|
1111
1627
|
patch: patch.stdout,
|
|
1112
1628
|
files
|
|
@@ -1125,45 +1641,73 @@ const asCheck = (a, fn) => a.phases ? a.phases.checks(fn) : fn();
|
|
|
1125
1641
|
/** The attempt's sandbox, started (with its session) by the first call that needs it. */
|
|
1126
1642
|
const sandboxFor = async (a, who) => (await a.start(who))[0];
|
|
1127
1643
|
/**
|
|
1128
|
-
* `c.exec` runs with the checks' variables and secrets, each run recorded and
|
|
1129
|
-
*
|
|
1644
|
+
* `c.exec` runs with the checks' variables and secrets, each run recorded and timed, and runs
|
|
1645
|
+
* again on a non-zero exit while it has `retries` left; `c.readFile` reads the sandbox as is;
|
|
1646
|
+
* `c.writeFile` places a file the diff leaves out.
|
|
1130
1647
|
*/
|
|
1131
1648
|
function createChecks(a) {
|
|
1132
1649
|
const checkEnv = () => a.env?.checkVars ?? {};
|
|
1133
|
-
const execForeground = async (command) => {
|
|
1650
|
+
const execForeground = async (command, retries, interval) => {
|
|
1134
1651
|
const box = await sandboxFor(a, "c.exec()");
|
|
1135
1652
|
const start = performance.now();
|
|
1653
|
+
const tries = [];
|
|
1136
1654
|
a.report({
|
|
1137
1655
|
kind: "check-start",
|
|
1138
1656
|
command
|
|
1139
1657
|
});
|
|
1140
|
-
const r = await a.timed("checksMs",
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1658
|
+
const r = await a.timed("checksMs", async () => {
|
|
1659
|
+
for (;;) {
|
|
1660
|
+
const began = performance.now();
|
|
1661
|
+
const r = await asCheck(a, () => box.exec(command, {
|
|
1662
|
+
signal: a.signal,
|
|
1663
|
+
env: checkEnv()
|
|
1664
|
+
}));
|
|
1665
|
+
tries.push({
|
|
1666
|
+
...r,
|
|
1667
|
+
durationMs: Math.round(performance.now() - began)
|
|
1668
|
+
});
|
|
1669
|
+
if (r.exitCode === 0 || tries.length > retries) return r;
|
|
1670
|
+
debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode} on try ${tries.length} of ${retries + 1}, again in ${formatDuration(interval)}`);
|
|
1671
|
+
await setTimeout$1(interval, void 0, { signal: a.signal });
|
|
1672
|
+
}
|
|
1673
|
+
});
|
|
1144
1674
|
const durationMs = Math.round(performance.now() - start);
|
|
1675
|
+
const retried = retries > 0 ? {
|
|
1676
|
+
retries,
|
|
1677
|
+
tries
|
|
1678
|
+
} : {};
|
|
1145
1679
|
a.record.checks.push({
|
|
1146
1680
|
command,
|
|
1147
1681
|
...r,
|
|
1148
|
-
durationMs
|
|
1682
|
+
durationMs,
|
|
1683
|
+
...retried
|
|
1149
1684
|
});
|
|
1150
|
-
|
|
1685
|
+
const of = retries > 0 ? ` on try ${tries.length} of ${retries + 1}` : "";
|
|
1686
|
+
debugLog("check", `$ ${clip(redactSecrets(command), 160)} → exit ${r.exitCode}${of} · ${formatDuration(durationMs)}`);
|
|
1151
1687
|
debugTrace("check", "output", {
|
|
1152
1688
|
command,
|
|
1153
1689
|
...r,
|
|
1154
|
-
durationMs
|
|
1155
|
-
|
|
1690
|
+
durationMs,
|
|
1691
|
+
...retried
|
|
1692
|
+
}, `check output: ${clip(redactSecrets(command), 80)}`);
|
|
1156
1693
|
a.report({
|
|
1157
1694
|
kind: "check-end",
|
|
1158
1695
|
command,
|
|
1159
1696
|
exitCode: r.exitCode,
|
|
1160
|
-
ms: durationMs
|
|
1697
|
+
ms: durationMs,
|
|
1698
|
+
...retries > 0 ? {
|
|
1699
|
+
try: tries.length,
|
|
1700
|
+
tries: retries + 1
|
|
1701
|
+
} : {}
|
|
1161
1702
|
});
|
|
1162
1703
|
return r;
|
|
1163
1704
|
};
|
|
1164
1705
|
function exec(command, options) {
|
|
1165
|
-
|
|
1166
|
-
|
|
1706
|
+
const problem = options === void 0 ? void 0 : execOptionsProblem(options);
|
|
1707
|
+
if (problem) return Promise.reject(new UsageError(problem));
|
|
1708
|
+
if (options?.background) return import("./capture-DSK8O5bA.js").then(({ startBackground }) => startBackground(a, command, "c.exec({ background: true })"));
|
|
1709
|
+
const run = execForeground(command, options?.retries ?? 0, options?.interval ?? 1e3);
|
|
1710
|
+
return Object.defineProperty(run, SUBJECT, { value: command });
|
|
1167
1711
|
}
|
|
1168
1712
|
const readFile = async (path) => (await sandboxFor(a, "c.readFile()")).readFile(path);
|
|
1169
1713
|
const writeFile = async (path, contents) => {
|
|
@@ -1188,25 +1732,26 @@ function createChecks(a) {
|
|
|
1188
1732
|
const BG_LOG = "/tmp/reeval-bg.log";
|
|
1189
1733
|
const BG_TAG = "REEVAL_BACKGROUND";
|
|
1190
1734
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
1191
|
-
|
|
1735
|
+
const languageModel = (model) => typeof model === "string" && !isEvaluationModel(model) ? model : void 0;
|
|
1736
|
+
/**
|
|
1737
|
+
* `analysis.model`, else the attempt's judge (a profile's, group's or eval's over the config's),
|
|
1738
|
+
* whichever is first a language model.
|
|
1739
|
+
*/
|
|
1740
|
+
function defaultCaptureModel(a) {
|
|
1192
1741
|
try {
|
|
1193
|
-
|
|
1194
|
-
const analysis = config.analysis.model;
|
|
1195
|
-
if (typeof analysis === "string" && !isEvaluationModel(analysis)) return analysis;
|
|
1196
|
-
const judge = config.judge.model;
|
|
1197
|
-
if (typeof judge === "string" && !isEvaluationModel(judge)) return judge;
|
|
1742
|
+
return languageModel(getRunContext().config.analysis.model) ?? languageModel(a.spec.judge.model);
|
|
1198
1743
|
} catch {
|
|
1199
1744
|
return;
|
|
1200
1745
|
}
|
|
1201
1746
|
}
|
|
1202
|
-
function resolveUse(resolved, fakeSandbox) {
|
|
1747
|
+
function resolveUse(a, resolved, fakeSandbox) {
|
|
1203
1748
|
if (resolved.fake) return { fake: resolved.fake };
|
|
1204
1749
|
if (resolved.model) return { model: resolved.model };
|
|
1205
1750
|
if (!resolved.instructions.length) return void 0;
|
|
1206
|
-
const fallback = defaultCaptureModel();
|
|
1751
|
+
const fallback = defaultCaptureModel(a);
|
|
1207
1752
|
if (fallback) return { model: fallback };
|
|
1208
1753
|
if (fakeSandbox) return void 0;
|
|
1209
|
-
throw new Error("c.capture() instructions need a vision model. Set capture.model, pass model on the call, or analysis.model to a language model that accepts images. Tests pass fakeInstructions from reeval/testing.");
|
|
1754
|
+
throw new Error("c.capture() instructions need a vision model. Set capture.model, pass model on the call, or set analysis.model or the judge model to a language model that accepts images. Tests pass fakeInstructions from reeval/testing.");
|
|
1210
1755
|
}
|
|
1211
1756
|
/**
|
|
1212
1757
|
* Stops `pid` and its descendants under Git for Windows' MSYS. Native children (node.exe) are
|
|
@@ -1282,7 +1827,7 @@ async function capturePage(a, target, call) {
|
|
|
1282
1827
|
throw new Error(`nothing listening on port ${resolved.port}`);
|
|
1283
1828
|
}
|
|
1284
1829
|
const fake = a.spec.sandbox.kind === "fake";
|
|
1285
|
-
const use = resolveUse(resolved, fake);
|
|
1830
|
+
const use = resolveUse(a, resolved, fake);
|
|
1286
1831
|
a.captureNames.add(resolved.name);
|
|
1287
1832
|
const started = Date.now();
|
|
1288
1833
|
const remaining = () => Math.max(0, resolved.timeoutMs - (Date.now() - started));
|
|
@@ -1346,10 +1891,12 @@ async function shot(a, url, resolved, remaining, onMoves) {
|
|
|
1346
1891
|
use: resolved.use,
|
|
1347
1892
|
remaining,
|
|
1348
1893
|
loadModel: async (name) => {
|
|
1349
|
-
const
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1894
|
+
const { config, profile } = getRunContext();
|
|
1895
|
+
const scope = gatewayScope(config, profile);
|
|
1896
|
+
const gateway = resolveModelGateway(name, void 0, scope, "capture model");
|
|
1897
|
+
return hostModel(name, gateway, "capture model", {
|
|
1898
|
+
fallbacks: openRouterFallbacks(scope.gateways),
|
|
1899
|
+
...gateway === "vertex" ? { vertex: vertexSettingsIn(scope.gateways) } : {}
|
|
1353
1900
|
});
|
|
1354
1901
|
}
|
|
1355
1902
|
});
|
|
@@ -1374,6 +1921,6 @@ async function writeCaptureSidecars(stagingDir, recordFile, blobs) {
|
|
|
1374
1921
|
for (const [name, png] of blobs) await writeFile(join(dir, `${name}.png`), png);
|
|
1375
1922
|
}
|
|
1376
1923
|
//#endregion
|
|
1377
|
-
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
|
|
1924
|
+
export { writeCaptureSidecars as a, sandboxFor as c, workspaceFiles as d, fakeInstructions as f, startBackground as i, diffSinceSnapshot as l, msysTreeKill as n, asCheck as o, attemptRequestProblem as p, serveCommand as r, createChecks as s, capturePage as t, snapshot as u };
|
|
1378
1925
|
|
|
1379
|
-
//# sourceMappingURL=capture-
|
|
1926
|
+
//# sourceMappingURL=capture-BM_3100g.js.map
|