reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,29 +1,31 @@
|
|
|
1
1
|
import { t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
|
+
import { r as withLease } from "./client-Ct6s6OnF.js";
|
|
2
3
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
|
|
3
|
-
import { n as dataProblem, p as
|
|
4
|
-
import { A as
|
|
5
|
-
import {
|
|
4
|
+
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-BOFgzFDg.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-D-CP6mMd.js";
|
|
6
7
|
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import "./
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
8
|
+
import { $ as isDirect, A as harnessId, At as badJudgeArtifacts, B as sandboxSetBy, D as checkOptions, Dt as judgeProblem, G as resolveStateOptions, H as unbrokered, J as allowedHosts, L as refuseIllegalCompare, M as matchesTags, Mt as filePartsOf, N as plainHeaderError, Nt as resolveJudgeArtifacts, O as evalPlan, P as plainHeaders, R as sandboxId, S as resolveGateway, St as writePluginFiles, T as cascade, U as unbrokeredError, V as secretVarsIn, W as mergeJudge, Z as egressLabel, _ as fellBackToDirect, _t as expectedBlockedFor, b as resolveModelGateway, bt as refusedMcp, et as unenforceable, ft as agentMcp, ht as claudeMcpServers, jt as decideState, k as expand, kt as artifactRefsOf, mt as brokeringFor, nt as createFakeSandbox, v as gatewayScope, w as brokeredIn, xt as refusedPlugins, yt as layoutOf, z as sandboxProvider } from "./config-k6m7XbAz.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-CKdgEXN7.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
12
|
+
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
|
|
13
|
+
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
|
+
import { ft as shellQuote, t as MCP_TIMER } from "./src-D91Hd6hT.js";
|
|
13
15
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-jI0OWxlz.js";
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-d56Gg3J5.js";
|
|
17
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Ba2QerOK.js";
|
|
17
18
|
import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
|
|
18
|
-
import {
|
|
19
|
-
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-
|
|
19
|
+
import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
|
|
20
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BM_3100g.js";
|
|
20
21
|
import { t as contextTaker } from "./context-Dgn48BqT.js";
|
|
21
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
22
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-DWSJpKHX.js";
|
|
22
23
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
23
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
24
|
+
import { i as preparedWorkspace } from "./workspace-BRxjfa8N.js";
|
|
25
|
+
import { t as patchPath } from "./patch-CquGAGX5.js";
|
|
26
|
+
import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
|
|
27
|
+
import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
|
|
28
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-Dje3rERU.js";
|
|
27
29
|
import { dirname, join } from "node:path";
|
|
28
30
|
import { parse, stringify } from "yaml";
|
|
29
31
|
import { z } from "zod";
|
|
@@ -32,7 +34,11 @@ import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
|
32
34
|
//#region src/attempt/assertions.ts
|
|
33
35
|
const MESSAGE_LIMIT = 1024;
|
|
34
36
|
const LABEL_LIMIT = 60;
|
|
35
|
-
|
|
37
|
+
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
38
|
+
const clipped = (text, limit) => {
|
|
39
|
+
const s = redactSecrets(text);
|
|
40
|
+
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
41
|
+
};
|
|
36
42
|
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
37
43
|
const shown = (value) => {
|
|
38
44
|
if (typeof value === "string") return value;
|
|
@@ -46,17 +52,26 @@ const shown = (value) => {
|
|
|
46
52
|
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
47
53
|
/**
|
|
48
54
|
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
49
|
-
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3
|
|
55
|
+
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
56
|
+
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
57
|
+
* `build`s is `build #2 #2`).
|
|
50
58
|
*/
|
|
51
59
|
function assertionRecords(raw) {
|
|
52
60
|
const seen = /* @__PURE__ */ new Map();
|
|
61
|
+
const used = /* @__PURE__ */ new Set();
|
|
53
62
|
return raw.map((r) => {
|
|
54
63
|
const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
55
64
|
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
56
|
-
|
|
65
|
+
let n = seen.get(base) ?? 0;
|
|
66
|
+
let label;
|
|
67
|
+
do {
|
|
68
|
+
n += 1;
|
|
69
|
+
label = n > 1 ? `${base} #${n}` : base;
|
|
70
|
+
} while (used.has(label));
|
|
57
71
|
seen.set(base, n);
|
|
72
|
+
used.add(label);
|
|
58
73
|
return {
|
|
59
|
-
label
|
|
74
|
+
label,
|
|
60
75
|
labelled: r.label !== void 0,
|
|
61
76
|
matcher: r.matcher,
|
|
62
77
|
status: r.status,
|
|
@@ -66,11 +81,22 @@ function assertionRecords(raw) {
|
|
|
66
81
|
};
|
|
67
82
|
});
|
|
68
83
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
84
|
+
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
85
|
+
/**
|
|
86
|
+
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
87
|
+
* that didn't pass, an `expect.soft` as what it threw, a hard check the eval caught with
|
|
88
|
+
* `CAUGHT_HINT` after its message.
|
|
89
|
+
*/
|
|
90
|
+
function checkFailure(raw) {
|
|
91
|
+
const failed = raw.find((r) => r.status !== "pass");
|
|
72
92
|
if (!failed) return void 0;
|
|
73
|
-
|
|
93
|
+
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
94
|
+
if (failed.soft) return error;
|
|
95
|
+
const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
|
|
96
|
+
caught.name = error.name;
|
|
97
|
+
caught.message = `${error.message} ${CAUGHT_HINT}`;
|
|
98
|
+
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
99
|
+
return caught;
|
|
74
100
|
}
|
|
75
101
|
//#endregion
|
|
76
102
|
//#region src/attempt/debug.ts
|
|
@@ -86,7 +112,7 @@ function tracePlan(debug, spec, ctx) {
|
|
|
86
112
|
const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
|
|
87
113
|
const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
|
|
88
114
|
debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
|
|
89
|
-
debug.log("plan", `egress ${egressLabel(spec.egress) || "
|
|
115
|
+
debug.log("plan", `egress ${egressLabel(spec.egress) || "proxy"} ${from("egress")}`);
|
|
90
116
|
const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
|
|
91
117
|
debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
|
|
92
118
|
const judge = spec.judge;
|
|
@@ -122,7 +148,7 @@ function traceEnvironment(debug, spec, env) {
|
|
|
122
148
|
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
123
149
|
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
124
150
|
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
125
|
-
const brokering = brokeringFor(spec.harness);
|
|
151
|
+
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
126
152
|
if (brokering) {
|
|
127
153
|
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
128
154
|
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
@@ -408,10 +434,10 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
408
434
|
}
|
|
409
435
|
//#endregion
|
|
410
436
|
//#region src/harness/index.ts
|
|
411
|
-
async function createHarness(descriptor) {
|
|
437
|
+
async function createHarness(descriptor, vertex) {
|
|
412
438
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
413
|
-
const { createAgentHarness } = await import("./agent-
|
|
414
|
-
return createAgentHarness(descriptor);
|
|
439
|
+
const { createAgentHarness } = await import("./agent-Zgn-kiqq.js");
|
|
440
|
+
return createAgentHarness(descriptor, vertex);
|
|
415
441
|
}
|
|
416
442
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
417
443
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
@@ -430,9 +456,9 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
430
456
|
});
|
|
431
457
|
a.env = env;
|
|
432
458
|
record.environment = env.record;
|
|
433
|
-
const { secrets, brokered } = env.record;
|
|
459
|
+
const { vars, secrets, brokered } = env.record;
|
|
434
460
|
record.env = {
|
|
435
|
-
vars
|
|
461
|
+
vars,
|
|
436
462
|
secrets,
|
|
437
463
|
brokered
|
|
438
464
|
};
|
|
@@ -440,8 +466,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
440
466
|
if (debug) traceEnvironment(debug, spec, env);
|
|
441
467
|
record.image = null;
|
|
442
468
|
if (spec.sandbox.kind === "docker") {
|
|
443
|
-
const { preparedFor } = await import("./preflight-
|
|
444
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
469
|
+
const { preparedFor } = await import("./preflight-DTiVv0gY.js");
|
|
470
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DkOupEBl.js");
|
|
445
471
|
signal.throwIfAborted();
|
|
446
472
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
447
473
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -474,7 +500,7 @@ const CHARS_PER_TOKEN = 4;
|
|
|
474
500
|
const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
|
|
475
501
|
const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
|
|
476
502
|
const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
|
|
477
|
-
name:
|
|
503
|
+
name: patchPath(part) ?? "",
|
|
478
504
|
chars: part.length
|
|
479
505
|
})).filter((item) => item.name);
|
|
480
506
|
/** The state's parts by size, each with its biggest files where it has any. */
|
|
@@ -617,13 +643,21 @@ function checksShown(which, ran) {
|
|
|
617
643
|
}));
|
|
618
644
|
}
|
|
619
645
|
/**
|
|
646
|
+
* Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
|
|
647
|
+
* what they record stays in the order they were made.
|
|
648
|
+
*/
|
|
649
|
+
function placeFor(calls, order) {
|
|
650
|
+
const after = calls.findIndex((c) => c > order);
|
|
651
|
+
return after < 0 ? calls.length : after;
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
620
654
|
* Grades a turn (the last by default) or a value against a question, under
|
|
621
655
|
* the judge provider's concurrency lease. Every call's input is recorded, including calls that
|
|
622
656
|
* fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
|
|
623
657
|
* checks after it still run.
|
|
624
658
|
*/
|
|
625
659
|
function judgeCaller(a, readFile) {
|
|
626
|
-
const ask = async (question, { on, ...call } = {}) => {
|
|
660
|
+
const ask = async (order, question, { on, ...call } = {}) => {
|
|
627
661
|
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
628
662
|
const problem = judgeProblem(call, { at: "call" });
|
|
629
663
|
if (problem) throw new Error(problem);
|
|
@@ -634,8 +668,9 @@ function judgeCaller(a, readFile) {
|
|
|
634
668
|
const options = mergeJudge(a.spec.judge, call);
|
|
635
669
|
const { model } = options;
|
|
636
670
|
if (model === void 0) {
|
|
637
|
-
|
|
638
|
-
|
|
671
|
+
const message = noJudgeModel(getRunContext().config.file);
|
|
672
|
+
a.judgeThrown.add(message);
|
|
673
|
+
throw new Error(message);
|
|
639
674
|
}
|
|
640
675
|
const wanted = resolveStateOptions(options.state);
|
|
641
676
|
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
|
|
@@ -651,7 +686,7 @@ function judgeCaller(a, readFile) {
|
|
|
651
686
|
listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
|
|
652
687
|
});
|
|
653
688
|
const name = judgeModelName(model);
|
|
654
|
-
const state = stateOf(draft);
|
|
689
|
+
const state = redactValue(stateOf(draft));
|
|
655
690
|
const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
|
|
656
691
|
const refs = artifactRefsOf(resolved);
|
|
657
692
|
const decide = decideState(state, filePartsOf(resolved));
|
|
@@ -663,7 +698,9 @@ function judgeCaller(a, readFile) {
|
|
|
663
698
|
...refs.length ? { artifacts: refs } : {},
|
|
664
699
|
...dropped.length ? { dropped } : {}
|
|
665
700
|
};
|
|
666
|
-
a.
|
|
701
|
+
const place = placeFor(a.judgeInputCalls, order);
|
|
702
|
+
a.judgeInputCalls.splice(place, 0, order);
|
|
703
|
+
a.record.judgeInputs?.splice(place, 0, input);
|
|
667
704
|
const start = performance.now();
|
|
668
705
|
const ms = () => Math.round(performance.now() - start);
|
|
669
706
|
const atLeast = options.threshold ?? .8;
|
|
@@ -671,12 +708,14 @@ function judgeCaller(a, readFile) {
|
|
|
671
708
|
kind: "judge-start",
|
|
672
709
|
question
|
|
673
710
|
});
|
|
674
|
-
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
|
|
711
|
+
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
|
|
675
712
|
if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
|
|
676
713
|
debugTrace("judge", "prompt", input, "judge prompt");
|
|
677
714
|
const failed = (message, retries) => {
|
|
678
|
-
a.
|
|
679
|
-
|
|
715
|
+
a.judgeErrors.push({
|
|
716
|
+
call: order,
|
|
717
|
+
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
|
|
718
|
+
});
|
|
680
719
|
input.error = message;
|
|
681
720
|
if (retries?.length) input.retries = retries;
|
|
682
721
|
debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
|
|
@@ -704,8 +743,17 @@ function judgeCaller(a, readFile) {
|
|
|
704
743
|
};
|
|
705
744
|
const provider = name.split("/")[0];
|
|
706
745
|
try {
|
|
746
|
+
const { config, profile } = getRunContext();
|
|
747
|
+
const scope = gatewayScope(config, profile);
|
|
748
|
+
const gateway = resolveModelGateway(model, judgeOptions.gateway, scope, "judge model");
|
|
749
|
+
const { gateway: _pin, ...unpinned } = judgeOptions;
|
|
750
|
+
const entry = gateway && scope.gateways.find((g) => g.kind === gateway);
|
|
751
|
+
const routed = gateway ? {
|
|
752
|
+
...unpinned,
|
|
753
|
+
gateway: entry || gateway
|
|
754
|
+
} : unpinned;
|
|
707
755
|
const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
708
|
-
...
|
|
756
|
+
...routed,
|
|
709
757
|
model,
|
|
710
758
|
threshold: atLeast
|
|
711
759
|
}, a.signal), a.signal));
|
|
@@ -713,10 +761,12 @@ function judgeCaller(a, readFile) {
|
|
|
713
761
|
...verdict,
|
|
714
762
|
artifacts: refs
|
|
715
763
|
} : verdict;
|
|
716
|
-
a.record
|
|
764
|
+
const { judgements: judgments } = a.record;
|
|
765
|
+
judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
|
|
717
766
|
...recorded,
|
|
718
|
-
retries
|
|
719
|
-
|
|
767
|
+
...retries.length ? { retries } : {},
|
|
768
|
+
call: order
|
|
769
|
+
});
|
|
720
770
|
const { score } = verdict;
|
|
721
771
|
debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
|
|
722
772
|
debugTrace("judge", "response", {
|
|
@@ -733,21 +783,19 @@ function judgeCaller(a, readFile) {
|
|
|
733
783
|
});
|
|
734
784
|
return recorded;
|
|
735
785
|
} catch (error) {
|
|
736
|
-
if (a.signal.aborted)
|
|
737
|
-
a.judgeErrored = true;
|
|
738
|
-
throw error;
|
|
739
|
-
}
|
|
786
|
+
if (a.signal.aborted) throw error;
|
|
740
787
|
const { retries } = error;
|
|
741
788
|
const message = error instanceof Error ? error.message : String(error);
|
|
742
789
|
return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
|
|
743
790
|
}
|
|
744
791
|
};
|
|
745
792
|
return async (question, { on, ...options } = {}) => {
|
|
793
|
+
const order = a.judgeCalls++;
|
|
746
794
|
const call = on === void 0 ? options : {
|
|
747
795
|
...options,
|
|
748
796
|
on: judgeOn(a.turns, on)
|
|
749
797
|
};
|
|
750
|
-
return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
|
|
798
|
+
return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
|
|
751
799
|
};
|
|
752
800
|
}
|
|
753
801
|
//#endregion
|
|
@@ -757,7 +805,7 @@ async function createSandbox(descriptor, options) {
|
|
|
757
805
|
switch (descriptor.kind) {
|
|
758
806
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
759
807
|
case "docker": {
|
|
760
|
-
const { createDockerSandbox } = await import("./docker-
|
|
808
|
+
const { createDockerSandbox } = await import("./docker-YFj4H1Cr.js");
|
|
761
809
|
return await createDockerSandbox(descriptor, options);
|
|
762
810
|
}
|
|
763
811
|
}
|
|
@@ -776,7 +824,7 @@ function classify(error, signal) {
|
|
|
776
824
|
message: "The eval timed out"
|
|
777
825
|
} : {
|
|
778
826
|
kind: "aborted",
|
|
779
|
-
message: "The run was
|
|
827
|
+
message: "The run was canceled"
|
|
780
828
|
};
|
|
781
829
|
if (error instanceof InfraError) return {
|
|
782
830
|
kind: "infra",
|
|
@@ -787,17 +835,39 @@ function classify(error, signal) {
|
|
|
787
835
|
message
|
|
788
836
|
};
|
|
789
837
|
}
|
|
838
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
839
|
+
const judged = (r) => /(^|\.)toPass$/.test(r.matcher);
|
|
840
|
+
/**
|
|
841
|
+
* Whether `error` is a judge's: one a judge threw, or what a `toPass()` or `.not.toPass()` call
|
|
842
|
+
* failed with. A failing verdict the eval didn't assert, or asserted with `.not.toPass()`, isn't.
|
|
843
|
+
* By message too, since another language's runner hands back its own copy of the error.
|
|
844
|
+
*/
|
|
845
|
+
function isJudges(a, error, raw) {
|
|
846
|
+
const message = messageOf(error);
|
|
847
|
+
if (a.judgeThrown.has(message)) return true;
|
|
848
|
+
return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
852
|
+
* (an `expect.soft`, or a hard check the eval caught), whichever came first, so the attempt fails
|
|
853
|
+
* as that check.
|
|
854
|
+
*/
|
|
855
|
+
function attemptError(a, error, raw) {
|
|
856
|
+
if (!isJudges(a, error, raw)) return error;
|
|
857
|
+
const check = raw.find((r) => r.status !== "pass" && !judged(r));
|
|
858
|
+
return check ? checkFailure([check]) : error;
|
|
859
|
+
}
|
|
790
860
|
/**
|
|
791
861
|
* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
|
|
792
|
-
* assertion, which is the judge's when
|
|
862
|
+
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
|
|
793
863
|
*/
|
|
794
|
-
function recordFailure(a, error, timedOut) {
|
|
864
|
+
function recordFailure(a, error, timedOut, raw) {
|
|
795
865
|
const { record } = a;
|
|
796
866
|
record.error = timedOut ? {
|
|
797
867
|
kind: "timeout",
|
|
798
868
|
message: "The eval timed out"
|
|
799
869
|
} : classify(error, a.signal);
|
|
800
|
-
record.error.message = capMiddle(record.error.message);
|
|
870
|
+
record.error.message = capMiddle(redactSecrets(record.error.message));
|
|
801
871
|
if (error instanceof PrepareError && !timedOut) {
|
|
802
872
|
const { class: kind, stage, ref, cause, log, setBy } = error.failure;
|
|
803
873
|
record.error.prepare = {
|
|
@@ -809,8 +879,7 @@ function recordFailure(a, error, timedOut) {
|
|
|
809
879
|
setBy: setBy ?? []
|
|
810
880
|
};
|
|
811
881
|
}
|
|
812
|
-
|
|
813
|
-
if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
|
|
882
|
+
if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
|
|
814
883
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
815
884
|
}
|
|
816
885
|
/**
|
|
@@ -837,11 +906,11 @@ async function recordOom(a, sandbox) {
|
|
|
837
906
|
/**
|
|
838
907
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
839
908
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
840
|
-
*
|
|
909
|
+
* summarized in one `-v` line.
|
|
841
910
|
*/
|
|
842
911
|
function recordEgress(a, sandbox) {
|
|
843
912
|
const traffic = sandbox?.traffic?.();
|
|
844
|
-
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
|
|
913
|
+
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
|
|
845
914
|
const refused = traffic?.blocked ?? [];
|
|
846
915
|
const isExpected = (h) => !!expected?.hosts.includes(h.host);
|
|
847
916
|
const expectedBlocked = refused.filter(isExpected);
|
|
@@ -849,6 +918,7 @@ function recordEgress(a, sandbox) {
|
|
|
849
918
|
policy: a.spec.egress.policy,
|
|
850
919
|
allowed: allowedHosts(a.spec.egress),
|
|
851
920
|
recorded: !!traffic,
|
|
921
|
+
...traffic && isDirect(a.spec.egress.policy) ? { partial: true } : {},
|
|
852
922
|
contacted: traffic?.contacted ?? [],
|
|
853
923
|
blocked: refused.filter((h) => !isExpected(h)),
|
|
854
924
|
expectedBlocked
|
|
@@ -858,6 +928,26 @@ function recordEgress(a, sandbox) {
|
|
|
858
928
|
debugLog("gateway", `expected blocks, not logged per request: ${hosts} · ${expected.why}`);
|
|
859
929
|
}
|
|
860
930
|
a.record.servedModels = sandbox?.servedModels?.() ?? null;
|
|
931
|
+
const asked = sandbox?.askedModels?.() ?? [];
|
|
932
|
+
if (asked.length) a.record.askedModels = asked;
|
|
933
|
+
const upstreams = sandbox?.upstreams?.() ?? [];
|
|
934
|
+
if (upstreams.length) a.record.upstreams = upstreams;
|
|
935
|
+
const cost = sandbox?.reportedCostUsd?.();
|
|
936
|
+
if (cost !== void 0) a.record.reportedCostUsd = cost;
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* The record as its staging file holds it: redacted, with a secret split across a turn's raw
|
|
940
|
+
* stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
|
|
941
|
+
*/
|
|
942
|
+
function stagedRecordJson(record) {
|
|
943
|
+
const turns = record.turns.map((turn) => turn.raw ? {
|
|
944
|
+
...turn,
|
|
945
|
+
raw: redactSplitSecrets(turn.raw.map((part) => JSON.stringify(part))).map((line) => JSON.parse(line))
|
|
946
|
+
} : turn);
|
|
947
|
+
return redactJson({
|
|
948
|
+
...record,
|
|
949
|
+
turns
|
|
950
|
+
}, 2);
|
|
861
951
|
}
|
|
862
952
|
/** Writes the attempt's record, redacted, and returns what the runner reports. */
|
|
863
953
|
async function writeRecord(a, stagingDir, id) {
|
|
@@ -866,7 +956,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
866
956
|
await mkdir(join(stagingDir, "evals"), { recursive: true });
|
|
867
957
|
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
|
|
868
958
|
const tmp = `${recordPath}.${process.pid}.tmp`;
|
|
869
|
-
await writeFile(tmp,
|
|
959
|
+
await writeFile(tmp, stagedRecordJson(record));
|
|
870
960
|
await rename(tmp, recordPath);
|
|
871
961
|
return {
|
|
872
962
|
outcome: record.outcome,
|
|
@@ -888,7 +978,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
888
978
|
function turnErrors(transcripts) {
|
|
889
979
|
return transcripts.flatMap((events, i) => {
|
|
890
980
|
const found = events.find((e) => e.type === "error");
|
|
891
|
-
return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(
|
|
981
|
+
return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
|
|
892
982
|
});
|
|
893
983
|
}
|
|
894
984
|
//#endregion
|
|
@@ -920,7 +1010,7 @@ function hookHandlers(plugin, text, hooks) {
|
|
|
920
1010
|
const what = `the ${plugin.name} plugin's hooks/hooks.json`;
|
|
921
1011
|
let raw;
|
|
922
1012
|
try {
|
|
923
|
-
raw = JSON.parse(text);
|
|
1013
|
+
raw = JSON.parse(withoutBom(text));
|
|
924
1014
|
} catch {
|
|
925
1015
|
throw new InfraError(`${what} isn't valid JSON`);
|
|
926
1016
|
}
|
|
@@ -981,7 +1071,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
981
1071
|
if (!hooks) {
|
|
982
1072
|
let own = {};
|
|
983
1073
|
try {
|
|
984
|
-
own = hooksFile === null ? {} : JSON.parse(hooksFile);
|
|
1074
|
+
own = hooksFile === null ? {} : JSON.parse(withoutBom(hooksFile));
|
|
985
1075
|
} catch {
|
|
986
1076
|
throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
|
|
987
1077
|
}
|
|
@@ -1149,8 +1239,10 @@ function createAttempt(spec, signal, progress, start) {
|
|
|
1149
1239
|
}
|
|
1150
1240
|
},
|
|
1151
1241
|
signal,
|
|
1152
|
-
|
|
1242
|
+
judgeThrown: /* @__PURE__ */ new Set(),
|
|
1153
1243
|
judgeErrors: [],
|
|
1244
|
+
judgeCalls: 0,
|
|
1245
|
+
judgeInputCalls: [],
|
|
1154
1246
|
captureNames: /* @__PURE__ */ new Set(),
|
|
1155
1247
|
backgrounds: [],
|
|
1156
1248
|
artifactBlobs: /* @__PURE__ */ new Map()
|
|
@@ -1198,7 +1290,7 @@ async function runSetup(a, box, env) {
|
|
|
1198
1290
|
const steps = env.setup;
|
|
1199
1291
|
record.setup = [];
|
|
1200
1292
|
for (const [i, step] of steps.entries()) {
|
|
1201
|
-
const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
|
|
1293
|
+
const what = `setup step ${i + 1}/${steps.length} "${clip(redactSecrets(step.run), 80)}" in environment ${spec.environment.name}`;
|
|
1202
1294
|
const start = performance.now();
|
|
1203
1295
|
const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
|
|
1204
1296
|
const r = await asCheck(a, () => box.exec(step.run, {
|
|
@@ -1218,7 +1310,7 @@ async function runSetup(a, box, env) {
|
|
|
1218
1310
|
durationMs,
|
|
1219
1311
|
output
|
|
1220
1312
|
});
|
|
1221
|
-
debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
|
|
1313
|
+
debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(redactSecrets(step.run), 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
|
|
1222
1314
|
debugTrace("env", "setup", {
|
|
1223
1315
|
run: step.run,
|
|
1224
1316
|
cwd: step.cwd,
|
|
@@ -1231,6 +1323,54 @@ async function runSetup(a, box, env) {
|
|
|
1231
1323
|
throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
|
|
1232
1324
|
}
|
|
1233
1325
|
}
|
|
1326
|
+
/** The `to: 'checks'` secrets the setup steps got as their real value: those without `hosts`. */
|
|
1327
|
+
function unbrokeredChecks(env) {
|
|
1328
|
+
const brokered = new Set(env.substitutions.filter((s) => s.phase === "checks").map((s) => s.name));
|
|
1329
|
+
return Object.keys(env.checkVars).filter((name) => !brokered.has(name));
|
|
1330
|
+
}
|
|
1331
|
+
/** How many of the processes setup left running the record keeps, and how much of each command line. */
|
|
1332
|
+
const SETUP_PROCESSES = 20;
|
|
1333
|
+
const SETUP_COMMAND_CHARS = 200;
|
|
1334
|
+
/**
|
|
1335
|
+
* The processes the setup steps left running with an unbrokered `to: 'checks'` secret in their
|
|
1336
|
+
* environment, by the secret's variable name: a copy under another name or in a process's
|
|
1337
|
+
* arguments isn't found. The listing runs as the agent's user, with no checks secrets of its own,
|
|
1338
|
+
* so it sees the environments the agent could read, and prints names, never values. Docker only:
|
|
1339
|
+
* the fake sandbox runs on the host, with no separation to report on.
|
|
1340
|
+
*/
|
|
1341
|
+
async function leftRunning(a, box, env) {
|
|
1342
|
+
const names = unbrokeredChecks(env);
|
|
1343
|
+
if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
|
|
1344
|
+
const r = await box.exec([
|
|
1345
|
+
"for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
|
|
1346
|
+
"for p in /proc/[0-9]*; do",
|
|
1347
|
+
" [ \"$p\" = \"/proc/$$\" ] && continue",
|
|
1348
|
+
` n=$(tr '\\0' '\\n' < "$p/environ" 2>/dev/null | sed -n -E 's/^(${names.join("|")})=.*/\\1/p' | sort -u | tr '\\n' ' ')`,
|
|
1349
|
+
" [ -n \"$n\" ] || continue",
|
|
1350
|
+
` c=$(tr '\\0\\t\\n' ' ' < "$p/cmdline" 2>/dev/null)`,
|
|
1351
|
+
` [ -n "$c" ] || c=$(tr '\\t\\n' ' ' < "$p/comm" 2>/dev/null)`,
|
|
1352
|
+
` printf '%s\\t%s\\n' "$n" "$c"`,
|
|
1353
|
+
"done",
|
|
1354
|
+
"true"
|
|
1355
|
+
].join("\n"), { signal: a.signal });
|
|
1356
|
+
if (r.exitCode !== 0) {
|
|
1357
|
+
debugLog("env", `couldn't list the processes setup left running: ${tailOf(r.stderr)}`);
|
|
1358
|
+
return [];
|
|
1359
|
+
}
|
|
1360
|
+
const known = new Set(names);
|
|
1361
|
+
const found = r.stdout.split("\n").flatMap((line) => {
|
|
1362
|
+
const tab = line.indexOf(" ");
|
|
1363
|
+
if (tab < 0) return [];
|
|
1364
|
+
const secrets = line.slice(0, tab).split(" ").filter((n) => known.has(n));
|
|
1365
|
+
const command = clip(redactSecrets(line.slice(tab + 1).trim()), SETUP_COMMAND_CHARS);
|
|
1366
|
+
return secrets.length ? [{
|
|
1367
|
+
command,
|
|
1368
|
+
secrets
|
|
1369
|
+
}] : [];
|
|
1370
|
+
});
|
|
1371
|
+
if (found.length > SETUP_PROCESSES) debugLog("env", `setup left ${found.length} processes running with a checks secret; the record keeps the first ${SETUP_PROCESSES}`);
|
|
1372
|
+
return found.slice(0, SETUP_PROCESSES);
|
|
1373
|
+
}
|
|
1234
1374
|
const WORKSPACE_TAR = ".reeval-workspace.tar";
|
|
1235
1375
|
/** Unpacks the git workspace's tarball into the (empty) workspace. */
|
|
1236
1376
|
async function copyWorkspace(box, workspace, signal) {
|
|
@@ -1426,8 +1566,8 @@ async function startSession(a, runId) {
|
|
|
1426
1566
|
signal: a.signal,
|
|
1427
1567
|
env: env.vars,
|
|
1428
1568
|
image: a.image,
|
|
1429
|
-
brokering: brokeringFor(spec.harness),
|
|
1430
|
-
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
|
|
1569
|
+
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
1570
|
+
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1431
1571
|
substitutions: env.substitutions,
|
|
1432
1572
|
egress: spec.egress
|
|
1433
1573
|
});
|
|
@@ -1465,6 +1605,11 @@ async function startSession(a, runId) {
|
|
|
1465
1605
|
for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
|
|
1466
1606
|
for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
|
|
1467
1607
|
await runSetup(a, box, env);
|
|
1608
|
+
const running = await leftRunning(a, box, env);
|
|
1609
|
+
if (running.length) {
|
|
1610
|
+
a.record.setupProcesses = running;
|
|
1611
|
+
debugLog("env", `setup left ${running.length} process${running.length === 1 ? "" : "es"} running with ${[...new Set(running.flatMap((p) => p.secrets))].join(", ")} (to: 'checks', no hosts) in the environment: ${running.map((p) => clip(p.command, 80)).join("; ")}`);
|
|
1612
|
+
}
|
|
1468
1613
|
await snapshot(box, a.signal);
|
|
1469
1614
|
debugLog("env", "workspace snapshot taken: the diff starts here");
|
|
1470
1615
|
const timed = await timeMcpServers(box, env.mcp, a.signal);
|
|
@@ -1473,7 +1618,7 @@ async function startSession(a, runId) {
|
|
|
1473
1618
|
debugLog("env", "stdio MCP servers start through reeval's MCP timer");
|
|
1474
1619
|
} else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
|
|
1475
1620
|
if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
|
|
1476
|
-
const harness = await createHarness(spec.harness);
|
|
1621
|
+
const harness = await createHarness(spec.harness, spec.vertex);
|
|
1477
1622
|
const harnessStart = performance.now();
|
|
1478
1623
|
debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
|
|
1479
1624
|
const session = await startHarness(a, harness, box, {
|
|
@@ -1506,7 +1651,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1506
1651
|
const cutoff = new Promise((resolve) => {
|
|
1507
1652
|
onAbort = () => {
|
|
1508
1653
|
timer = setTimeout(() => {
|
|
1509
|
-
const message = `The agent didn't stop within ${grace / 1e3}s of being
|
|
1654
|
+
const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
|
|
1510
1655
|
const at = Date.now() - Date.parse(startedAt);
|
|
1511
1656
|
resolve({
|
|
1512
1657
|
text: "",
|
|
@@ -1602,7 +1747,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
1602
1747
|
kind: "turn-start",
|
|
1603
1748
|
turn: turnNumber
|
|
1604
1749
|
});
|
|
1605
|
-
debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
|
|
1750
|
+
debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(redactSecrets(prompt), 160))}`);
|
|
1606
1751
|
debugTrace("harness", "prompt", {
|
|
1607
1752
|
turn: turnNumber,
|
|
1608
1753
|
prompt
|
|
@@ -1663,10 +1808,9 @@ function turnSender(a, takeContext, checks) {
|
|
|
1663
1808
|
ms: result.durationMs,
|
|
1664
1809
|
tools: count("tool", "web-search", "web-fetch"),
|
|
1665
1810
|
fileEdits: count("file-edit"),
|
|
1666
|
-
...failure?.type === "error" ? { error: redactSecrets(
|
|
1811
|
+
...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
|
|
1667
1812
|
});
|
|
1668
|
-
|
|
1669
|
-
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
1813
|
+
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
1670
1814
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
1671
1815
|
if (timedOut) {
|
|
1672
1816
|
const error = new InfraError(timedOut);
|
|
@@ -1760,13 +1904,19 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1760
1904
|
throw a.startFailure ? a.startFailure.error : error;
|
|
1761
1905
|
});
|
|
1762
1906
|
a.signal.throwIfAborted();
|
|
1763
|
-
const missed = await unawaited() ??
|
|
1907
|
+
const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
|
|
1764
1908
|
if (a.startFailure) throw a.startFailure.error;
|
|
1765
1909
|
if (missed) throw missed;
|
|
1766
|
-
const
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1910
|
+
const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
|
|
1911
|
+
const [only, ...more] = judgeErrors;
|
|
1912
|
+
if (only) {
|
|
1913
|
+
const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
|
|
1914
|
+
a.judgeThrown.add(message);
|
|
1915
|
+
throw new Error(message);
|
|
1916
|
+
}
|
|
1917
|
+
} catch (thrown) {
|
|
1918
|
+
const error = attemptError(a, thrown, assertions());
|
|
1919
|
+
recordFailure(a, error, timedOut(), assertions());
|
|
1770
1920
|
failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
|
|
1771
1921
|
} finally {
|
|
1772
1922
|
if (!a.signal.aborted) await unawaited().catch(() => {});
|
|
@@ -1788,7 +1938,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1788
1938
|
const checked = assertionRecords(assertions());
|
|
1789
1939
|
if (checked.length) record.assertions = checked;
|
|
1790
1940
|
for (const x of checked) {
|
|
1791
|
-
const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
|
|
1941
|
+
const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
|
|
1792
1942
|
debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
|
|
1793
1943
|
}
|
|
1794
1944
|
const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
|
|
@@ -1810,39 +1960,45 @@ const declaredModeSchema = z.enum([
|
|
|
1810
1960
|
"run",
|
|
1811
1961
|
"skip",
|
|
1812
1962
|
"only"
|
|
1813
|
-
])
|
|
1963
|
+
]);
|
|
1814
1964
|
const declaredFamilySchema = z.strictObject({
|
|
1815
|
-
name: z.string()
|
|
1816
|
-
case: z.number().int().min(0)
|
|
1817
|
-
cases: z.number().int().min(2)
|
|
1818
|
-
})
|
|
1965
|
+
name: z.string(),
|
|
1966
|
+
case: z.number().int().min(0),
|
|
1967
|
+
cases: z.number().int().min(2)
|
|
1968
|
+
});
|
|
1819
1969
|
const declaredTodoSchema = z.strictObject({
|
|
1820
1970
|
type: z.literal("todo"),
|
|
1821
|
-
name: z.string()
|
|
1822
|
-
})
|
|
1971
|
+
name: z.string()
|
|
1972
|
+
});
|
|
1973
|
+
const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
|
|
1974
|
+
const declaredSourceSchema = z.strictObject({
|
|
1975
|
+
body: hash(),
|
|
1976
|
+
case: hash().optional()
|
|
1977
|
+
});
|
|
1823
1978
|
const declaredEvalSchema = z.strictObject({
|
|
1824
1979
|
type: z.literal("eval"),
|
|
1825
|
-
name: z.string()
|
|
1980
|
+
name: z.string(),
|
|
1826
1981
|
mode: declaredModeSchema,
|
|
1827
|
-
options: evalOptionsDataSchema
|
|
1828
|
-
body: z.string()
|
|
1829
|
-
|
|
1830
|
-
|
|
1982
|
+
options: evalOptionsDataSchema,
|
|
1983
|
+
body: z.string(),
|
|
1984
|
+
source: declaredSourceSchema,
|
|
1985
|
+
family: declaredFamilySchema.optional()
|
|
1986
|
+
});
|
|
1831
1987
|
const declaredGroupSchema = z.strictObject({
|
|
1832
1988
|
type: z.literal("group"),
|
|
1833
|
-
name: z.string()
|
|
1989
|
+
name: z.string(),
|
|
1834
1990
|
mode: declaredModeSchema,
|
|
1835
|
-
options: evalOptionsDataSchema
|
|
1991
|
+
options: evalOptionsDataSchema,
|
|
1836
1992
|
get children() {
|
|
1837
1993
|
return z.array(declaredNodeSchema);
|
|
1838
1994
|
}
|
|
1839
|
-
})
|
|
1995
|
+
});
|
|
1840
1996
|
const declaredNodeSchema = z.discriminatedUnion("type", [
|
|
1841
1997
|
declaredEvalSchema,
|
|
1842
1998
|
declaredGroupSchema,
|
|
1843
1999
|
declaredTodoSchema
|
|
1844
|
-
])
|
|
1845
|
-
const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) })
|
|
2000
|
+
]);
|
|
2001
|
+
const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) });
|
|
1846
2002
|
const NODE_SCHEMAS = {
|
|
1847
2003
|
eval: declaredEvalSchema,
|
|
1848
2004
|
group: declaredGroupSchema,
|
|
@@ -1870,20 +2026,18 @@ function configLevel(config) {
|
|
|
1870
2026
|
judge: config.judge,
|
|
1871
2027
|
capture: config.capture,
|
|
1872
2028
|
question: config.question,
|
|
2029
|
+
questions: [],
|
|
1873
2030
|
timeout: void 0,
|
|
1874
2031
|
tags: [],
|
|
2032
|
+
revision: void 0,
|
|
1875
2033
|
pin: [],
|
|
1876
2034
|
sources: {}
|
|
1877
2035
|
};
|
|
1878
2036
|
}
|
|
1879
|
-
/**
|
|
1880
|
-
* The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
|
|
1881
|
-
* `order`). Throws a UsageError for options the plan refuses.
|
|
1882
|
-
*/
|
|
2037
|
+
/** The tests an eval registers. Throws a UsageError for options the plan refuses. */
|
|
1883
2038
|
function planEval(declaration, ctx) {
|
|
1884
|
-
const { name,
|
|
1885
|
-
const
|
|
1886
|
-
const where = `evaluate("${[...groups, name].join(" › ")}")`;
|
|
2039
|
+
const { name, groups: enclosing, options: own } = declaration;
|
|
2040
|
+
const where = `evaluate("${[...enclosing.map((g) => g.name), name].join(" › ")}")`;
|
|
1887
2041
|
checkOptions(own, where);
|
|
1888
2042
|
const levels = [...enclosing.map((g) => ({
|
|
1889
2043
|
options: g.options,
|
|
@@ -1894,15 +2048,82 @@ function planEval(declaration, ctx) {
|
|
|
1894
2048
|
}];
|
|
1895
2049
|
const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
|
|
1896
2050
|
profile: ctx.profile,
|
|
1897
|
-
runs: ctx.runs
|
|
2051
|
+
runs: ctx.runs,
|
|
2052
|
+
skipped: declaration.mode === "skip" || declaration.mode === "todo",
|
|
2053
|
+
pooled: declaration.family !== void 0
|
|
1898
2054
|
});
|
|
2055
|
+
const { harnesses } = resolvedEval;
|
|
2056
|
+
if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
|
|
2057
|
+
tests: null,
|
|
2058
|
+
harnesses
|
|
2059
|
+
};
|
|
2060
|
+
return {
|
|
2061
|
+
tests: planTests(declaration, ctx, where, resolvedEval),
|
|
2062
|
+
harnesses
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
function planTests(declaration, ctx, where, resolvedEval) {
|
|
2066
|
+
const { name, mode, groups: enclosing } = declaration;
|
|
2067
|
+
const groups = enclosing.map((g) => g.name);
|
|
1899
2068
|
const { resolved } = resolvedEval;
|
|
1900
|
-
|
|
2069
|
+
const source = {
|
|
2070
|
+
...declaration.source,
|
|
2071
|
+
options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
|
|
2072
|
+
};
|
|
1901
2073
|
const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
|
|
1902
|
-
const
|
|
2074
|
+
const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
|
|
2075
|
+
const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
|
|
2076
|
+
const routes = /* @__PURE__ */ new Map();
|
|
2077
|
+
const gatewayNotes = [];
|
|
2078
|
+
let vertex;
|
|
2079
|
+
const routed = resolvedEval.skip ? resolved.harness : resolved.harness.map((harness) => {
|
|
2080
|
+
const route = resolveGateway(gateways, harness, where);
|
|
2081
|
+
const hid = harnessId(harness);
|
|
2082
|
+
if (gatewaySet || harness.gateway !== void 0) {
|
|
2083
|
+
routes.set(hid, route);
|
|
2084
|
+
gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
|
|
2085
|
+
}
|
|
2086
|
+
if (route.name === "direct") return harness;
|
|
2087
|
+
if (harness.kind !== "fake") try {
|
|
2088
|
+
if (route.name === "vercel") {
|
|
2089
|
+
const problem = vercelCredentialProblem();
|
|
2090
|
+
if (problem) throw new UsageError(problem);
|
|
2091
|
+
}
|
|
2092
|
+
if (route.name === "vertex") {
|
|
2093
|
+
const entry = gateways.find((g) => g.kind === "vertex");
|
|
2094
|
+
vertex ??= vertexSettings(entry?.kind === "vertex" ? entry : {});
|
|
2095
|
+
}
|
|
2096
|
+
} catch (error) {
|
|
2097
|
+
if (error instanceof UsageError) throw new UsageError(`${where} on ${hid}: ${error.message}`);
|
|
2098
|
+
throw error;
|
|
2099
|
+
}
|
|
2100
|
+
return {
|
|
2101
|
+
...harness,
|
|
2102
|
+
gateway: route.name
|
|
2103
|
+
};
|
|
2104
|
+
});
|
|
2105
|
+
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
2106
|
+
...resolved,
|
|
2107
|
+
harness: routed
|
|
2108
|
+
}, vertex);
|
|
2109
|
+
let judge;
|
|
2110
|
+
if (plannedAttempts.length) try {
|
|
2111
|
+
const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
|
|
2112
|
+
const { model } = resolved.judge;
|
|
2113
|
+
if (typeof model === "string") judge = {
|
|
2114
|
+
model,
|
|
2115
|
+
...gateway ? { gateway } : {}
|
|
2116
|
+
};
|
|
2117
|
+
if (gatewaySet && fellBackToDirect(gateway, resolved.judge.gateway, { gateways })) gatewayNotes.push(`gateway: judge ${model} → direct (fallback)`);
|
|
2118
|
+
} catch (error) {
|
|
2119
|
+
if (error instanceof UsageError) throw new UsageError(`${where}: ${error.message}`);
|
|
2120
|
+
throw error;
|
|
2121
|
+
}
|
|
1903
2122
|
if (ctx.config.secrets.unbrokered === "error") {
|
|
1904
2123
|
const [first] = unbrokered(resolved.env, plannedAttempts);
|
|
1905
2124
|
if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
|
|
2125
|
+
const [plain] = plainHeaders(plannedAttempts);
|
|
2126
|
+
if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
|
|
1906
2127
|
}
|
|
1907
2128
|
const pluginNotes = [];
|
|
1908
2129
|
const pluginLabels = /* @__PURE__ */ new Map();
|
|
@@ -1928,7 +2149,8 @@ function planEval(declaration, ctx) {
|
|
|
1928
2149
|
configFile: ctx.config.file,
|
|
1929
2150
|
file: path
|
|
1930
2151
|
});
|
|
1931
|
-
|
|
2152
|
+
if (gatewayNotes.length && openRouterFallbacks(gateways)) gatewayNotes.push("gateway: openrouter() fallbacks on: reeval's own calls may fail over to another upstream provider; the harnesses' calls take OpenRouter's own routing");
|
|
2153
|
+
plan.notes.push(...gatewayNotes, ...pluginNotes);
|
|
1932
2154
|
const setByOf = (planned) => ({
|
|
1933
2155
|
sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
|
|
1934
2156
|
harness: harnessBy
|
|
@@ -1940,6 +2162,8 @@ function planEval(declaration, ctx) {
|
|
|
1940
2162
|
name,
|
|
1941
2163
|
groups,
|
|
1942
2164
|
tags: resolved.tags,
|
|
2165
|
+
...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
|
|
2166
|
+
source,
|
|
1943
2167
|
cell: first.cell,
|
|
1944
2168
|
n: 1,
|
|
1945
2169
|
label: "",
|
|
@@ -1976,6 +2200,8 @@ function planEval(declaration, ctx) {
|
|
|
1976
2200
|
name,
|
|
1977
2201
|
groups,
|
|
1978
2202
|
tags: resolved.tags,
|
|
2203
|
+
...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
|
|
2204
|
+
source,
|
|
1979
2205
|
cell: planned.cell,
|
|
1980
2206
|
n: planned.n,
|
|
1981
2207
|
label: planned.label,
|
|
@@ -1996,13 +2222,15 @@ function planEval(declaration, ctx) {
|
|
|
1996
2222
|
skip
|
|
1997
2223
|
} : plan,
|
|
1998
2224
|
...skip ? { skip } : {},
|
|
1999
|
-
...family ? { family } : {}
|
|
2225
|
+
...family ? { family } : {},
|
|
2226
|
+
...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {},
|
|
2227
|
+
...judge ? { judge } : {}
|
|
2000
2228
|
};
|
|
2001
2229
|
return {
|
|
2002
2230
|
name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
|
|
2003
2231
|
mode: skip ? "skip" : mode,
|
|
2004
2232
|
attempt,
|
|
2005
|
-
timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
|
|
2233
|
+
timeout: skip ? void 0 : Math.min(timeoutMs + setupAllowance(planned) + GRACE_MS, MAX_TIMER_MS),
|
|
2006
2234
|
gate: skip ? void 0 : {
|
|
2007
2235
|
limits: attemptLimits(planned, ctx.concurrency),
|
|
2008
2236
|
rank: {
|
|
@@ -2033,7 +2261,7 @@ function planEval(declaration, ctx) {
|
|
|
2033
2261
|
/**
|
|
2034
2262
|
* What the plan can tell of an attempt's plugins before any is fetched: a build outside Docker, a
|
|
2035
2263
|
* local plugin's content, and the harness's refusal, as a refusal; the commands that fetch a
|
|
2036
|
-
* package at start, as notes under `allow-all`; and a label for each plugin. Git and npm plugins
|
|
2264
|
+
* package at start, as notes under `proxy` and `allow-all`; and a label for each plugin. Git and npm plugins
|
|
2037
2265
|
* are read when the attempt resolves its environment, which refuses them the same way.
|
|
2038
2266
|
*/
|
|
2039
2267
|
function planPlugins(planned, root) {
|
|
@@ -2182,16 +2410,54 @@ function registerTodo(name) {
|
|
|
2182
2410
|
const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
|
|
2183
2411
|
if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
|
|
2184
2412
|
}
|
|
2185
|
-
/**
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2413
|
+
/**
|
|
2414
|
+
* The evals declared so far, by id, with their name paths and whether each is a family's case,
|
|
2415
|
+
* so a second id is refused.
|
|
2416
|
+
*/
|
|
2417
|
+
const declared = /* @__PURE__ */ new Map();
|
|
2418
|
+
/** The pooled families declared so far, by id: their name path, length and the cases seen. */
|
|
2419
|
+
const families = /* @__PURE__ */ new Map();
|
|
2420
|
+
/** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
|
|
2421
|
+
const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
|
|
2422
|
+
/** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
|
|
2423
|
+
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
|
|
2424
|
+
function checkUnique(where, groups, name, hint, inFamily) {
|
|
2425
|
+
const path = [...groups, name];
|
|
2426
|
+
const id = evalId(collectingFile() ?? "", path);
|
|
2427
|
+
const earlier = declared.get(id);
|
|
2428
|
+
if (earlier) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}${separatorHint(path, earlier.path)}`);
|
|
2429
|
+
const family = inFamily ? void 0 : families.get(id);
|
|
2430
|
+
if (family) throw namedLikeFamily(path, family.path);
|
|
2431
|
+
declared.set(id, {
|
|
2432
|
+
path,
|
|
2433
|
+
inFamily
|
|
2434
|
+
});
|
|
2191
2435
|
}
|
|
2192
|
-
|
|
2436
|
+
/**
|
|
2437
|
+
* A family's cases pool by its id, the groups and the template, so each case may be declared
|
|
2438
|
+
* once and every case must agree on the list's length.
|
|
2439
|
+
*/
|
|
2440
|
+
function checkFamily(groups, family) {
|
|
2441
|
+
const path = [...groups, family.name];
|
|
2442
|
+
const id = evalId(collectingFile() ?? "", path);
|
|
2443
|
+
const earlier = families.get(id);
|
|
2444
|
+
if (!earlier) {
|
|
2445
|
+
const plain = declared.get(id);
|
|
2446
|
+
if (plain && !plain.inFamily) throw namedLikeFamily(plain.path, path);
|
|
2447
|
+
families.set(id, {
|
|
2448
|
+
path,
|
|
2449
|
+
cases: family.cases,
|
|
2450
|
+
seen: /* @__PURE__ */ new Set([family.case])
|
|
2451
|
+
});
|
|
2452
|
+
return;
|
|
2453
|
+
}
|
|
2454
|
+
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
|
|
2455
|
+
earlier.seen.add(family.case);
|
|
2456
|
+
}
|
|
2457
|
+
function register(mode, name, own, fn, source, hint = "", family) {
|
|
2193
2458
|
const groups = groupStack.map((g) => g.name);
|
|
2194
|
-
|
|
2459
|
+
if (family) checkFamily(groups, family);
|
|
2460
|
+
checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
|
|
2195
2461
|
const body = `b${bodies.size + 1}`;
|
|
2196
2462
|
bodies.set(body, fn);
|
|
2197
2463
|
declare({
|
|
@@ -2200,9 +2466,10 @@ function register(mode, name, own, fn, hint = "", family) {
|
|
|
2200
2466
|
mode,
|
|
2201
2467
|
options: data(own),
|
|
2202
2468
|
body,
|
|
2469
|
+
source,
|
|
2203
2470
|
...family ? { family } : {}
|
|
2204
2471
|
});
|
|
2205
|
-
const tests = planEval({
|
|
2472
|
+
const { tests, harnesses } = planEval({
|
|
2206
2473
|
name,
|
|
2207
2474
|
mode,
|
|
2208
2475
|
groups: groupStack.map(({ name, options }) => ({
|
|
@@ -2210,22 +2477,25 @@ function register(mode, name, own, fn, hint = "", family) {
|
|
|
2210
2477
|
options
|
|
2211
2478
|
})),
|
|
2212
2479
|
options: own,
|
|
2480
|
+
source,
|
|
2213
2481
|
...family ? { family } : {}
|
|
2214
2482
|
}, {
|
|
2215
2483
|
...getRunContext(),
|
|
2216
2484
|
file: collectingFile(),
|
|
2217
2485
|
order: registered
|
|
2218
2486
|
});
|
|
2487
|
+
declareHarnesses(harnesses);
|
|
2219
2488
|
if (!tests) return;
|
|
2220
2489
|
registered++;
|
|
2221
2490
|
for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
2222
2491
|
}
|
|
2223
2492
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2224
2493
|
let registered = 0;
|
|
2225
|
-
const defineEval = (mode) => (name, a, b) => {
|
|
2494
|
+
const defineEval = (mode, declared) => (name, a, b) => {
|
|
2226
2495
|
checkOnly(mode, "evaluate");
|
|
2227
2496
|
const [own, fn] = splitArgs("evaluate", name, a, b);
|
|
2228
|
-
|
|
2497
|
+
const where = `evaluate("${name}")`;
|
|
2498
|
+
register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
|
|
2229
2499
|
};
|
|
2230
2500
|
/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
|
|
2231
2501
|
function formatName(template, item, index) {
|
|
@@ -2251,13 +2521,19 @@ const evalEach = (cases, options) => (name, a, b) => {
|
|
|
2251
2521
|
const own = withPaths(written, `evaluate.each("${name}")`);
|
|
2252
2522
|
const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
|
|
2253
2523
|
const pooled = pool && cases.length >= 2;
|
|
2524
|
+
const file = collectingFile();
|
|
2525
|
+
const body = bodyHash(fn, `evaluate.each("${name}")`, file);
|
|
2254
2526
|
cases.forEach((item, i) => {
|
|
2255
2527
|
const family = pooled ? {
|
|
2256
2528
|
name,
|
|
2257
2529
|
case: i,
|
|
2258
2530
|
cases: cases.length
|
|
2259
2531
|
} : void 0;
|
|
2260
|
-
|
|
2532
|
+
const source = {
|
|
2533
|
+
body,
|
|
2534
|
+
case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
|
|
2535
|
+
};
|
|
2536
|
+
register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
|
|
2261
2537
|
});
|
|
2262
2538
|
};
|
|
2263
2539
|
const evaluate = Object.assign(defineEval("run"), {
|
|
@@ -2287,8 +2563,8 @@ function declareNodes(nodes, bodyFor, base) {
|
|
|
2287
2563
|
} else if (node.family) {
|
|
2288
2564
|
const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
|
|
2289
2565
|
checkOnly(node.mode, "evaluate");
|
|
2290
|
-
register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
|
|
2291
|
-
} else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
2566
|
+
register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
|
|
2567
|
+
} else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
2292
2568
|
}
|
|
2293
2569
|
/**
|
|
2294
2570
|
* Options from their data forms, with host paths absolute against `base`: environments become
|
|
@@ -2325,4 +2601,4 @@ function optionsFrom(written, where, base) {
|
|
|
2325
2601
|
//#endregion
|
|
2326
2602
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2327
2603
|
|
|
2328
|
-
//# sourceMappingURL=evaluate-
|
|
2604
|
+
//# sourceMappingURL=evaluate-BPP9s3s6.js.map
|