reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,36 +1,47 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-
|
|
2
|
-
import { r as withLease } from "./client-
|
|
1
|
+
import { t as isObjectLike } from "./object-BkXGPKWY.js";
|
|
2
|
+
import { r as withLease } from "./client-DMmOkbRA.js";
|
|
3
3
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
|
|
4
|
-
import { n as dataProblem, p as
|
|
5
|
-
import {
|
|
4
|
+
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Cf5HMjib.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-ngQEJBjJ.js";
|
|
6
7
|
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import { A as
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
8
|
+
import { $ as egressLabel, A as harnessId, At as NO_JUDGE_INPUT, B as sandboxId, Bt as decideState, D as checkOptions, Dt as writePluginFiles, Et as refusedPlugins, F as plainHeaders, Ft as judgeProblem, G as unbrokeredError, H as sandboxSetBy, Ht as resolveJudgeArtifacts, K as mergeJudge, M as majorityOf, Mt as inputsOn, N as matchesTags, O as evalPlan, P as plainHeaderError, R as refuseIllegalCompare, Rt as artifactRefsOf, S as resolveGateway, St as expectedBlockedFor, T as cascade, Tt as refusedMcp, U as secretVarsIn, V as sandboxProvider, Vt as filePartsOf, W as unbrokered, X as allowedHosts, _ as fellBackToDirect, _t as agentMcp, b as resolveModelGateway, bt as claudeMcpServers, it as createFakeSandbox, k as expand, kt as INPUT_KEYS, nt as unenforceable, q as resolveStateOptions, tt as isDirect, v as gatewayScope, w as brokeredIn, wt as layoutOf, yt as brokeringFor, z as runsOf, zt as badJudgeArtifacts } from "./config-CneBZB84.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-Bg4v4e_K.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-DVIaahLv.js";
|
|
12
|
+
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
|
|
13
|
+
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
|
+
import { pt as shellQuote, t as MCP_TIMER, ut as OPAQUE_OUTPUT } from "./src-C0ySiej5.js";
|
|
13
15
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
14
|
-
import { i as
|
|
15
|
-
import {
|
|
16
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
17
|
-
import { t as attemptDebugDir } from "./files-
|
|
18
|
-
import {
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CZVpkr04.js";
|
|
17
|
+
import { c as formatBytes } from "./debug-CgG3T071.js";
|
|
18
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CpbZ9arV.js";
|
|
19
|
+
import { t as attemptDebugDir } from "./files-D1y8Pg4Z.js";
|
|
20
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
|
|
21
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-iQ6kYJV0.js";
|
|
19
22
|
import { t as contextTaker } from "./context-Dgn48BqT.js";
|
|
20
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
23
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-D_cLa4tW.js";
|
|
21
24
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
22
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
+
import { i as preparedWorkspace } from "./workspace-CBuI9Uds.js";
|
|
26
|
+
import { t as patchPath } from "./patch-CquGAGX5.js";
|
|
27
|
+
import { n as judgeDiff } from "./patch-6T9XAipm.js";
|
|
28
|
+
import { t as transcriptForJudge } from "./transcript-BJrKfgRW.js";
|
|
29
|
+
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CHcahyNK.js";
|
|
30
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-CAE4y2Sr.js";
|
|
25
31
|
import { dirname, join } from "node:path";
|
|
26
32
|
import { parse, stringify } from "yaml";
|
|
27
33
|
import { z } from "zod";
|
|
28
34
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
29
|
-
import {
|
|
35
|
+
import { gunzipSync, gzipSync } from "node:zlib";
|
|
36
|
+
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
30
37
|
//#region src/attempt/assertions.ts
|
|
31
38
|
const MESSAGE_LIMIT = 1024;
|
|
32
39
|
const LABEL_LIMIT = 60;
|
|
33
|
-
|
|
40
|
+
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
41
|
+
const clipped = (text, limit) => {
|
|
42
|
+
const s = redactSecrets(text);
|
|
43
|
+
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
44
|
+
};
|
|
34
45
|
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
35
46
|
const shown = (value) => {
|
|
36
47
|
if (typeof value === "string") return value;
|
|
@@ -44,31 +55,52 @@ const shown = (value) => {
|
|
|
44
55
|
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
45
56
|
/**
|
|
46
57
|
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
47
|
-
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3
|
|
58
|
+
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
59
|
+
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
60
|
+
* `build`s is `build #2 #2`).
|
|
48
61
|
*/
|
|
49
62
|
function assertionRecords(raw) {
|
|
50
63
|
const seen = /* @__PURE__ */ new Map();
|
|
64
|
+
const used = /* @__PURE__ */ new Set();
|
|
51
65
|
return raw.map((r) => {
|
|
52
66
|
const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
53
67
|
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
54
|
-
|
|
68
|
+
let n = seen.get(base) ?? 0;
|
|
69
|
+
let label;
|
|
70
|
+
do {
|
|
71
|
+
n += 1;
|
|
72
|
+
label = n > 1 ? `${base} #${n}` : base;
|
|
73
|
+
} while (used.has(label));
|
|
55
74
|
seen.set(base, n);
|
|
75
|
+
used.add(label);
|
|
56
76
|
return {
|
|
57
|
-
label
|
|
77
|
+
label,
|
|
58
78
|
labelled: r.label !== void 0,
|
|
59
79
|
matcher: r.matcher,
|
|
60
80
|
status: r.status,
|
|
61
81
|
soft: r.soft,
|
|
82
|
+
...r.at ? { at: r.at } : {},
|
|
62
83
|
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
63
84
|
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
64
85
|
};
|
|
65
86
|
});
|
|
66
87
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
88
|
+
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
89
|
+
/**
|
|
90
|
+
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
91
|
+
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
92
|
+
* `CAUGHT_HINT` after its message.
|
|
93
|
+
*/
|
|
94
|
+
function checkFailure(raw) {
|
|
95
|
+
const failed = raw.find((r) => r.status !== "pass");
|
|
70
96
|
if (!failed) return void 0;
|
|
71
|
-
|
|
97
|
+
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
98
|
+
if (failed.soft) return error;
|
|
99
|
+
const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
|
|
100
|
+
caught.name = error.name;
|
|
101
|
+
caught.message = `${error.message} ${CAUGHT_HINT}`;
|
|
102
|
+
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
103
|
+
return caught;
|
|
72
104
|
}
|
|
73
105
|
//#endregion
|
|
74
106
|
//#region src/attempt/debug.ts
|
|
@@ -120,7 +152,7 @@ function traceEnvironment(debug, spec, env) {
|
|
|
120
152
|
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
121
153
|
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
122
154
|
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
123
|
-
const brokering = brokeringFor(spec.harness);
|
|
155
|
+
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
124
156
|
if (brokering) {
|
|
125
157
|
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
126
158
|
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
@@ -406,10 +438,10 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
406
438
|
}
|
|
407
439
|
//#endregion
|
|
408
440
|
//#region src/harness/index.ts
|
|
409
|
-
async function createHarness(descriptor) {
|
|
441
|
+
async function createHarness(descriptor, vertex) {
|
|
410
442
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
411
|
-
const { createAgentHarness } = await import("./agent-
|
|
412
|
-
return createAgentHarness(descriptor);
|
|
443
|
+
const { createAgentHarness } = await import("./agent-BgH4-kFc.js");
|
|
444
|
+
return createAgentHarness(descriptor, vertex);
|
|
413
445
|
}
|
|
414
446
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
415
447
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
@@ -428,9 +460,9 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
428
460
|
});
|
|
429
461
|
a.env = env;
|
|
430
462
|
record.environment = env.record;
|
|
431
|
-
const { secrets, brokered } = env.record;
|
|
463
|
+
const { vars, secrets, brokered } = env.record;
|
|
432
464
|
record.env = {
|
|
433
|
-
vars
|
|
465
|
+
vars,
|
|
434
466
|
secrets,
|
|
435
467
|
brokered
|
|
436
468
|
};
|
|
@@ -438,8 +470,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
438
470
|
if (debug) traceEnvironment(debug, spec, env);
|
|
439
471
|
record.image = null;
|
|
440
472
|
if (spec.sandbox.kind === "docker") {
|
|
441
|
-
const { preparedFor } = await import("./preflight-
|
|
442
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
473
|
+
const { preparedFor } = await import("./preflight-BYXkweNN.js");
|
|
474
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DDNMbv1T.js");
|
|
443
475
|
signal.throwIfAborted();
|
|
444
476
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
445
477
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -472,7 +504,7 @@ const CHARS_PER_TOKEN = 4;
|
|
|
472
504
|
const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
|
|
473
505
|
const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
|
|
474
506
|
const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
|
|
475
|
-
name:
|
|
507
|
+
name: patchPath(part) ?? "",
|
|
476
508
|
chars: part.length
|
|
477
509
|
})).filter((item) => item.name);
|
|
478
510
|
/** The state's parts by size, each with its biggest files where it has any. */
|
|
@@ -508,16 +540,16 @@ function describeSize(state, question) {
|
|
|
508
540
|
return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
|
|
509
541
|
}).join("; ");
|
|
510
542
|
}
|
|
511
|
-
/** `Narrow
|
|
543
|
+
/** `Narrow diff, transcript or output.`: the inputs that were on, largest first. */
|
|
512
544
|
function stateHint(state, question) {
|
|
513
|
-
const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) =>
|
|
545
|
+
const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => p.name);
|
|
514
546
|
if (!keys.length) return "";
|
|
515
547
|
const last = keys.pop();
|
|
516
548
|
return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
|
|
517
549
|
}
|
|
518
550
|
/**
|
|
519
551
|
* A provider's error, as the provider worded it, then what the judge's input held: its estimated
|
|
520
|
-
* size and biggest parts and files, and the
|
|
552
|
+
* size and biggest parts and files, and the inputs that were on, largest first. reeval knows
|
|
521
553
|
* no model's limit, so this follows every provider error.
|
|
522
554
|
*/
|
|
523
555
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
@@ -557,7 +589,7 @@ const workspacePath = (path) => path.replace(/^\.\//, "");
|
|
|
557
589
|
/** A `contents` entry that names one file rather than matching many. */
|
|
558
590
|
const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
|
|
559
591
|
/**
|
|
560
|
-
* The judge's state for
|
|
592
|
+
* The judge's state for the inputs the call turned on: the graded turn's task and output (or every turn's),
|
|
561
593
|
* its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
|
|
562
594
|
* transcript.
|
|
563
595
|
*/
|
|
@@ -605,7 +637,7 @@ function checksShown(which, ran) {
|
|
|
605
637
|
if (which === false) return void 0;
|
|
606
638
|
if (which !== true) {
|
|
607
639
|
const missing = which.find((command) => !ran.some((r) => r.command === command));
|
|
608
|
-
if (missing !== void 0) throw new Error(`c.judge()
|
|
640
|
+
if (missing !== void 0) throw new Error(`c.judge() checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
|
|
609
641
|
}
|
|
610
642
|
return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
|
|
611
643
|
command,
|
|
@@ -615,29 +647,38 @@ function checksShown(which, ran) {
|
|
|
615
647
|
}));
|
|
616
648
|
}
|
|
617
649
|
/**
|
|
650
|
+
* Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
|
|
651
|
+
* what they record stays in the order they were made.
|
|
652
|
+
*/
|
|
653
|
+
function placeFor(calls, order) {
|
|
654
|
+
const after = calls.findIndex((c) => c > order);
|
|
655
|
+
return after < 0 ? calls.length : after;
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
618
658
|
* Grades a turn (the last by default) or a value against a question, under
|
|
619
659
|
* the judge provider's concurrency lease. Every call's input is recorded, including calls that
|
|
620
660
|
* fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
|
|
621
661
|
* checks after it still run.
|
|
622
662
|
*/
|
|
623
663
|
function judgeCaller(a, readFile) {
|
|
624
|
-
const ask = async (question, { on, ...call }
|
|
625
|
-
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
664
|
+
const ask = async (order, question, { on, ...call }) => {
|
|
626
665
|
const problem = judgeProblem(call, { at: "call" });
|
|
627
666
|
if (problem) throw new Error(problem);
|
|
628
667
|
const badArts = badJudgeArtifacts(call.artifacts);
|
|
629
668
|
if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
|
|
630
669
|
const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
|
|
631
670
|
if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
|
|
632
|
-
const
|
|
671
|
+
const wanted = resolveStateOptions(call);
|
|
672
|
+
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
|
|
673
|
+
const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
|
|
674
|
+
const options = mergeJudge(a.spec.judge, callJudge);
|
|
633
675
|
const { model } = options;
|
|
634
676
|
if (model === void 0) {
|
|
635
|
-
|
|
636
|
-
|
|
677
|
+
const message = noJudgeModel(getRunContext().config.file);
|
|
678
|
+
a.judgeThrown.add(message);
|
|
679
|
+
throw new Error(message);
|
|
637
680
|
}
|
|
638
|
-
const
|
|
639
|
-
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
|
|
640
|
-
const { state: _state, maxConcurrent, ...judgeOptions } = options;
|
|
681
|
+
const { maxConcurrent, ...judgeOptions } = options;
|
|
641
682
|
const output = on && "value" in on ? on.value : subject.text;
|
|
642
683
|
const { draft, dropped } = await draftState({
|
|
643
684
|
turns: a.turns,
|
|
@@ -649,8 +690,9 @@ function judgeCaller(a, readFile) {
|
|
|
649
690
|
listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
|
|
650
691
|
});
|
|
651
692
|
const name = judgeModelName(model);
|
|
652
|
-
const state = stateOf(draft);
|
|
653
|
-
const resolved = resolveJudgeArtifacts(
|
|
693
|
+
const state = redactValue(stateOf(draft));
|
|
694
|
+
const resolved = resolveJudgeArtifacts(artifacts, a.artifactBlobs);
|
|
695
|
+
if (artifacts === true && !resolved.length && inputsOn(call, INPUT_KEYS).length === 1) throw new Error("c.judge() artifacts: true shows the judge nothing yet: this attempt has no captures. Call c.capture() first, or turn on another input.");
|
|
654
696
|
const refs = artifactRefsOf(resolved);
|
|
655
697
|
const decide = decideState(state, filePartsOf(resolved));
|
|
656
698
|
const input = {
|
|
@@ -661,7 +703,9 @@ function judgeCaller(a, readFile) {
|
|
|
661
703
|
...refs.length ? { artifacts: refs } : {},
|
|
662
704
|
...dropped.length ? { dropped } : {}
|
|
663
705
|
};
|
|
664
|
-
a.
|
|
706
|
+
const place = placeFor(a.judgeInputCalls, order);
|
|
707
|
+
a.judgeInputCalls.splice(place, 0, order);
|
|
708
|
+
a.record.judgeInputs?.splice(place, 0, input);
|
|
665
709
|
const start = performance.now();
|
|
666
710
|
const ms = () => Math.round(performance.now() - start);
|
|
667
711
|
const atLeast = options.threshold ?? .8;
|
|
@@ -669,12 +713,14 @@ function judgeCaller(a, readFile) {
|
|
|
669
713
|
kind: "judge-start",
|
|
670
714
|
question
|
|
671
715
|
});
|
|
672
|
-
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
|
|
716
|
+
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
|
|
673
717
|
if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
|
|
674
718
|
debugTrace("judge", "prompt", input, "judge prompt");
|
|
675
719
|
const failed = (message, retries) => {
|
|
676
|
-
a.
|
|
677
|
-
|
|
720
|
+
a.judgeErrors.push({
|
|
721
|
+
call: order,
|
|
722
|
+
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
|
|
723
|
+
});
|
|
678
724
|
input.error = message;
|
|
679
725
|
if (retries?.length) input.retries = retries;
|
|
680
726
|
debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
|
|
@@ -703,11 +749,13 @@ function judgeCaller(a, readFile) {
|
|
|
703
749
|
const provider = name.split("/")[0];
|
|
704
750
|
try {
|
|
705
751
|
const { config, profile } = getRunContext();
|
|
706
|
-
const
|
|
752
|
+
const scope = gatewayScope(config, profile);
|
|
753
|
+
const gateway = resolveModelGateway(model, judgeOptions.gateway, scope, "judge model");
|
|
707
754
|
const { gateway: _pin, ...unpinned } = judgeOptions;
|
|
755
|
+
const entry = gateway && scope.gateways.find((g) => g.kind === gateway);
|
|
708
756
|
const routed = gateway ? {
|
|
709
757
|
...unpinned,
|
|
710
|
-
gateway
|
|
758
|
+
gateway: entry || gateway
|
|
711
759
|
} : unpinned;
|
|
712
760
|
const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
713
761
|
...routed,
|
|
@@ -718,10 +766,12 @@ function judgeCaller(a, readFile) {
|
|
|
718
766
|
...verdict,
|
|
719
767
|
artifacts: refs
|
|
720
768
|
} : verdict;
|
|
721
|
-
a.record
|
|
769
|
+
const { judgements: judgments } = a.record;
|
|
770
|
+
judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
|
|
722
771
|
...recorded,
|
|
723
|
-
retries
|
|
724
|
-
|
|
772
|
+
...retries.length ? { retries } : {},
|
|
773
|
+
call: order
|
|
774
|
+
});
|
|
725
775
|
const { score } = verdict;
|
|
726
776
|
debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
|
|
727
777
|
debugTrace("judge", "response", {
|
|
@@ -738,21 +788,22 @@ function judgeCaller(a, readFile) {
|
|
|
738
788
|
});
|
|
739
789
|
return recorded;
|
|
740
790
|
} catch (error) {
|
|
741
|
-
if (a.signal.aborted)
|
|
742
|
-
a.judgeErrored = true;
|
|
743
|
-
throw error;
|
|
744
|
-
}
|
|
791
|
+
if (a.signal.aborted) throw error;
|
|
745
792
|
const { retries } = error;
|
|
746
793
|
const message = error instanceof Error ? error.message : String(error);
|
|
747
794
|
return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
|
|
748
795
|
}
|
|
749
796
|
};
|
|
750
|
-
return async (question,
|
|
797
|
+
return async (question, given) => {
|
|
798
|
+
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
799
|
+
if (given === null || typeof given !== "object" || Array.isArray(given)) throw new Error(judgeProblem(given, { at: "call" }) ?? NO_JUDGE_INPUT);
|
|
800
|
+
const { on, ...options } = given;
|
|
801
|
+
const order = a.judgeCalls++;
|
|
751
802
|
const call = on === void 0 ? options : {
|
|
752
803
|
...options,
|
|
753
804
|
on: judgeOn(a.turns, on)
|
|
754
805
|
};
|
|
755
|
-
return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
|
|
806
|
+
return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
|
|
756
807
|
};
|
|
757
808
|
}
|
|
758
809
|
//#endregion
|
|
@@ -762,7 +813,7 @@ async function createSandbox(descriptor, options) {
|
|
|
762
813
|
switch (descriptor.kind) {
|
|
763
814
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
764
815
|
case "docker": {
|
|
765
|
-
const { createDockerSandbox } = await import("./docker-
|
|
816
|
+
const { createDockerSandbox } = await import("./docker-DCAtFMo9.js");
|
|
766
817
|
return await createDockerSandbox(descriptor, options);
|
|
767
818
|
}
|
|
768
819
|
}
|
|
@@ -781,7 +832,7 @@ function classify(error, signal) {
|
|
|
781
832
|
message: "The eval timed out"
|
|
782
833
|
} : {
|
|
783
834
|
kind: "aborted",
|
|
784
|
-
message: "The run was
|
|
835
|
+
message: "The run was canceled"
|
|
785
836
|
};
|
|
786
837
|
if (error instanceof InfraError) return {
|
|
787
838
|
kind: "infra",
|
|
@@ -792,17 +843,39 @@ function classify(error, signal) {
|
|
|
792
843
|
message
|
|
793
844
|
};
|
|
794
845
|
}
|
|
846
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
847
|
+
const judged = (r) => /(^|\.)toPass$/.test(r.matcher);
|
|
848
|
+
/**
|
|
849
|
+
* Whether `error` is a judge's: one a judge threw, or what a `toPass()` or `.not.toPass()` call
|
|
850
|
+
* failed with. A failing verdict the eval didn't assert, or asserted with `.not.toPass()`, isn't.
|
|
851
|
+
* By message too, since another language's runner hands back its own copy of the error.
|
|
852
|
+
*/
|
|
853
|
+
function isJudges(a, error, raw) {
|
|
854
|
+
const message = messageOf(error);
|
|
855
|
+
if (a.judgeThrown.has(message)) return true;
|
|
856
|
+
return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
|
|
857
|
+
}
|
|
858
|
+
/**
|
|
859
|
+
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
860
|
+
* (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
|
|
861
|
+
* fails as that check.
|
|
862
|
+
*/
|
|
863
|
+
function attemptError(a, error, raw) {
|
|
864
|
+
if (!isJudges(a, error, raw)) return error;
|
|
865
|
+
const check = raw.find((r) => r.status !== "pass" && !judged(r));
|
|
866
|
+
return check ? checkFailure([check]) : error;
|
|
867
|
+
}
|
|
795
868
|
/**
|
|
796
869
|
* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
|
|
797
|
-
* assertion, which is the judge's when
|
|
870
|
+
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
|
|
798
871
|
*/
|
|
799
|
-
function recordFailure(a, error, timedOut) {
|
|
872
|
+
function recordFailure(a, error, timedOut, raw) {
|
|
800
873
|
const { record } = a;
|
|
801
874
|
record.error = timedOut ? {
|
|
802
875
|
kind: "timeout",
|
|
803
876
|
message: "The eval timed out"
|
|
804
877
|
} : classify(error, a.signal);
|
|
805
|
-
record.error.message = capMiddle(record.error.message);
|
|
878
|
+
record.error.message = capMiddle(redactSecrets(record.error.message));
|
|
806
879
|
if (error instanceof PrepareError && !timedOut) {
|
|
807
880
|
const { class: kind, stage, ref, cause, log, setBy } = error.failure;
|
|
808
881
|
record.error.prepare = {
|
|
@@ -814,8 +887,7 @@ function recordFailure(a, error, timedOut) {
|
|
|
814
887
|
setBy: setBy ?? []
|
|
815
888
|
};
|
|
816
889
|
}
|
|
817
|
-
|
|
818
|
-
if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
|
|
890
|
+
if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
|
|
819
891
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
820
892
|
}
|
|
821
893
|
/**
|
|
@@ -842,11 +914,11 @@ async function recordOom(a, sandbox) {
|
|
|
842
914
|
/**
|
|
843
915
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
844
916
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
845
|
-
*
|
|
917
|
+
* summarized in one `-v` line.
|
|
846
918
|
*/
|
|
847
919
|
function recordEgress(a, sandbox) {
|
|
848
920
|
const traffic = sandbox?.traffic?.();
|
|
849
|
-
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
|
|
921
|
+
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
|
|
850
922
|
const refused = traffic?.blocked ?? [];
|
|
851
923
|
const isExpected = (h) => !!expected?.hosts.includes(h.host);
|
|
852
924
|
const expectedBlocked = refused.filter(isExpected);
|
|
@@ -866,6 +938,24 @@ function recordEgress(a, sandbox) {
|
|
|
866
938
|
a.record.servedModels = sandbox?.servedModels?.() ?? null;
|
|
867
939
|
const asked = sandbox?.askedModels?.() ?? [];
|
|
868
940
|
if (asked.length) a.record.askedModels = asked;
|
|
941
|
+
const upstreams = sandbox?.upstreams?.() ?? [];
|
|
942
|
+
if (upstreams.length) a.record.upstreams = upstreams;
|
|
943
|
+
const cost = sandbox?.reportedCostUsd?.();
|
|
944
|
+
if (cost !== void 0) a.record.reportedCostUsd = cost;
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* The record as its staging file holds it: redacted, with a secret split across a turn's raw
|
|
948
|
+
* stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
|
|
949
|
+
*/
|
|
950
|
+
function stagedRecordJson(record) {
|
|
951
|
+
const turns = record.turns.map((turn) => turn.raw ? {
|
|
952
|
+
...turn,
|
|
953
|
+
raw: redactSplitSecrets(turn.raw.map((part) => JSON.stringify(part))).map((line) => JSON.parse(line))
|
|
954
|
+
} : turn);
|
|
955
|
+
return redactJson({
|
|
956
|
+
...record,
|
|
957
|
+
turns
|
|
958
|
+
}, 2);
|
|
869
959
|
}
|
|
870
960
|
/** Writes the attempt's record, redacted, and returns what the runner reports. */
|
|
871
961
|
async function writeRecord(a, stagingDir, id) {
|
|
@@ -874,7 +964,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
874
964
|
await mkdir(join(stagingDir, "evals"), { recursive: true });
|
|
875
965
|
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
|
|
876
966
|
const tmp = `${recordPath}.${process.pid}.tmp`;
|
|
877
|
-
await writeFile(tmp,
|
|
967
|
+
await writeFile(tmp, stagedRecordJson(record));
|
|
878
968
|
await rename(tmp, recordPath);
|
|
879
969
|
return {
|
|
880
970
|
outcome: record.outcome,
|
|
@@ -886,17 +976,28 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
886
976
|
question: clip(j.question, 200)
|
|
887
977
|
})),
|
|
888
978
|
turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
|
|
979
|
+
...failedChecks(record),
|
|
889
980
|
...record.error?.prepare ? { prepare: {
|
|
890
981
|
ref: record.error.prepare.ref,
|
|
891
982
|
stage: record.error.prepare.stage
|
|
892
983
|
} } : {}
|
|
893
984
|
};
|
|
894
985
|
}
|
|
986
|
+
/** The record's checks that didn't pass, for the terminal; unset when all did. */
|
|
987
|
+
function failedChecks(record) {
|
|
988
|
+
const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
|
|
989
|
+
if (!failed.length) return {};
|
|
990
|
+
return { failedChecks: failed.map((x) => ({
|
|
991
|
+
label: x.label,
|
|
992
|
+
...x.at ? { at: x.at } : {},
|
|
993
|
+
message: clip(x.message?.split("\n")[0] ?? "", 500)
|
|
994
|
+
})) };
|
|
995
|
+
}
|
|
895
996
|
/** `turn N error: <message>` for each turn with an error event. */
|
|
896
997
|
function turnErrors(transcripts) {
|
|
897
998
|
return transcripts.flatMap((events, i) => {
|
|
898
999
|
const found = events.find((e) => e.type === "error");
|
|
899
|
-
return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(
|
|
1000
|
+
return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
|
|
900
1001
|
});
|
|
901
1002
|
}
|
|
902
1003
|
//#endregion
|
|
@@ -928,7 +1029,7 @@ function hookHandlers(plugin, text, hooks) {
|
|
|
928
1029
|
const what = `the ${plugin.name} plugin's hooks/hooks.json`;
|
|
929
1030
|
let raw;
|
|
930
1031
|
try {
|
|
931
|
-
raw = JSON.parse(text);
|
|
1032
|
+
raw = JSON.parse(withoutBom(text));
|
|
932
1033
|
} catch {
|
|
933
1034
|
throw new InfraError(`${what} isn't valid JSON`);
|
|
934
1035
|
}
|
|
@@ -989,7 +1090,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
989
1090
|
if (!hooks) {
|
|
990
1091
|
let own = {};
|
|
991
1092
|
try {
|
|
992
|
-
own = hooksFile === null ? {} : JSON.parse(hooksFile);
|
|
1093
|
+
own = hooksFile === null ? {} : JSON.parse(withoutBom(hooksFile));
|
|
993
1094
|
} catch {
|
|
994
1095
|
throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
|
|
995
1096
|
}
|
|
@@ -1108,7 +1209,7 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
|
1108
1209
|
//#region src/attempt/state.ts
|
|
1109
1210
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1110
1211
|
const SETUP_BUDGET_MS = 12e5;
|
|
1111
|
-
function createAttempt(spec, signal, progress, start) {
|
|
1212
|
+
function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
1112
1213
|
const record = {
|
|
1113
1214
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1114
1215
|
outcome: "pass",
|
|
@@ -1157,11 +1258,14 @@ function createAttempt(spec, signal, progress, start) {
|
|
|
1157
1258
|
}
|
|
1158
1259
|
},
|
|
1159
1260
|
signal,
|
|
1160
|
-
|
|
1261
|
+
judgeThrown: /* @__PURE__ */ new Set(),
|
|
1161
1262
|
judgeErrors: [],
|
|
1263
|
+
judgeCalls: 0,
|
|
1264
|
+
judgeInputCalls: [],
|
|
1162
1265
|
captureNames: /* @__PURE__ */ new Set(),
|
|
1163
1266
|
backgrounds: [],
|
|
1164
|
-
artifactBlobs: /* @__PURE__ */ new Map()
|
|
1267
|
+
artifactBlobs: /* @__PURE__ */ new Map(),
|
|
1268
|
+
snapshotDir
|
|
1165
1269
|
};
|
|
1166
1270
|
return a;
|
|
1167
1271
|
}
|
|
@@ -1206,7 +1310,7 @@ async function runSetup(a, box, env) {
|
|
|
1206
1310
|
const steps = env.setup;
|
|
1207
1311
|
record.setup = [];
|
|
1208
1312
|
for (const [i, step] of steps.entries()) {
|
|
1209
|
-
const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
|
|
1313
|
+
const what = `setup step ${i + 1}/${steps.length} "${clip(redactSecrets(step.run), 80)}" in environment ${spec.environment.name}`;
|
|
1210
1314
|
const start = performance.now();
|
|
1211
1315
|
const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
|
|
1212
1316
|
const r = await asCheck(a, () => box.exec(step.run, {
|
|
@@ -1226,7 +1330,7 @@ async function runSetup(a, box, env) {
|
|
|
1226
1330
|
durationMs,
|
|
1227
1331
|
output
|
|
1228
1332
|
});
|
|
1229
|
-
debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
|
|
1333
|
+
debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(redactSecrets(step.run), 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
|
|
1230
1334
|
debugTrace("env", "setup", {
|
|
1231
1335
|
run: step.run,
|
|
1232
1336
|
cwd: step.cwd,
|
|
@@ -1239,6 +1343,54 @@ async function runSetup(a, box, env) {
|
|
|
1239
1343
|
throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
|
|
1240
1344
|
}
|
|
1241
1345
|
}
|
|
1346
|
+
/** The `to: 'checks'` secrets the setup steps got as their real value: those without `hosts`. */
|
|
1347
|
+
function unbrokeredChecks(env) {
|
|
1348
|
+
const brokered = new Set(env.substitutions.filter((s) => s.phase === "checks").map((s) => s.name));
|
|
1349
|
+
return Object.keys(env.checkVars).filter((name) => !brokered.has(name));
|
|
1350
|
+
}
|
|
1351
|
+
/** How many of the processes setup left running the record keeps, and how much of each command line. */
|
|
1352
|
+
const SETUP_PROCESSES = 20;
|
|
1353
|
+
const SETUP_COMMAND_CHARS = 200;
|
|
1354
|
+
/**
|
|
1355
|
+
* The processes the setup steps left running with an unbrokered `to: 'checks'` secret in their
|
|
1356
|
+
* environment, by the secret's variable name: a copy under another name or in a process's
|
|
1357
|
+
* arguments isn't found. The listing runs as the agent's user, with no checks secrets of its own,
|
|
1358
|
+
* so it sees the environments the agent could read, and prints names, never values. Docker only:
|
|
1359
|
+
* the fake sandbox runs on the host, with no separation to report on.
|
|
1360
|
+
*/
|
|
1361
|
+
async function leftRunning(a, box, env) {
|
|
1362
|
+
const names = unbrokeredChecks(env);
|
|
1363
|
+
if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
|
|
1364
|
+
const r = await box.exec([
|
|
1365
|
+
"for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
|
|
1366
|
+
"for p in /proc/[0-9]*; do",
|
|
1367
|
+
" [ \"$p\" = \"/proc/$$\" ] && continue",
|
|
1368
|
+
` n=$(tr '\\0' '\\n' < "$p/environ" 2>/dev/null | sed -n -E 's/^(${names.join("|")})=.*/\\1/p' | sort -u | tr '\\n' ' ')`,
|
|
1369
|
+
" [ -n \"$n\" ] || continue",
|
|
1370
|
+
` c=$(tr '\\0\\t\\n' ' ' < "$p/cmdline" 2>/dev/null)`,
|
|
1371
|
+
` [ -n "$c" ] || c=$(tr '\\t\\n' ' ' < "$p/comm" 2>/dev/null)`,
|
|
1372
|
+
` printf '%s\\t%s\\n' "$n" "$c"`,
|
|
1373
|
+
"done",
|
|
1374
|
+
"true"
|
|
1375
|
+
].join("\n"), { signal: a.signal });
|
|
1376
|
+
if (r.exitCode !== 0) {
|
|
1377
|
+
debugLog("env", `couldn't list the processes setup left running: ${tailOf(r.stderr)}`);
|
|
1378
|
+
return [];
|
|
1379
|
+
}
|
|
1380
|
+
const known = new Set(names);
|
|
1381
|
+
const found = r.stdout.split("\n").flatMap((line) => {
|
|
1382
|
+
const tab = line.indexOf(" ");
|
|
1383
|
+
if (tab < 0) return [];
|
|
1384
|
+
const secrets = line.slice(0, tab).split(" ").filter((n) => known.has(n));
|
|
1385
|
+
const command = clip(redactSecrets(line.slice(tab + 1).trim()), SETUP_COMMAND_CHARS);
|
|
1386
|
+
return secrets.length ? [{
|
|
1387
|
+
command,
|
|
1388
|
+
secrets
|
|
1389
|
+
}] : [];
|
|
1390
|
+
});
|
|
1391
|
+
if (found.length > SETUP_PROCESSES) debugLog("env", `setup left ${found.length} processes running with a checks secret; the record keeps the first ${SETUP_PROCESSES}`);
|
|
1392
|
+
return found.slice(0, SETUP_PROCESSES);
|
|
1393
|
+
}
|
|
1242
1394
|
const WORKSPACE_TAR = ".reeval-workspace.tar";
|
|
1243
1395
|
/** Unpacks the git workspace's tarball into the (empty) workspace. */
|
|
1244
1396
|
async function copyWorkspace(box, workspace, signal) {
|
|
@@ -1434,8 +1586,8 @@ async function startSession(a, runId) {
|
|
|
1434
1586
|
signal: a.signal,
|
|
1435
1587
|
env: env.vars,
|
|
1436
1588
|
image: a.image,
|
|
1437
|
-
brokering: brokeringFor(spec.harness),
|
|
1438
|
-
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
|
|
1589
|
+
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
1590
|
+
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1439
1591
|
substitutions: env.substitutions,
|
|
1440
1592
|
egress: spec.egress
|
|
1441
1593
|
});
|
|
@@ -1473,6 +1625,11 @@ async function startSession(a, runId) {
|
|
|
1473
1625
|
for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
|
|
1474
1626
|
for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
|
|
1475
1627
|
await runSetup(a, box, env);
|
|
1628
|
+
const running = await leftRunning(a, box, env);
|
|
1629
|
+
if (running.length) {
|
|
1630
|
+
a.record.setupProcesses = running;
|
|
1631
|
+
debugLog("env", `setup left ${running.length} process${running.length === 1 ? "" : "es"} running with ${[...new Set(running.flatMap((p) => p.secrets))].join(", ")} (to: 'checks', no hosts) in the environment: ${running.map((p) => clip(p.command, 80)).join("; ")}`);
|
|
1632
|
+
}
|
|
1476
1633
|
await snapshot(box, a.signal);
|
|
1477
1634
|
debugLog("env", "workspace snapshot taken: the diff starts here");
|
|
1478
1635
|
const timed = await timeMcpServers(box, env.mcp, a.signal);
|
|
@@ -1481,7 +1638,7 @@ async function startSession(a, runId) {
|
|
|
1481
1638
|
debugLog("env", "stdio MCP servers start through reeval's MCP timer");
|
|
1482
1639
|
} else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
|
|
1483
1640
|
if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
|
|
1484
|
-
const harness = await createHarness(spec.harness);
|
|
1641
|
+
const harness = await createHarness(spec.harness, spec.vertex);
|
|
1485
1642
|
const harnessStart = performance.now();
|
|
1486
1643
|
debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
|
|
1487
1644
|
const session = await startHarness(a, harness, box, {
|
|
@@ -1501,6 +1658,237 @@ async function startSession(a, runId) {
|
|
|
1501
1658
|
return [box, session];
|
|
1502
1659
|
}
|
|
1503
1660
|
//#endregion
|
|
1661
|
+
//#region src/attempt/snapshot.ts
|
|
1662
|
+
/** The most a snapshot directory's files may add up to; a larger one isn't taken. */
|
|
1663
|
+
const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
|
|
1664
|
+
/** The most a compressed snapshot may be. */
|
|
1665
|
+
const SNAPSHOT_TAR_LIMIT = 10 * 2 ** 20;
|
|
1666
|
+
/** The most a compressed snapshot may be uncompressed, which the host holds in memory to mask. */
|
|
1667
|
+
const SNAPSHOT_RAW_LIMIT = 200 * 2 ** 20;
|
|
1668
|
+
/** How long one snapshot may take before the attempt goes on without it. */
|
|
1669
|
+
const SNAPSHOT_TIMEOUT_MS = 12e4;
|
|
1670
|
+
/** Installed dependencies, caches and git's objects: left out by directory name, at any depth. */
|
|
1671
|
+
const EXCLUDED_DIRS = [
|
|
1672
|
+
".git",
|
|
1673
|
+
"node_modules",
|
|
1674
|
+
".pnpm-store",
|
|
1675
|
+
".venv",
|
|
1676
|
+
"venv",
|
|
1677
|
+
".tox",
|
|
1678
|
+
"__pycache__",
|
|
1679
|
+
".pytest_cache",
|
|
1680
|
+
".mypy_cache",
|
|
1681
|
+
".ruff_cache",
|
|
1682
|
+
".next",
|
|
1683
|
+
".nuxt",
|
|
1684
|
+
".svelte-kit",
|
|
1685
|
+
".turbo",
|
|
1686
|
+
".parcel-cache",
|
|
1687
|
+
".cache",
|
|
1688
|
+
".dart_tool",
|
|
1689
|
+
".gradle"
|
|
1690
|
+
];
|
|
1691
|
+
const LISTED = 50;
|
|
1692
|
+
const SIZE_MARK = "--reeval-size--";
|
|
1693
|
+
const excluded = (path) => path.split("/").some((part) => EXCLUDED_DIRS.includes(part));
|
|
1694
|
+
const utf8 = new TextDecoder("utf-8", { fatal: true });
|
|
1695
|
+
function asText(data) {
|
|
1696
|
+
if (data.subarray(0, 8192).includes(0)) return null;
|
|
1697
|
+
try {
|
|
1698
|
+
return utf8.decode(data);
|
|
1699
|
+
} catch {
|
|
1700
|
+
return null;
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
/** The sandbox's archive, with the excluded directories gone and secret-looking values masked. */
|
|
1704
|
+
function cleanArchive(tar) {
|
|
1705
|
+
const entries = [];
|
|
1706
|
+
const masked = [];
|
|
1707
|
+
const dropped = [];
|
|
1708
|
+
const left = /* @__PURE__ */ new Set();
|
|
1709
|
+
for (const e of readTar(tar)) {
|
|
1710
|
+
if (excluded(e.path)) {
|
|
1711
|
+
const parts = e.path.split("/");
|
|
1712
|
+
const at = parts.findIndex((p) => EXCLUDED_DIRS.includes(p));
|
|
1713
|
+
left.add(redactSecrets(`${parts.slice(0, at + 1).join("/")}/`));
|
|
1714
|
+
continue;
|
|
1715
|
+
}
|
|
1716
|
+
if (findSecrets(e.path).length || e.type === "symlink" && findSecrets(e.target).length) {
|
|
1717
|
+
dropped.push(redactSecrets(e.path));
|
|
1718
|
+
continue;
|
|
1719
|
+
}
|
|
1720
|
+
if (e.type !== "file") {
|
|
1721
|
+
entries.push(e);
|
|
1722
|
+
continue;
|
|
1723
|
+
}
|
|
1724
|
+
const text = asText(e.data);
|
|
1725
|
+
if (text === null) {
|
|
1726
|
+
if (findSecrets(e.data.toString("latin1")).length) dropped.push(e.path);
|
|
1727
|
+
else entries.push(e);
|
|
1728
|
+
continue;
|
|
1729
|
+
}
|
|
1730
|
+
const clean = redactSecrets(text);
|
|
1731
|
+
if (clean === text) entries.push(e);
|
|
1732
|
+
else {
|
|
1733
|
+
masked.push(e.path);
|
|
1734
|
+
entries.push({
|
|
1735
|
+
...e,
|
|
1736
|
+
data: Buffer.from(clean)
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
return {
|
|
1741
|
+
entries,
|
|
1742
|
+
masked,
|
|
1743
|
+
dropped,
|
|
1744
|
+
excluded: [...left]
|
|
1745
|
+
};
|
|
1746
|
+
}
|
|
1747
|
+
const tarCommand = (workdir) => {
|
|
1748
|
+
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
|
|
1749
|
+
return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
1750
|
+
};
|
|
1751
|
+
const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
|
|
1752
|
+
const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
|
|
1753
|
+
const listed = (paths) => paths.slice(0, LISTED);
|
|
1754
|
+
const LIMITS = {
|
|
1755
|
+
dir: SNAPSHOT_DIR_LIMIT,
|
|
1756
|
+
tar: SNAPSHOT_TAR_LIMIT,
|
|
1757
|
+
raw: SNAPSHOT_RAW_LIMIT
|
|
1758
|
+
};
|
|
1759
|
+
/**
|
|
1760
|
+
* Archives the sandbox's workspace and returns what to record and the masked entries to write,
|
|
1761
|
+
* or why none: over its form's cap, or an archive the sandbox couldn't make.
|
|
1762
|
+
*/
|
|
1763
|
+
async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
|
|
1764
|
+
const none = (cause, message, dirs = []) => ({
|
|
1765
|
+
taken: {
|
|
1766
|
+
bytes: 0,
|
|
1767
|
+
files: 0,
|
|
1768
|
+
excluded: listed(dirs),
|
|
1769
|
+
masked: [],
|
|
1770
|
+
dropped: [],
|
|
1771
|
+
skipped: {
|
|
1772
|
+
cause,
|
|
1773
|
+
message
|
|
1774
|
+
}
|
|
1775
|
+
},
|
|
1776
|
+
entries: null
|
|
1777
|
+
});
|
|
1778
|
+
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
|
|
1779
|
+
const tar = tarCommand(box.workdir);
|
|
1780
|
+
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
1781
|
+
const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
|
|
1782
|
+
const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
|
|
1783
|
+
const rawBytes = Number(raw.trim());
|
|
1784
|
+
const gzBytes = Number(size.trim());
|
|
1785
|
+
if (sized.exitCode !== 0 || !(rawBytes > 0) || !(gzBytes > 0)) return none("error", `couldn't archive the workspace: ${(sized.stderr || sized.stdout).trim()}`);
|
|
1786
|
+
const without = dirs.length ? dirs.join(", ") : "its dependency and cache directories";
|
|
1787
|
+
const rawLimit = compress ? limits.raw : limits.dir;
|
|
1788
|
+
if (rawBytes > rawLimit) return none("size", `the workspace is ${formatBytes(rawBytes)} without ${without}, over the ${formatBytes(rawLimit)} cap${compress ? "" : ` (compressed, the cap is ${formatBytes(limits.tar)})`}`, dirs);
|
|
1789
|
+
if (compress && gzBytes > limits.tar) return none("size", `the workspace is ${formatBytes(gzBytes)} compressed without ${without}, over the ${formatBytes(limits.tar)} cap`, dirs);
|
|
1790
|
+
const r = await box.execOwn(`${NEEDS_TAR}; echo ${OPAQUE_OUTPUT}; ${tar} | { ${GZIP}; } | base64`, { signal });
|
|
1791
|
+
const encoded = r.stdout.startsWith(`--opaque-output--\n`) ? r.stdout.slice(OPAQUE_OUTPUT.length + 1) : "";
|
|
1792
|
+
const fetched = Buffer.from(encoded, "base64");
|
|
1793
|
+
if (r.exitCode !== 0 || fetched.length === 0) return none("error", `couldn't archive the workspace: ${r.stderr.trim() || "no output"}`);
|
|
1794
|
+
const clean = cleanArchive(fetched[0] === 31 && fetched[1] === 139 ? gunzipSync(fetched, { maxOutputLength: rawLimit + 2 ** 20 }) : fetched);
|
|
1795
|
+
const all = [.../* @__PURE__ */ new Set([...dirs, ...clean.excluded])].sort();
|
|
1796
|
+
return {
|
|
1797
|
+
taken: {
|
|
1798
|
+
bytes: 0,
|
|
1799
|
+
files: clean.entries.filter((e) => e.type !== "dir").length,
|
|
1800
|
+
excluded: listed(all),
|
|
1801
|
+
masked: listed(clean.masked),
|
|
1802
|
+
dropped: listed(clean.dropped),
|
|
1803
|
+
skipped: null
|
|
1804
|
+
},
|
|
1805
|
+
entries: clean.entries
|
|
1806
|
+
};
|
|
1807
|
+
}
|
|
1808
|
+
/** Where an attempt's snapshots wait, beside its staging record, until the run is assembled. */
|
|
1809
|
+
const snapshotSidecar = (stagingDir, key) => join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".snapshots"));
|
|
1810
|
+
/**
|
|
1811
|
+
* Takes snapshot `name` of the workspace into the attempt's sidecar and adds it to the record;
|
|
1812
|
+
* never fails the attempt. Adds nothing when the attempt's signal stopped it.
|
|
1813
|
+
*/
|
|
1814
|
+
async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SNAPSHOT_TIMEOUT_MS) {
|
|
1815
|
+
const { compress } = a.spec.snapshot;
|
|
1816
|
+
const form = compress ? "tar.gz" : "dir";
|
|
1817
|
+
const dir = a.snapshotDir;
|
|
1818
|
+
const target = join(dir, compress ? `${name}.tar.gz` : name);
|
|
1819
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
1820
|
+
const add = (taken) => {
|
|
1821
|
+
a.record.snapshots ??= [];
|
|
1822
|
+
a.record.snapshots.push({
|
|
1823
|
+
name,
|
|
1824
|
+
form,
|
|
1825
|
+
path: null,
|
|
1826
|
+
...taken
|
|
1827
|
+
});
|
|
1828
|
+
};
|
|
1829
|
+
const failed = (message) => {
|
|
1830
|
+
debugLog(phase, `no ${name} snapshot: ${message}`);
|
|
1831
|
+
add({
|
|
1832
|
+
bytes: 0,
|
|
1833
|
+
files: 0,
|
|
1834
|
+
excluded: [],
|
|
1835
|
+
masked: [],
|
|
1836
|
+
dropped: [],
|
|
1837
|
+
skipped: {
|
|
1838
|
+
cause: "error",
|
|
1839
|
+
message: `couldn't archive the workspace: ${message}`
|
|
1840
|
+
}
|
|
1841
|
+
});
|
|
1842
|
+
};
|
|
1843
|
+
try {
|
|
1844
|
+
const { taken, entries } = await archiveWorkspace(box, AbortSignal.any([a.signal, timeout]), compress, limits);
|
|
1845
|
+
if (!entries) {
|
|
1846
|
+
debugLog(phase, `no ${name} snapshot: ${taken.skipped?.message}`);
|
|
1847
|
+
add(taken);
|
|
1848
|
+
return;
|
|
1849
|
+
}
|
|
1850
|
+
await rm(target, {
|
|
1851
|
+
recursive: true,
|
|
1852
|
+
force: true
|
|
1853
|
+
});
|
|
1854
|
+
await mkdir(dir, { recursive: true });
|
|
1855
|
+
if (compress) {
|
|
1856
|
+
const archive = gzipSync(writeTar(entries));
|
|
1857
|
+
if (archive.length > limits.tar) {
|
|
1858
|
+
const message = `the workspace is ${formatBytes(archive.length)} compressed, over the ${formatBytes(limits.tar)} cap`;
|
|
1859
|
+
debugLog(phase, `no ${name} snapshot: ${message}`);
|
|
1860
|
+
add({
|
|
1861
|
+
...taken,
|
|
1862
|
+
files: 0,
|
|
1863
|
+
skipped: {
|
|
1864
|
+
cause: "size",
|
|
1865
|
+
message
|
|
1866
|
+
}
|
|
1867
|
+
});
|
|
1868
|
+
return;
|
|
1869
|
+
}
|
|
1870
|
+
await writeFile(target, archive);
|
|
1871
|
+
taken.bytes = archive.length;
|
|
1872
|
+
} else {
|
|
1873
|
+
const skipped = await extractTar(entries, target);
|
|
1874
|
+
taken.bytes = entries.reduce((n, e) => n + (e.type === "file" ? e.data.length : 0), 0);
|
|
1875
|
+
taken.dropped = listed([...taken.dropped, ...skipped]);
|
|
1876
|
+
}
|
|
1877
|
+
const masked = taken.masked.length ? ` · masked ${taken.masked.length}` : "";
|
|
1878
|
+
const left = taken.excluded.length ? ` · left out ${taken.excluded.join(", ")}` : "";
|
|
1879
|
+
debugLog(phase, `${name} snapshot: ${taken.files} files, ${formatBytes(taken.bytes)} (${form})${masked}${left}`);
|
|
1880
|
+
add(taken);
|
|
1881
|
+
} catch (error) {
|
|
1882
|
+
await rm(target, {
|
|
1883
|
+
recursive: true,
|
|
1884
|
+
force: true
|
|
1885
|
+
}).catch(() => {});
|
|
1886
|
+
if (a.signal.aborted) return;
|
|
1887
|
+
if (timeout.aborted) return failed(`it took over ${Math.round(timeoutMs / 1e3)}s`);
|
|
1888
|
+
failed(error instanceof Error ? error.message : String(error));
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
//#endregion
|
|
1504
1892
|
//#region src/attempt/turns.ts
|
|
1505
1893
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1506
1894
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
@@ -1514,7 +1902,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1514
1902
|
const cutoff = new Promise((resolve) => {
|
|
1515
1903
|
onAbort = () => {
|
|
1516
1904
|
timer = setTimeout(() => {
|
|
1517
|
-
const message = `The agent didn't stop within ${grace / 1e3}s of being
|
|
1905
|
+
const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
|
|
1518
1906
|
const at = Date.now() - Date.parse(startedAt);
|
|
1519
1907
|
resolve({
|
|
1520
1908
|
text: "",
|
|
@@ -1610,7 +1998,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
1610
1998
|
kind: "turn-start",
|
|
1611
1999
|
turn: turnNumber
|
|
1612
2000
|
});
|
|
1613
|
-
debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
|
|
2001
|
+
debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(redactSecrets(prompt), 160))}`);
|
|
1614
2002
|
debugTrace("harness", "prompt", {
|
|
1615
2003
|
turn: turnNumber,
|
|
1616
2004
|
prompt
|
|
@@ -1671,10 +2059,9 @@ function turnSender(a, takeContext, checks) {
|
|
|
1671
2059
|
ms: result.durationMs,
|
|
1672
2060
|
tools: count("tool", "web-search", "web-fetch"),
|
|
1673
2061
|
fileEdits: count("file-edit"),
|
|
1674
|
-
...failure?.type === "error" ? { error: redactSecrets(
|
|
2062
|
+
...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
|
|
1675
2063
|
});
|
|
1676
|
-
|
|
1677
|
-
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
2064
|
+
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
1678
2065
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
1679
2066
|
if (timedOut) {
|
|
1680
2067
|
const error = new InfraError(timedOut);
|
|
@@ -1684,6 +2071,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
1684
2071
|
a.signal.throwIfAborted();
|
|
1685
2072
|
const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
|
|
1686
2073
|
a.record.diff = diff;
|
|
2074
|
+
if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
|
|
1687
2075
|
const data = {
|
|
1688
2076
|
prompt,
|
|
1689
2077
|
...result,
|
|
@@ -1727,7 +2115,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1727
2115
|
} finally {
|
|
1728
2116
|
clock?.resume();
|
|
1729
2117
|
}
|
|
1730
|
-
});
|
|
2118
|
+
}, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
|
|
1731
2119
|
const destroy = () => void a.sandbox?.destroy();
|
|
1732
2120
|
testSignal.addEventListener("abort", destroy, { once: true });
|
|
1733
2121
|
const takeContext = contextTaker(a);
|
|
@@ -1768,13 +2156,19 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1768
2156
|
throw a.startFailure ? a.startFailure.error : error;
|
|
1769
2157
|
});
|
|
1770
2158
|
a.signal.throwIfAborted();
|
|
1771
|
-
const missed = await unawaited() ??
|
|
2159
|
+
const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
|
|
1772
2160
|
if (a.startFailure) throw a.startFailure.error;
|
|
1773
2161
|
if (missed) throw missed;
|
|
1774
|
-
const
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
2162
|
+
const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
|
|
2163
|
+
const [only, ...more] = judgeErrors;
|
|
2164
|
+
if (only) {
|
|
2165
|
+
const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
|
|
2166
|
+
a.judgeThrown.add(message);
|
|
2167
|
+
throw new Error(message);
|
|
2168
|
+
}
|
|
2169
|
+
} catch (thrown) {
|
|
2170
|
+
const error = attemptError(a, thrown, assertions());
|
|
2171
|
+
recordFailure(a, error, timedOut(), assertions());
|
|
1778
2172
|
failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
|
|
1779
2173
|
} finally {
|
|
1780
2174
|
if (!a.signal.aborted) await unawaited().catch(() => {});
|
|
@@ -1790,13 +2184,14 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1790
2184
|
if (a.session) debug.log("teardown", "harness session closed");
|
|
1791
2185
|
recordEgress(a, sandbox);
|
|
1792
2186
|
oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
|
|
2187
|
+
if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
|
|
1793
2188
|
await sandbox?.destroy();
|
|
1794
2189
|
if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
|
|
1795
2190
|
const { record } = a;
|
|
1796
2191
|
const checked = assertionRecords(assertions());
|
|
1797
2192
|
if (checked.length) record.assertions = checked;
|
|
1798
2193
|
for (const x of checked) {
|
|
1799
|
-
const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
|
|
2194
|
+
const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
|
|
1800
2195
|
debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
|
|
1801
2196
|
}
|
|
1802
2197
|
const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
|
|
@@ -1811,6 +2206,30 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1811
2206
|
if (failure) throw failure.error;
|
|
1812
2207
|
}
|
|
1813
2208
|
//#endregion
|
|
2209
|
+
//#region src/best-of.ts
|
|
2210
|
+
/** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
|
|
2211
|
+
function sideOf({ state, meta }) {
|
|
2212
|
+
const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
|
|
2213
|
+
return outcome === "pass" || outcome === "fail" ? outcome : null;
|
|
2214
|
+
}
|
|
2215
|
+
/** Whether a best-of cell still needs another attempt after these. */
|
|
2216
|
+
function undecided(sides, bestOf) {
|
|
2217
|
+
const majority = majorityOf(bestOf);
|
|
2218
|
+
const pass = sides.filter((s) => s === "pass").length;
|
|
2219
|
+
const fail = sides.filter((s) => s === "fail").length;
|
|
2220
|
+
return pass < majority && fail < majority;
|
|
2221
|
+
}
|
|
2222
|
+
/** One best-of cell: the `ready` and `settled` hooks of its attempt `n`, from 1 to `bestOf`. */
|
|
2223
|
+
function bestOfCell(bestOf) {
|
|
2224
|
+
const sides = [];
|
|
2225
|
+
const settle = [];
|
|
2226
|
+
for (let n = 1; n <= bestOf; n++) sides.push(new Promise((resolve) => settle.push(resolve)));
|
|
2227
|
+
return (n) => ({
|
|
2228
|
+
...n > majorityOf(bestOf) ? { ready: async () => undecided(await Promise.all(sides.slice(0, n - 1)), bestOf) } : {},
|
|
2229
|
+
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2230
|
+
});
|
|
2231
|
+
}
|
|
2232
|
+
//#endregion
|
|
1814
2233
|
//#region src/contract/declaration.ts
|
|
1815
2234
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
1816
2235
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
@@ -1818,39 +2237,45 @@ const declaredModeSchema = z.enum([
|
|
|
1818
2237
|
"run",
|
|
1819
2238
|
"skip",
|
|
1820
2239
|
"only"
|
|
1821
|
-
])
|
|
2240
|
+
]);
|
|
1822
2241
|
const declaredFamilySchema = z.strictObject({
|
|
1823
|
-
name: z.string()
|
|
1824
|
-
case: z.number().int().min(0)
|
|
1825
|
-
cases: z.number().int().min(2)
|
|
1826
|
-
})
|
|
2242
|
+
name: z.string(),
|
|
2243
|
+
case: z.number().int().min(0),
|
|
2244
|
+
cases: z.number().int().min(2)
|
|
2245
|
+
});
|
|
1827
2246
|
const declaredTodoSchema = z.strictObject({
|
|
1828
2247
|
type: z.literal("todo"),
|
|
1829
|
-
name: z.string()
|
|
1830
|
-
})
|
|
2248
|
+
name: z.string()
|
|
2249
|
+
});
|
|
2250
|
+
const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
|
|
2251
|
+
const declaredSourceSchema = z.strictObject({
|
|
2252
|
+
body: hash(),
|
|
2253
|
+
case: hash().optional()
|
|
2254
|
+
});
|
|
1831
2255
|
const declaredEvalSchema = z.strictObject({
|
|
1832
2256
|
type: z.literal("eval"),
|
|
1833
|
-
name: z.string()
|
|
2257
|
+
name: z.string(),
|
|
1834
2258
|
mode: declaredModeSchema,
|
|
1835
|
-
options: evalOptionsDataSchema
|
|
1836
|
-
body: z.string()
|
|
1837
|
-
|
|
1838
|
-
|
|
2259
|
+
options: evalOptionsDataSchema,
|
|
2260
|
+
body: z.string(),
|
|
2261
|
+
source: declaredSourceSchema,
|
|
2262
|
+
family: declaredFamilySchema.optional()
|
|
2263
|
+
});
|
|
1839
2264
|
const declaredGroupSchema = z.strictObject({
|
|
1840
2265
|
type: z.literal("group"),
|
|
1841
|
-
name: z.string()
|
|
2266
|
+
name: z.string(),
|
|
1842
2267
|
mode: declaredModeSchema,
|
|
1843
|
-
options: evalOptionsDataSchema
|
|
2268
|
+
options: evalOptionsDataSchema,
|
|
1844
2269
|
get children() {
|
|
1845
2270
|
return z.array(declaredNodeSchema);
|
|
1846
2271
|
}
|
|
1847
|
-
})
|
|
2272
|
+
});
|
|
1848
2273
|
const declaredNodeSchema = z.discriminatedUnion("type", [
|
|
1849
2274
|
declaredEvalSchema,
|
|
1850
2275
|
declaredGroupSchema,
|
|
1851
2276
|
declaredTodoSchema
|
|
1852
|
-
])
|
|
1853
|
-
const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) })
|
|
2277
|
+
]);
|
|
2278
|
+
const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) });
|
|
1854
2279
|
const NODE_SCHEMAS = {
|
|
1855
2280
|
eval: declaredEvalSchema,
|
|
1856
2281
|
group: declaredGroupSchema,
|
|
@@ -1866,12 +2291,14 @@ const declarationProblem = (children) => dataProblem(declarationTreeSchema, { ch
|
|
|
1866
2291
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
1867
2292
|
const GRACE_MS = 5e3;
|
|
1868
2293
|
function configLevel(config) {
|
|
2294
|
+
const { runs, bestOf } = config.runs !== void 0 ? runsOf(config.runs) : { runs: void 0 };
|
|
1869
2295
|
return {
|
|
1870
2296
|
harness: config.harness,
|
|
1871
2297
|
sandbox: config.sandbox,
|
|
1872
2298
|
environment: config.environment,
|
|
1873
2299
|
compare: void 0,
|
|
1874
|
-
runs
|
|
2300
|
+
runs,
|
|
2301
|
+
bestOf,
|
|
1875
2302
|
gate: config.gate,
|
|
1876
2303
|
env: config.env,
|
|
1877
2304
|
egress: config.egress !== void 0 ? { inherited: config.egress } : {},
|
|
@@ -1880,19 +2307,17 @@ function configLevel(config) {
|
|
|
1880
2307
|
question: config.question,
|
|
1881
2308
|
questions: [],
|
|
1882
2309
|
timeout: void 0,
|
|
2310
|
+
snapshot: config.snapshot,
|
|
1883
2311
|
tags: [],
|
|
2312
|
+
revision: void 0,
|
|
1884
2313
|
pin: [],
|
|
1885
2314
|
sources: {}
|
|
1886
2315
|
};
|
|
1887
2316
|
}
|
|
1888
|
-
/**
|
|
1889
|
-
* The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
|
|
1890
|
-
* `order`). Throws a UsageError for options the plan refuses.
|
|
1891
|
-
*/
|
|
2317
|
+
/** The tests an eval registers. Throws a UsageError for options the plan refuses. */
|
|
1892
2318
|
function planEval(declaration, ctx) {
|
|
1893
|
-
const { name,
|
|
1894
|
-
const
|
|
1895
|
-
const where = `evaluate("${[...groups, name].join(" › ")}")`;
|
|
2319
|
+
const { name, groups: enclosing, options: own } = declaration;
|
|
2320
|
+
const where = `evaluate("${[...enclosing.map((g) => g.name), name].join(" › ")}")`;
|
|
1896
2321
|
checkOptions(own, where);
|
|
1897
2322
|
const levels = [...enclosing.map((g) => ({
|
|
1898
2323
|
options: g.options,
|
|
@@ -1904,25 +2329,54 @@ function planEval(declaration, ctx) {
|
|
|
1904
2329
|
const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
|
|
1905
2330
|
profile: ctx.profile,
|
|
1906
2331
|
runs: ctx.runs,
|
|
1907
|
-
skipped: mode === "skip" || mode === "todo"
|
|
2332
|
+
skipped: declaration.mode === "skip" || declaration.mode === "todo",
|
|
2333
|
+
pooled: declaration.family !== void 0
|
|
1908
2334
|
});
|
|
2335
|
+
const { harnesses } = resolvedEval;
|
|
2336
|
+
if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
|
|
2337
|
+
tests: null,
|
|
2338
|
+
harnesses
|
|
2339
|
+
};
|
|
2340
|
+
return {
|
|
2341
|
+
tests: planTests(declaration, ctx, where, resolvedEval),
|
|
2342
|
+
harnesses
|
|
2343
|
+
};
|
|
2344
|
+
}
|
|
2345
|
+
function planTests(declaration, ctx, where, resolvedEval) {
|
|
2346
|
+
const { name, mode, groups: enclosing } = declaration;
|
|
2347
|
+
const groups = enclosing.map((g) => g.name);
|
|
1909
2348
|
const { resolved } = resolvedEval;
|
|
1910
|
-
|
|
2349
|
+
const source = {
|
|
2350
|
+
...declaration.source,
|
|
2351
|
+
options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
|
|
2352
|
+
};
|
|
1911
2353
|
const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
|
|
1912
2354
|
const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
|
|
1913
2355
|
const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
|
|
1914
2356
|
const routes = /* @__PURE__ */ new Map();
|
|
1915
2357
|
const gatewayNotes = [];
|
|
2358
|
+
let vertex;
|
|
1916
2359
|
const routed = resolvedEval.skip ? resolved.harness : resolved.harness.map((harness) => {
|
|
1917
2360
|
const route = resolveGateway(gateways, harness, where);
|
|
1918
2361
|
const hid = harnessId(harness);
|
|
1919
2362
|
if (gatewaySet || harness.gateway !== void 0) {
|
|
1920
2363
|
routes.set(hid, route);
|
|
1921
|
-
gatewayNotes.push(`gateway: ${hid} → ${route.name}`);
|
|
2364
|
+
gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
|
|
1922
2365
|
}
|
|
1923
2366
|
if (route.name === "direct") return harness;
|
|
1924
|
-
|
|
1925
|
-
|
|
2367
|
+
if (harness.kind !== "fake") try {
|
|
2368
|
+
if (route.name === "vercel") {
|
|
2369
|
+
const problem = vercelCredentialProblem();
|
|
2370
|
+
if (problem) throw new UsageError(problem);
|
|
2371
|
+
}
|
|
2372
|
+
if (route.name === "vertex") {
|
|
2373
|
+
const entry = gateways.find((g) => g.kind === "vertex");
|
|
2374
|
+
vertex ??= vertexSettings(entry?.kind === "vertex" ? entry : {});
|
|
2375
|
+
}
|
|
2376
|
+
} catch (error) {
|
|
2377
|
+
if (error instanceof UsageError) throw new UsageError(`${where} on ${hid}: ${error.message}`);
|
|
2378
|
+
throw error;
|
|
2379
|
+
}
|
|
1926
2380
|
return {
|
|
1927
2381
|
...harness,
|
|
1928
2382
|
gateway: route.name
|
|
@@ -1931,9 +2385,16 @@ function planEval(declaration, ctx) {
|
|
|
1931
2385
|
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
1932
2386
|
...resolved,
|
|
1933
2387
|
harness: routed
|
|
1934
|
-
});
|
|
2388
|
+
}, vertex);
|
|
2389
|
+
let judge;
|
|
1935
2390
|
if (plannedAttempts.length) try {
|
|
1936
|
-
resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
|
|
2391
|
+
const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
|
|
2392
|
+
const { model } = resolved.judge;
|
|
2393
|
+
if (typeof model === "string") judge = {
|
|
2394
|
+
model,
|
|
2395
|
+
...gateway ? { gateway } : {}
|
|
2396
|
+
};
|
|
2397
|
+
if (gatewaySet && fellBackToDirect(gateway, resolved.judge.gateway, { gateways })) gatewayNotes.push(`gateway: judge ${model} → direct (fallback)`);
|
|
1937
2398
|
} catch (error) {
|
|
1938
2399
|
if (error instanceof UsageError) throw new UsageError(`${where}: ${error.message}`);
|
|
1939
2400
|
throw error;
|
|
@@ -1941,6 +2402,8 @@ function planEval(declaration, ctx) {
|
|
|
1941
2402
|
if (ctx.config.secrets.unbrokered === "error") {
|
|
1942
2403
|
const [first] = unbrokered(resolved.env, plannedAttempts);
|
|
1943
2404
|
if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
|
|
2405
|
+
const [plain] = plainHeaders(plannedAttempts);
|
|
2406
|
+
if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
|
|
1944
2407
|
}
|
|
1945
2408
|
const pluginNotes = [];
|
|
1946
2409
|
const pluginLabels = /* @__PURE__ */ new Map();
|
|
@@ -1966,6 +2429,7 @@ function planEval(declaration, ctx) {
|
|
|
1966
2429
|
configFile: ctx.config.file,
|
|
1967
2430
|
file: path
|
|
1968
2431
|
});
|
|
2432
|
+
if (gatewayNotes.length && openRouterFallbacks(gateways)) gatewayNotes.push("gateway: openrouter() fallbacks on: reeval's own calls may fail over to another upstream provider; the harnesses' calls take OpenRouter's own routing");
|
|
1969
2433
|
plan.notes.push(...gatewayNotes, ...pluginNotes);
|
|
1970
2434
|
const setByOf = (planned) => ({
|
|
1971
2435
|
sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
|
|
@@ -1978,6 +2442,8 @@ function planEval(declaration, ctx) {
|
|
|
1978
2442
|
name,
|
|
1979
2443
|
groups,
|
|
1980
2444
|
tags: resolved.tags,
|
|
2445
|
+
...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
|
|
2446
|
+
source,
|
|
1981
2447
|
cell: first.cell,
|
|
1982
2448
|
n: 1,
|
|
1983
2449
|
label: "",
|
|
@@ -2014,6 +2480,8 @@ function planEval(declaration, ctx) {
|
|
|
2014
2480
|
name,
|
|
2015
2481
|
groups,
|
|
2016
2482
|
tags: resolved.tags,
|
|
2483
|
+
...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
|
|
2484
|
+
source,
|
|
2017
2485
|
cell: planned.cell,
|
|
2018
2486
|
n: planned.n,
|
|
2019
2487
|
label: planned.label,
|
|
@@ -2022,6 +2490,7 @@ function planEval(declaration, ctx) {
|
|
|
2022
2490
|
setBy,
|
|
2023
2491
|
gate: planned.gate,
|
|
2024
2492
|
...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
|
|
2493
|
+
...planned.bestOf !== void 0 ? { bestOf: planned.bestOf } : {},
|
|
2025
2494
|
brokered: brokeredIn(resolved.env, planned),
|
|
2026
2495
|
secretVars: secretVarsIn(resolved.env, planned),
|
|
2027
2496
|
egress: planned.egress,
|
|
@@ -2035,13 +2504,14 @@ function planEval(declaration, ctx) {
|
|
|
2035
2504
|
} : plan,
|
|
2036
2505
|
...skip ? { skip } : {},
|
|
2037
2506
|
...family ? { family } : {},
|
|
2038
|
-
...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {}
|
|
2507
|
+
...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {},
|
|
2508
|
+
...judge ? { judge } : {}
|
|
2039
2509
|
};
|
|
2040
2510
|
return {
|
|
2041
2511
|
name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
|
|
2042
2512
|
mode: skip ? "skip" : mode,
|
|
2043
2513
|
attempt,
|
|
2044
|
-
timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
|
|
2514
|
+
timeout: skip ? void 0 : Math.min(timeoutMs + setupAllowance(planned) + GRACE_MS, MAX_TIMER_MS),
|
|
2045
2515
|
gate: skip ? void 0 : {
|
|
2046
2516
|
limits: attemptLimits(planned, ctx.concurrency),
|
|
2047
2517
|
rank: {
|
|
@@ -2060,6 +2530,7 @@ function planEval(declaration, ctx) {
|
|
|
2060
2530
|
env: resolved.env,
|
|
2061
2531
|
judge: resolved.judge,
|
|
2062
2532
|
capture: resolved.capture,
|
|
2533
|
+
snapshot: resolved.snapshot,
|
|
2063
2534
|
sources: {
|
|
2064
2535
|
...resolved.sources,
|
|
2065
2536
|
sandbox: setBy.sandbox
|
|
@@ -2221,16 +2692,54 @@ function registerTodo(name) {
|
|
|
2221
2692
|
const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
|
|
2222
2693
|
if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
|
|
2223
2694
|
}
|
|
2224
|
-
/**
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2695
|
+
/**
|
|
2696
|
+
* The evals declared so far, by id, with their name paths and whether each is a family's case,
|
|
2697
|
+
* so a second id is refused.
|
|
2698
|
+
*/
|
|
2699
|
+
const declared = /* @__PURE__ */ new Map();
|
|
2700
|
+
/** The pooled families declared so far, by id: their name path, length and the cases seen. */
|
|
2701
|
+
const families = /* @__PURE__ */ new Map();
|
|
2702
|
+
/** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
|
|
2703
|
+
const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
|
|
2704
|
+
/** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
|
|
2705
|
+
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
|
|
2706
|
+
function checkUnique(where, groups, name, hint, inFamily) {
|
|
2707
|
+
const path = [...groups, name];
|
|
2708
|
+
const id = evalId(collectingFile() ?? "", path);
|
|
2709
|
+
const earlier = declared.get(id);
|
|
2710
|
+
if (earlier) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}${separatorHint(path, earlier.path)}`);
|
|
2711
|
+
const family = inFamily ? void 0 : families.get(id);
|
|
2712
|
+
if (family) throw namedLikeFamily(path, family.path);
|
|
2713
|
+
declared.set(id, {
|
|
2714
|
+
path,
|
|
2715
|
+
inFamily
|
|
2716
|
+
});
|
|
2717
|
+
}
|
|
2718
|
+
/**
|
|
2719
|
+
* A family's cases pool by its id, the groups and the template, so each case may be declared
|
|
2720
|
+
* once and every case must agree on the list's length.
|
|
2721
|
+
*/
|
|
2722
|
+
function checkFamily(groups, family) {
|
|
2723
|
+
const path = [...groups, family.name];
|
|
2724
|
+
const id = evalId(collectingFile() ?? "", path);
|
|
2725
|
+
const earlier = families.get(id);
|
|
2726
|
+
if (!earlier) {
|
|
2727
|
+
const plain = declared.get(id);
|
|
2728
|
+
if (plain && !plain.inFamily) throw namedLikeFamily(plain.path, path);
|
|
2729
|
+
families.set(id, {
|
|
2730
|
+
path,
|
|
2731
|
+
cases: family.cases,
|
|
2732
|
+
seen: /* @__PURE__ */ new Set([family.case])
|
|
2733
|
+
});
|
|
2734
|
+
return;
|
|
2735
|
+
}
|
|
2736
|
+
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
|
|
2737
|
+
earlier.seen.add(family.case);
|
|
2230
2738
|
}
|
|
2231
|
-
function register(mode, name, own, fn, hint = "", family) {
|
|
2739
|
+
function register(mode, name, own, fn, source, hint = "", family) {
|
|
2232
2740
|
const groups = groupStack.map((g) => g.name);
|
|
2233
|
-
|
|
2741
|
+
if (family) checkFamily(groups, family);
|
|
2742
|
+
checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
|
|
2234
2743
|
const body = `b${bodies.size + 1}`;
|
|
2235
2744
|
bodies.set(body, fn);
|
|
2236
2745
|
declare({
|
|
@@ -2239,9 +2748,10 @@ function register(mode, name, own, fn, hint = "", family) {
|
|
|
2239
2748
|
mode,
|
|
2240
2749
|
options: data(own),
|
|
2241
2750
|
body,
|
|
2751
|
+
source,
|
|
2242
2752
|
...family ? { family } : {}
|
|
2243
2753
|
});
|
|
2244
|
-
const tests = planEval({
|
|
2754
|
+
const { tests, harnesses } = planEval({
|
|
2245
2755
|
name,
|
|
2246
2756
|
mode,
|
|
2247
2757
|
groups: groupStack.map(({ name, options }) => ({
|
|
@@ -2249,22 +2759,38 @@ function register(mode, name, own, fn, hint = "", family) {
|
|
|
2249
2759
|
options
|
|
2250
2760
|
})),
|
|
2251
2761
|
options: own,
|
|
2762
|
+
source,
|
|
2252
2763
|
...family ? { family } : {}
|
|
2253
2764
|
}, {
|
|
2254
2765
|
...getRunContext(),
|
|
2255
2766
|
file: collectingFile(),
|
|
2256
2767
|
order: registered
|
|
2257
2768
|
});
|
|
2769
|
+
declareHarnesses(harnesses);
|
|
2258
2770
|
if (!tests) return;
|
|
2259
2771
|
registered++;
|
|
2260
|
-
|
|
2772
|
+
const bestOfCells = /* @__PURE__ */ new Map();
|
|
2773
|
+
for (const { name: testName, spec, ...options } of tests) {
|
|
2774
|
+
const { cell, n, bestOf } = options.attempt;
|
|
2775
|
+
let hooks = {};
|
|
2776
|
+
if (bestOf !== void 0) {
|
|
2777
|
+
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
2778
|
+
bestOfCells.set(cell.key, of);
|
|
2779
|
+
hooks = of(n);
|
|
2780
|
+
}
|
|
2781
|
+
registerTest(testName, {
|
|
2782
|
+
...options,
|
|
2783
|
+
...hooks
|
|
2784
|
+
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
2785
|
+
}
|
|
2261
2786
|
}
|
|
2262
2787
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2263
2788
|
let registered = 0;
|
|
2264
|
-
const defineEval = (mode) => (name, a, b) => {
|
|
2789
|
+
const defineEval = (mode, declared) => (name, a, b) => {
|
|
2265
2790
|
checkOnly(mode, "evaluate");
|
|
2266
2791
|
const [own, fn] = splitArgs("evaluate", name, a, b);
|
|
2267
|
-
|
|
2792
|
+
const where = `evaluate("${name}")`;
|
|
2793
|
+
register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
|
|
2268
2794
|
};
|
|
2269
2795
|
/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
|
|
2270
2796
|
function formatName(template, item, index) {
|
|
@@ -2290,13 +2816,19 @@ const evalEach = (cases, options) => (name, a, b) => {
|
|
|
2290
2816
|
const own = withPaths(written, `evaluate.each("${name}")`);
|
|
2291
2817
|
const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
|
|
2292
2818
|
const pooled = pool && cases.length >= 2;
|
|
2819
|
+
const file = collectingFile();
|
|
2820
|
+
const body = bodyHash(fn, `evaluate.each("${name}")`, file);
|
|
2293
2821
|
cases.forEach((item, i) => {
|
|
2294
2822
|
const family = pooled ? {
|
|
2295
2823
|
name,
|
|
2296
2824
|
case: i,
|
|
2297
2825
|
cases: cases.length
|
|
2298
2826
|
} : void 0;
|
|
2299
|
-
|
|
2827
|
+
const source = {
|
|
2828
|
+
body,
|
|
2829
|
+
case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
|
|
2830
|
+
};
|
|
2831
|
+
register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
|
|
2300
2832
|
});
|
|
2301
2833
|
};
|
|
2302
2834
|
const evaluate = Object.assign(defineEval("run"), {
|
|
@@ -2326,8 +2858,8 @@ function declareNodes(nodes, bodyFor, base) {
|
|
|
2326
2858
|
} else if (node.family) {
|
|
2327
2859
|
const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
|
|
2328
2860
|
checkOnly(node.mode, "evaluate");
|
|
2329
|
-
register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
|
|
2330
|
-
} else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
2861
|
+
register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
|
|
2862
|
+
} else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
2331
2863
|
}
|
|
2332
2864
|
/**
|
|
2333
2865
|
* Options from their data forms, with host paths absolute against `base`: environments become
|
|
@@ -2364,4 +2896,4 @@ function optionsFrom(written, where, base) {
|
|
|
2364
2896
|
//#endregion
|
|
2365
2897
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2366
2898
|
|
|
2367
|
-
//# sourceMappingURL=evaluate-
|
|
2899
|
+
//# sourceMappingURL=evaluate-CUtgyr43.js.map
|