reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,27 +1,31 @@
|
|
|
1
1
|
import { t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
|
-
import { r as withLease } from "./client-
|
|
2
|
+
import { r as withLease } from "./client-Ct6s6OnF.js";
|
|
3
3
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
|
|
4
|
-
import { n as dataProblem, p as
|
|
5
|
-
import {
|
|
4
|
+
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-BOFgzFDg.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-D-CP6mMd.js";
|
|
6
7
|
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
-
import { A as
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
8
|
+
import { $ as isDirect, A as harnessId, At as badJudgeArtifacts, B as sandboxSetBy, D as checkOptions, Dt as judgeProblem, G as resolveStateOptions, H as unbrokered, J as allowedHosts, L as refuseIllegalCompare, M as matchesTags, Mt as filePartsOf, N as plainHeaderError, Nt as resolveJudgeArtifacts, O as evalPlan, P as plainHeaders, R as sandboxId, S as resolveGateway, St as writePluginFiles, T as cascade, U as unbrokeredError, V as secretVarsIn, W as mergeJudge, Z as egressLabel, _ as fellBackToDirect, _t as expectedBlockedFor, b as resolveModelGateway, bt as refusedMcp, et as unenforceable, ft as agentMcp, ht as claudeMcpServers, jt as decideState, k as expand, kt as artifactRefsOf, mt as brokeringFor, nt as createFakeSandbox, v as gatewayScope, w as brokeredIn, xt as refusedPlugins, yt as layoutOf, z as sandboxProvider } from "./config-k6m7XbAz.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-CKdgEXN7.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
12
|
+
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
|
|
13
|
+
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
|
+
import { ft as shellQuote, t as MCP_TIMER } from "./src-D91Hd6hT.js";
|
|
13
15
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
14
|
-
import { i as
|
|
15
|
-
import {
|
|
16
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-jI0OWxlz.js";
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-d56Gg3J5.js";
|
|
17
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-Ba2QerOK.js";
|
|
17
18
|
import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
|
|
18
|
-
import {
|
|
19
|
+
import { r as openRouterFallbacks } from "./openrouter-models-CWRwLX-G.js";
|
|
20
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BM_3100g.js";
|
|
19
21
|
import { t as contextTaker } from "./context-Dgn48BqT.js";
|
|
20
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
22
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-DWSJpKHX.js";
|
|
21
23
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
22
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
24
|
+
import { i as preparedWorkspace } from "./workspace-BRxjfa8N.js";
|
|
25
|
+
import { t as patchPath } from "./patch-CquGAGX5.js";
|
|
26
|
+
import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
|
|
27
|
+
import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
|
|
28
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-Dje3rERU.js";
|
|
25
29
|
import { dirname, join } from "node:path";
|
|
26
30
|
import { parse, stringify } from "yaml";
|
|
27
31
|
import { z } from "zod";
|
|
@@ -30,7 +34,11 @@ import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
|
30
34
|
//#region src/attempt/assertions.ts
|
|
31
35
|
const MESSAGE_LIMIT = 1024;
|
|
32
36
|
const LABEL_LIMIT = 60;
|
|
33
|
-
|
|
37
|
+
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
38
|
+
const clipped = (text, limit) => {
|
|
39
|
+
const s = redactSecrets(text);
|
|
40
|
+
return s.length > limit ? `${s.slice(0, limit)}…` : s;
|
|
41
|
+
};
|
|
34
42
|
const text = (error) => error instanceof Error ? error.message : String(error);
|
|
35
43
|
const shown = (value) => {
|
|
36
44
|
if (typeof value === "string") return value;
|
|
@@ -44,17 +52,26 @@ const shown = (value) => {
|
|
|
44
52
|
const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
|
|
45
53
|
/**
|
|
46
54
|
* The records of an attempt's matcher calls, in call order. Without a label, one is
|
|
47
|
-
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3
|
|
55
|
+
* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`, skipping
|
|
56
|
+
* any label already recorded, so each record's label is unique (a given `build #2` after two
|
|
57
|
+
* `build`s is `build #2 #2`).
|
|
48
58
|
*/
|
|
49
59
|
function assertionRecords(raw) {
|
|
50
60
|
const seen = /* @__PURE__ */ new Map();
|
|
61
|
+
const used = /* @__PURE__ */ new Set();
|
|
51
62
|
return raw.map((r) => {
|
|
52
63
|
const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
53
64
|
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
54
|
-
|
|
65
|
+
let n = seen.get(base) ?? 0;
|
|
66
|
+
let label;
|
|
67
|
+
do {
|
|
68
|
+
n += 1;
|
|
69
|
+
label = n > 1 ? `${base} #${n}` : base;
|
|
70
|
+
} while (used.has(label));
|
|
55
71
|
seen.set(base, n);
|
|
72
|
+
used.add(label);
|
|
56
73
|
return {
|
|
57
|
-
label
|
|
74
|
+
label,
|
|
58
75
|
labelled: r.label !== void 0,
|
|
59
76
|
matcher: r.matcher,
|
|
60
77
|
status: r.status,
|
|
@@ -64,11 +81,22 @@ function assertionRecords(raw) {
|
|
|
64
81
|
};
|
|
65
82
|
});
|
|
66
83
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
84
|
+
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
85
|
+
/**
|
|
86
|
+
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
87
|
+
* that didn't pass, an `expect.soft` as what it threw, a hard check the eval caught with
|
|
88
|
+
* `CAUGHT_HINT` after its message.
|
|
89
|
+
*/
|
|
90
|
+
function checkFailure(raw) {
|
|
91
|
+
const failed = raw.find((r) => r.status !== "pass");
|
|
70
92
|
if (!failed) return void 0;
|
|
71
|
-
|
|
93
|
+
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
94
|
+
if (failed.soft) return error;
|
|
95
|
+
const caught = Object.setPrototypeOf(Object.assign(/* @__PURE__ */ new Error(), error), Object.getPrototypeOf(error));
|
|
96
|
+
caught.name = error.name;
|
|
97
|
+
caught.message = `${error.message} ${CAUGHT_HINT}`;
|
|
98
|
+
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
99
|
+
return caught;
|
|
72
100
|
}
|
|
73
101
|
//#endregion
|
|
74
102
|
//#region src/attempt/debug.ts
|
|
@@ -120,7 +148,7 @@ function traceEnvironment(debug, spec, env) {
|
|
|
120
148
|
const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
|
|
121
149
|
debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
|
|
122
150
|
for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
|
|
123
|
-
const brokering = brokeringFor(spec.harness);
|
|
151
|
+
const brokering = brokeringFor(spec.harness, spec.vertex);
|
|
124
152
|
if (brokering) {
|
|
125
153
|
const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
|
|
126
154
|
debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
|
|
@@ -406,10 +434,10 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
406
434
|
}
|
|
407
435
|
//#endregion
|
|
408
436
|
//#region src/harness/index.ts
|
|
409
|
-
async function createHarness(descriptor) {
|
|
437
|
+
async function createHarness(descriptor, vertex) {
|
|
410
438
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
411
|
-
const { createAgentHarness } = await import("./agent-
|
|
412
|
-
return createAgentHarness(descriptor);
|
|
439
|
+
const { createAgentHarness } = await import("./agent-Zgn-kiqq.js");
|
|
440
|
+
return createAgentHarness(descriptor, vertex);
|
|
413
441
|
}
|
|
414
442
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
415
443
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
@@ -428,9 +456,9 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
428
456
|
});
|
|
429
457
|
a.env = env;
|
|
430
458
|
record.environment = env.record;
|
|
431
|
-
const { secrets, brokered } = env.record;
|
|
459
|
+
const { vars, secrets, brokered } = env.record;
|
|
432
460
|
record.env = {
|
|
433
|
-
vars
|
|
461
|
+
vars,
|
|
434
462
|
secrets,
|
|
435
463
|
brokered
|
|
436
464
|
};
|
|
@@ -438,8 +466,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
438
466
|
if (debug) traceEnvironment(debug, spec, env);
|
|
439
467
|
record.image = null;
|
|
440
468
|
if (spec.sandbox.kind === "docker") {
|
|
441
|
-
const { preparedFor } = await import("./preflight-
|
|
442
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
469
|
+
const { preparedFor } = await import("./preflight-DTiVv0gY.js");
|
|
470
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DkOupEBl.js");
|
|
443
471
|
signal.throwIfAborted();
|
|
444
472
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
445
473
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -472,7 +500,7 @@ const CHARS_PER_TOKEN = 4;
|
|
|
472
500
|
const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
|
|
473
501
|
const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
|
|
474
502
|
const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
|
|
475
|
-
name:
|
|
503
|
+
name: patchPath(part) ?? "",
|
|
476
504
|
chars: part.length
|
|
477
505
|
})).filter((item) => item.name);
|
|
478
506
|
/** The state's parts by size, each with its biggest files where it has any. */
|
|
@@ -615,13 +643,21 @@ function checksShown(which, ran) {
|
|
|
615
643
|
}));
|
|
616
644
|
}
|
|
617
645
|
/**
|
|
646
|
+
* Where `order` goes among `calls`, which are in ascending order: calls finish in any order, but
|
|
647
|
+
* what they record stays in the order they were made.
|
|
648
|
+
*/
|
|
649
|
+
function placeFor(calls, order) {
|
|
650
|
+
const after = calls.findIndex((c) => c > order);
|
|
651
|
+
return after < 0 ? calls.length : after;
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
618
654
|
* Grades a turn (the last by default) or a value against a question, under
|
|
619
655
|
* the judge provider's concurrency lease. Every call's input is recorded, including calls that
|
|
620
656
|
* fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
|
|
621
657
|
* checks after it still run.
|
|
622
658
|
*/
|
|
623
659
|
function judgeCaller(a, readFile) {
|
|
624
|
-
const ask = async (question, { on, ...call } = {}) => {
|
|
660
|
+
const ask = async (order, question, { on, ...call } = {}) => {
|
|
625
661
|
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
626
662
|
const problem = judgeProblem(call, { at: "call" });
|
|
627
663
|
if (problem) throw new Error(problem);
|
|
@@ -632,8 +668,9 @@ function judgeCaller(a, readFile) {
|
|
|
632
668
|
const options = mergeJudge(a.spec.judge, call);
|
|
633
669
|
const { model } = options;
|
|
634
670
|
if (model === void 0) {
|
|
635
|
-
|
|
636
|
-
|
|
671
|
+
const message = noJudgeModel(getRunContext().config.file);
|
|
672
|
+
a.judgeThrown.add(message);
|
|
673
|
+
throw new Error(message);
|
|
637
674
|
}
|
|
638
675
|
const wanted = resolveStateOptions(options.state);
|
|
639
676
|
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
|
|
@@ -649,7 +686,7 @@ function judgeCaller(a, readFile) {
|
|
|
649
686
|
listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
|
|
650
687
|
});
|
|
651
688
|
const name = judgeModelName(model);
|
|
652
|
-
const state = stateOf(draft);
|
|
689
|
+
const state = redactValue(stateOf(draft));
|
|
653
690
|
const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
|
|
654
691
|
const refs = artifactRefsOf(resolved);
|
|
655
692
|
const decide = decideState(state, filePartsOf(resolved));
|
|
@@ -661,7 +698,9 @@ function judgeCaller(a, readFile) {
|
|
|
661
698
|
...refs.length ? { artifacts: refs } : {},
|
|
662
699
|
...dropped.length ? { dropped } : {}
|
|
663
700
|
};
|
|
664
|
-
a.
|
|
701
|
+
const place = placeFor(a.judgeInputCalls, order);
|
|
702
|
+
a.judgeInputCalls.splice(place, 0, order);
|
|
703
|
+
a.record.judgeInputs?.splice(place, 0, input);
|
|
665
704
|
const start = performance.now();
|
|
666
705
|
const ms = () => Math.round(performance.now() - start);
|
|
667
706
|
const atLeast = options.threshold ?? .8;
|
|
@@ -669,12 +708,14 @@ function judgeCaller(a, readFile) {
|
|
|
669
708
|
kind: "judge-start",
|
|
670
709
|
question
|
|
671
710
|
});
|
|
672
|
-
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
|
|
711
|
+
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
|
|
673
712
|
if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
|
|
674
713
|
debugTrace("judge", "prompt", input, "judge prompt");
|
|
675
714
|
const failed = (message, retries) => {
|
|
676
|
-
a.
|
|
677
|
-
|
|
715
|
+
a.judgeErrors.push({
|
|
716
|
+
call: order,
|
|
717
|
+
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`
|
|
718
|
+
});
|
|
678
719
|
input.error = message;
|
|
679
720
|
if (retries?.length) input.retries = retries;
|
|
680
721
|
debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
|
|
@@ -703,11 +744,13 @@ function judgeCaller(a, readFile) {
|
|
|
703
744
|
const provider = name.split("/")[0];
|
|
704
745
|
try {
|
|
705
746
|
const { config, profile } = getRunContext();
|
|
706
|
-
const
|
|
747
|
+
const scope = gatewayScope(config, profile);
|
|
748
|
+
const gateway = resolveModelGateway(model, judgeOptions.gateway, scope, "judge model");
|
|
707
749
|
const { gateway: _pin, ...unpinned } = judgeOptions;
|
|
750
|
+
const entry = gateway && scope.gateways.find((g) => g.kind === gateway);
|
|
708
751
|
const routed = gateway ? {
|
|
709
752
|
...unpinned,
|
|
710
|
-
gateway
|
|
753
|
+
gateway: entry || gateway
|
|
711
754
|
} : unpinned;
|
|
712
755
|
const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
713
756
|
...routed,
|
|
@@ -718,10 +761,12 @@ function judgeCaller(a, readFile) {
|
|
|
718
761
|
...verdict,
|
|
719
762
|
artifacts: refs
|
|
720
763
|
} : verdict;
|
|
721
|
-
a.record
|
|
764
|
+
const { judgements: judgments } = a.record;
|
|
765
|
+
judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
|
|
722
766
|
...recorded,
|
|
723
|
-
retries
|
|
724
|
-
|
|
767
|
+
...retries.length ? { retries } : {},
|
|
768
|
+
call: order
|
|
769
|
+
});
|
|
725
770
|
const { score } = verdict;
|
|
726
771
|
debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
|
|
727
772
|
debugTrace("judge", "response", {
|
|
@@ -738,21 +783,19 @@ function judgeCaller(a, readFile) {
|
|
|
738
783
|
});
|
|
739
784
|
return recorded;
|
|
740
785
|
} catch (error) {
|
|
741
|
-
if (a.signal.aborted)
|
|
742
|
-
a.judgeErrored = true;
|
|
743
|
-
throw error;
|
|
744
|
-
}
|
|
786
|
+
if (a.signal.aborted) throw error;
|
|
745
787
|
const { retries } = error;
|
|
746
788
|
const message = error instanceof Error ? error.message : String(error);
|
|
747
789
|
return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
|
|
748
790
|
}
|
|
749
791
|
};
|
|
750
792
|
return async (question, { on, ...options } = {}) => {
|
|
793
|
+
const order = a.judgeCalls++;
|
|
751
794
|
const call = on === void 0 ? options : {
|
|
752
795
|
...options,
|
|
753
796
|
on: judgeOn(a.turns, on)
|
|
754
797
|
};
|
|
755
|
-
return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
|
|
798
|
+
return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
|
|
756
799
|
};
|
|
757
800
|
}
|
|
758
801
|
//#endregion
|
|
@@ -762,7 +805,7 @@ async function createSandbox(descriptor, options) {
|
|
|
762
805
|
switch (descriptor.kind) {
|
|
763
806
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
764
807
|
case "docker": {
|
|
765
|
-
const { createDockerSandbox } = await import("./docker-
|
|
808
|
+
const { createDockerSandbox } = await import("./docker-YFj4H1Cr.js");
|
|
766
809
|
return await createDockerSandbox(descriptor, options);
|
|
767
810
|
}
|
|
768
811
|
}
|
|
@@ -781,7 +824,7 @@ function classify(error, signal) {
|
|
|
781
824
|
message: "The eval timed out"
|
|
782
825
|
} : {
|
|
783
826
|
kind: "aborted",
|
|
784
|
-
message: "The run was
|
|
827
|
+
message: "The run was canceled"
|
|
785
828
|
};
|
|
786
829
|
if (error instanceof InfraError) return {
|
|
787
830
|
kind: "infra",
|
|
@@ -792,17 +835,39 @@ function classify(error, signal) {
|
|
|
792
835
|
message
|
|
793
836
|
};
|
|
794
837
|
}
|
|
838
|
+
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
839
|
+
const judged = (r) => /(^|\.)toPass$/.test(r.matcher);
|
|
840
|
+
/**
|
|
841
|
+
* Whether `error` is a judge's: one a judge threw, or what a `toPass()` or `.not.toPass()` call
|
|
842
|
+
* failed with. A failing verdict the eval didn't assert, or asserted with `.not.toPass()`, isn't.
|
|
843
|
+
* By message too, since another language's runner hands back its own copy of the error.
|
|
844
|
+
*/
|
|
845
|
+
function isJudges(a, error, raw) {
|
|
846
|
+
const message = messageOf(error);
|
|
847
|
+
if (a.judgeThrown.has(message)) return true;
|
|
848
|
+
return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
852
|
+
* (an `expect.soft`, or a hard check the eval caught), whichever came first, so the attempt fails
|
|
853
|
+
* as that check.
|
|
854
|
+
*/
|
|
855
|
+
function attemptError(a, error, raw) {
|
|
856
|
+
if (!isJudges(a, error, raw)) return error;
|
|
857
|
+
const check = raw.find((r) => r.status !== "pass" && !judged(r));
|
|
858
|
+
return check ? checkFailure([check]) : error;
|
|
859
|
+
}
|
|
795
860
|
/**
|
|
796
861
|
* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
|
|
797
|
-
* assertion, which is the judge's when
|
|
862
|
+
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
|
|
798
863
|
*/
|
|
799
|
-
function recordFailure(a, error, timedOut) {
|
|
864
|
+
function recordFailure(a, error, timedOut, raw) {
|
|
800
865
|
const { record } = a;
|
|
801
866
|
record.error = timedOut ? {
|
|
802
867
|
kind: "timeout",
|
|
803
868
|
message: "The eval timed out"
|
|
804
869
|
} : classify(error, a.signal);
|
|
805
|
-
record.error.message = capMiddle(record.error.message);
|
|
870
|
+
record.error.message = capMiddle(redactSecrets(record.error.message));
|
|
806
871
|
if (error instanceof PrepareError && !timedOut) {
|
|
807
872
|
const { class: kind, stage, ref, cause, log, setBy } = error.failure;
|
|
808
873
|
record.error.prepare = {
|
|
@@ -814,8 +879,7 @@ function recordFailure(a, error, timedOut) {
|
|
|
814
879
|
setBy: setBy ?? []
|
|
815
880
|
};
|
|
816
881
|
}
|
|
817
|
-
|
|
818
|
-
if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
|
|
882
|
+
if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = "judge";
|
|
819
883
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
820
884
|
}
|
|
821
885
|
/**
|
|
@@ -842,11 +906,11 @@ async function recordOom(a, sandbox) {
|
|
|
842
906
|
/**
|
|
843
907
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
844
908
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
845
|
-
*
|
|
909
|
+
* summarized in one `-v` line.
|
|
846
910
|
*/
|
|
847
911
|
function recordEgress(a, sandbox) {
|
|
848
912
|
const traffic = sandbox?.traffic?.();
|
|
849
|
-
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
|
|
913
|
+
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
|
|
850
914
|
const refused = traffic?.blocked ?? [];
|
|
851
915
|
const isExpected = (h) => !!expected?.hosts.includes(h.host);
|
|
852
916
|
const expectedBlocked = refused.filter(isExpected);
|
|
@@ -866,6 +930,24 @@ function recordEgress(a, sandbox) {
|
|
|
866
930
|
a.record.servedModels = sandbox?.servedModels?.() ?? null;
|
|
867
931
|
const asked = sandbox?.askedModels?.() ?? [];
|
|
868
932
|
if (asked.length) a.record.askedModels = asked;
|
|
933
|
+
const upstreams = sandbox?.upstreams?.() ?? [];
|
|
934
|
+
if (upstreams.length) a.record.upstreams = upstreams;
|
|
935
|
+
const cost = sandbox?.reportedCostUsd?.();
|
|
936
|
+
if (cost !== void 0) a.record.reportedCostUsd = cost;
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* The record as its staging file holds it: redacted, with a secret split across a turn's raw
|
|
940
|
+
* stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
|
|
941
|
+
*/
|
|
942
|
+
function stagedRecordJson(record) {
|
|
943
|
+
const turns = record.turns.map((turn) => turn.raw ? {
|
|
944
|
+
...turn,
|
|
945
|
+
raw: redactSplitSecrets(turn.raw.map((part) => JSON.stringify(part))).map((line) => JSON.parse(line))
|
|
946
|
+
} : turn);
|
|
947
|
+
return redactJson({
|
|
948
|
+
...record,
|
|
949
|
+
turns
|
|
950
|
+
}, 2);
|
|
869
951
|
}
|
|
870
952
|
/** Writes the attempt's record, redacted, and returns what the runner reports. */
|
|
871
953
|
async function writeRecord(a, stagingDir, id) {
|
|
@@ -874,7 +956,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
874
956
|
await mkdir(join(stagingDir, "evals"), { recursive: true });
|
|
875
957
|
await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
|
|
876
958
|
const tmp = `${recordPath}.${process.pid}.tmp`;
|
|
877
|
-
await writeFile(tmp,
|
|
959
|
+
await writeFile(tmp, stagedRecordJson(record));
|
|
878
960
|
await rename(tmp, recordPath);
|
|
879
961
|
return {
|
|
880
962
|
outcome: record.outcome,
|
|
@@ -896,7 +978,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
896
978
|
function turnErrors(transcripts) {
|
|
897
979
|
return transcripts.flatMap((events, i) => {
|
|
898
980
|
const found = events.find((e) => e.type === "error");
|
|
899
|
-
return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(
|
|
981
|
+
return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
|
|
900
982
|
});
|
|
901
983
|
}
|
|
902
984
|
//#endregion
|
|
@@ -928,7 +1010,7 @@ function hookHandlers(plugin, text, hooks) {
|
|
|
928
1010
|
const what = `the ${plugin.name} plugin's hooks/hooks.json`;
|
|
929
1011
|
let raw;
|
|
930
1012
|
try {
|
|
931
|
-
raw = JSON.parse(text);
|
|
1013
|
+
raw = JSON.parse(withoutBom(text));
|
|
932
1014
|
} catch {
|
|
933
1015
|
throw new InfraError(`${what} isn't valid JSON`);
|
|
934
1016
|
}
|
|
@@ -989,7 +1071,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
989
1071
|
if (!hooks) {
|
|
990
1072
|
let own = {};
|
|
991
1073
|
try {
|
|
992
|
-
own = hooksFile === null ? {} : JSON.parse(hooksFile);
|
|
1074
|
+
own = hooksFile === null ? {} : JSON.parse(withoutBom(hooksFile));
|
|
993
1075
|
} catch {
|
|
994
1076
|
throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
|
|
995
1077
|
}
|
|
@@ -1157,8 +1239,10 @@ function createAttempt(spec, signal, progress, start) {
|
|
|
1157
1239
|
}
|
|
1158
1240
|
},
|
|
1159
1241
|
signal,
|
|
1160
|
-
|
|
1242
|
+
judgeThrown: /* @__PURE__ */ new Set(),
|
|
1161
1243
|
judgeErrors: [],
|
|
1244
|
+
judgeCalls: 0,
|
|
1245
|
+
judgeInputCalls: [],
|
|
1162
1246
|
captureNames: /* @__PURE__ */ new Set(),
|
|
1163
1247
|
backgrounds: [],
|
|
1164
1248
|
artifactBlobs: /* @__PURE__ */ new Map()
|
|
@@ -1206,7 +1290,7 @@ async function runSetup(a, box, env) {
|
|
|
1206
1290
|
const steps = env.setup;
|
|
1207
1291
|
record.setup = [];
|
|
1208
1292
|
for (const [i, step] of steps.entries()) {
|
|
1209
|
-
const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
|
|
1293
|
+
const what = `setup step ${i + 1}/${steps.length} "${clip(redactSecrets(step.run), 80)}" in environment ${spec.environment.name}`;
|
|
1210
1294
|
const start = performance.now();
|
|
1211
1295
|
const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
|
|
1212
1296
|
const r = await asCheck(a, () => box.exec(step.run, {
|
|
@@ -1226,7 +1310,7 @@ async function runSetup(a, box, env) {
|
|
|
1226
1310
|
durationMs,
|
|
1227
1311
|
output
|
|
1228
1312
|
});
|
|
1229
|
-
debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
|
|
1313
|
+
debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(redactSecrets(step.run), 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
|
|
1230
1314
|
debugTrace("env", "setup", {
|
|
1231
1315
|
run: step.run,
|
|
1232
1316
|
cwd: step.cwd,
|
|
@@ -1239,6 +1323,54 @@ async function runSetup(a, box, env) {
|
|
|
1239
1323
|
throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
|
|
1240
1324
|
}
|
|
1241
1325
|
}
|
|
1326
|
+
/** The `to: 'checks'` secrets the setup steps got as their real value: those without `hosts`. */
|
|
1327
|
+
function unbrokeredChecks(env) {
|
|
1328
|
+
const brokered = new Set(env.substitutions.filter((s) => s.phase === "checks").map((s) => s.name));
|
|
1329
|
+
return Object.keys(env.checkVars).filter((name) => !brokered.has(name));
|
|
1330
|
+
}
|
|
1331
|
+
/** How many of the processes setup left running the record keeps, and how much of each command line. */
|
|
1332
|
+
const SETUP_PROCESSES = 20;
|
|
1333
|
+
const SETUP_COMMAND_CHARS = 200;
|
|
1334
|
+
/**
|
|
1335
|
+
* The processes the setup steps left running with an unbrokered `to: 'checks'` secret in their
|
|
1336
|
+
* environment, by the secret's variable name: a copy under another name or in a process's
|
|
1337
|
+
* arguments isn't found. The listing runs as the agent's user, with no checks secrets of its own,
|
|
1338
|
+
* so it sees the environments the agent could read, and prints names, never values. Docker only:
|
|
1339
|
+
* the fake sandbox runs on the host, with no separation to report on.
|
|
1340
|
+
*/
|
|
1341
|
+
async function leftRunning(a, box, env) {
|
|
1342
|
+
const names = unbrokeredChecks(env);
|
|
1343
|
+
if (a.spec.sandbox.kind !== "docker" || !env.setup.length || !names.length) return [];
|
|
1344
|
+
const r = await box.exec([
|
|
1345
|
+
"for t in tr sed sort; do command -v \"$t\" >/dev/null || { echo \"no $t in the image\" >&2; exit 1; }; done",
|
|
1346
|
+
"for p in /proc/[0-9]*; do",
|
|
1347
|
+
" [ \"$p\" = \"/proc/$$\" ] && continue",
|
|
1348
|
+
` n=$(tr '\\0' '\\n' < "$p/environ" 2>/dev/null | sed -n -E 's/^(${names.join("|")})=.*/\\1/p' | sort -u | tr '\\n' ' ')`,
|
|
1349
|
+
" [ -n \"$n\" ] || continue",
|
|
1350
|
+
` c=$(tr '\\0\\t\\n' ' ' < "$p/cmdline" 2>/dev/null)`,
|
|
1351
|
+
` [ -n "$c" ] || c=$(tr '\\t\\n' ' ' < "$p/comm" 2>/dev/null)`,
|
|
1352
|
+
` printf '%s\\t%s\\n' "$n" "$c"`,
|
|
1353
|
+
"done",
|
|
1354
|
+
"true"
|
|
1355
|
+
].join("\n"), { signal: a.signal });
|
|
1356
|
+
if (r.exitCode !== 0) {
|
|
1357
|
+
debugLog("env", `couldn't list the processes setup left running: ${tailOf(r.stderr)}`);
|
|
1358
|
+
return [];
|
|
1359
|
+
}
|
|
1360
|
+
const known = new Set(names);
|
|
1361
|
+
const found = r.stdout.split("\n").flatMap((line) => {
|
|
1362
|
+
const tab = line.indexOf(" ");
|
|
1363
|
+
if (tab < 0) return [];
|
|
1364
|
+
const secrets = line.slice(0, tab).split(" ").filter((n) => known.has(n));
|
|
1365
|
+
const command = clip(redactSecrets(line.slice(tab + 1).trim()), SETUP_COMMAND_CHARS);
|
|
1366
|
+
return secrets.length ? [{
|
|
1367
|
+
command,
|
|
1368
|
+
secrets
|
|
1369
|
+
}] : [];
|
|
1370
|
+
});
|
|
1371
|
+
if (found.length > SETUP_PROCESSES) debugLog("env", `setup left ${found.length} processes running with a checks secret; the record keeps the first ${SETUP_PROCESSES}`);
|
|
1372
|
+
return found.slice(0, SETUP_PROCESSES);
|
|
1373
|
+
}
|
|
1242
1374
|
const WORKSPACE_TAR = ".reeval-workspace.tar";
|
|
1243
1375
|
/** Unpacks the git workspace's tarball into the (empty) workspace. */
|
|
1244
1376
|
async function copyWorkspace(box, workspace, signal) {
|
|
@@ -1434,8 +1566,8 @@ async function startSession(a, runId) {
|
|
|
1434
1566
|
signal: a.signal,
|
|
1435
1567
|
env: env.vars,
|
|
1436
1568
|
image: a.image,
|
|
1437
|
-
brokering: brokeringFor(spec.harness),
|
|
1438
|
-
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
|
|
1569
|
+
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
1570
|
+
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1439
1571
|
substitutions: env.substitutions,
|
|
1440
1572
|
egress: spec.egress
|
|
1441
1573
|
});
|
|
@@ -1473,6 +1605,11 @@ async function startSession(a, runId) {
|
|
|
1473
1605
|
for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
|
|
1474
1606
|
for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
|
|
1475
1607
|
await runSetup(a, box, env);
|
|
1608
|
+
const running = await leftRunning(a, box, env);
|
|
1609
|
+
if (running.length) {
|
|
1610
|
+
a.record.setupProcesses = running;
|
|
1611
|
+
debugLog("env", `setup left ${running.length} process${running.length === 1 ? "" : "es"} running with ${[...new Set(running.flatMap((p) => p.secrets))].join(", ")} (to: 'checks', no hosts) in the environment: ${running.map((p) => clip(p.command, 80)).join("; ")}`);
|
|
1612
|
+
}
|
|
1476
1613
|
await snapshot(box, a.signal);
|
|
1477
1614
|
debugLog("env", "workspace snapshot taken: the diff starts here");
|
|
1478
1615
|
const timed = await timeMcpServers(box, env.mcp, a.signal);
|
|
@@ -1481,7 +1618,7 @@ async function startSession(a, runId) {
|
|
|
1481
1618
|
debugLog("env", "stdio MCP servers start through reeval's MCP timer");
|
|
1482
1619
|
} else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
|
|
1483
1620
|
if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
|
|
1484
|
-
const harness = await createHarness(spec.harness);
|
|
1621
|
+
const harness = await createHarness(spec.harness, spec.vertex);
|
|
1485
1622
|
const harnessStart = performance.now();
|
|
1486
1623
|
debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
|
|
1487
1624
|
const session = await startHarness(a, harness, box, {
|
|
@@ -1514,7 +1651,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1514
1651
|
const cutoff = new Promise((resolve) => {
|
|
1515
1652
|
onAbort = () => {
|
|
1516
1653
|
timer = setTimeout(() => {
|
|
1517
|
-
const message = `The agent didn't stop within ${grace / 1e3}s of being
|
|
1654
|
+
const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
|
|
1518
1655
|
const at = Date.now() - Date.parse(startedAt);
|
|
1519
1656
|
resolve({
|
|
1520
1657
|
text: "",
|
|
@@ -1610,7 +1747,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
1610
1747
|
kind: "turn-start",
|
|
1611
1748
|
turn: turnNumber
|
|
1612
1749
|
});
|
|
1613
|
-
debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
|
|
1750
|
+
debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(redactSecrets(prompt), 160))}`);
|
|
1614
1751
|
debugTrace("harness", "prompt", {
|
|
1615
1752
|
turn: turnNumber,
|
|
1616
1753
|
prompt
|
|
@@ -1671,10 +1808,9 @@ function turnSender(a, takeContext, checks) {
|
|
|
1671
1808
|
ms: result.durationMs,
|
|
1672
1809
|
tools: count("tool", "web-search", "web-fetch"),
|
|
1673
1810
|
fileEdits: count("file-edit"),
|
|
1674
|
-
...failure?.type === "error" ? { error: redactSecrets(
|
|
1811
|
+
...failure?.type === "error" ? { error: clip(redactSecrets(failure.message.trim()), 300) } : {}
|
|
1675
1812
|
});
|
|
1676
|
-
|
|
1677
|
-
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
1813
|
+
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
1678
1814
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
1679
1815
|
if (timedOut) {
|
|
1680
1816
|
const error = new InfraError(timedOut);
|
|
@@ -1768,13 +1904,19 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1768
1904
|
throw a.startFailure ? a.startFailure.error : error;
|
|
1769
1905
|
});
|
|
1770
1906
|
a.signal.throwIfAborted();
|
|
1771
|
-
const missed = await unawaited() ??
|
|
1907
|
+
const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
|
|
1772
1908
|
if (a.startFailure) throw a.startFailure.error;
|
|
1773
1909
|
if (missed) throw missed;
|
|
1774
|
-
const
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1910
|
+
const judgeErrors = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
|
|
1911
|
+
const [only, ...more] = judgeErrors;
|
|
1912
|
+
if (only) {
|
|
1913
|
+
const message = more.length ? `${more.length + 1} judge calls failed:\n${judgeErrors.join("\n")}` : only;
|
|
1914
|
+
a.judgeThrown.add(message);
|
|
1915
|
+
throw new Error(message);
|
|
1916
|
+
}
|
|
1917
|
+
} catch (thrown) {
|
|
1918
|
+
const error = attemptError(a, thrown, assertions());
|
|
1919
|
+
recordFailure(a, error, timedOut(), assertions());
|
|
1778
1920
|
failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
|
|
1779
1921
|
} finally {
|
|
1780
1922
|
if (!a.signal.aborted) await unawaited().catch(() => {});
|
|
@@ -1796,7 +1938,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1796
1938
|
const checked = assertionRecords(assertions());
|
|
1797
1939
|
if (checked.length) record.assertions = checked;
|
|
1798
1940
|
for (const x of checked) {
|
|
1799
|
-
const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
|
|
1941
|
+
const why = x.message ? ` · ${clip(redactSecrets(x.message.split("\n")[0] ?? ""), 160)}` : "";
|
|
1800
1942
|
debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
|
|
1801
1943
|
}
|
|
1802
1944
|
const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
|
|
@@ -1818,39 +1960,45 @@ const declaredModeSchema = z.enum([
|
|
|
1818
1960
|
"run",
|
|
1819
1961
|
"skip",
|
|
1820
1962
|
"only"
|
|
1821
|
-
])
|
|
1963
|
+
]);
|
|
1822
1964
|
const declaredFamilySchema = z.strictObject({
|
|
1823
|
-
name: z.string()
|
|
1824
|
-
case: z.number().int().min(0)
|
|
1825
|
-
cases: z.number().int().min(2)
|
|
1826
|
-
})
|
|
1965
|
+
name: z.string(),
|
|
1966
|
+
case: z.number().int().min(0),
|
|
1967
|
+
cases: z.number().int().min(2)
|
|
1968
|
+
});
|
|
1827
1969
|
const declaredTodoSchema = z.strictObject({
|
|
1828
1970
|
type: z.literal("todo"),
|
|
1829
|
-
name: z.string()
|
|
1830
|
-
})
|
|
1971
|
+
name: z.string()
|
|
1972
|
+
});
|
|
1973
|
+
const hash = () => z.string().regex(/^[0-9a-f]{64}$/, "must be a sha256 hash: 64 lowercase hex digits");
|
|
1974
|
+
const declaredSourceSchema = z.strictObject({
|
|
1975
|
+
body: hash(),
|
|
1976
|
+
case: hash().optional()
|
|
1977
|
+
});
|
|
1831
1978
|
const declaredEvalSchema = z.strictObject({
|
|
1832
1979
|
type: z.literal("eval"),
|
|
1833
|
-
name: z.string()
|
|
1980
|
+
name: z.string(),
|
|
1834
1981
|
mode: declaredModeSchema,
|
|
1835
|
-
options: evalOptionsDataSchema
|
|
1836
|
-
body: z.string()
|
|
1837
|
-
|
|
1838
|
-
|
|
1982
|
+
options: evalOptionsDataSchema,
|
|
1983
|
+
body: z.string(),
|
|
1984
|
+
source: declaredSourceSchema,
|
|
1985
|
+
family: declaredFamilySchema.optional()
|
|
1986
|
+
});
|
|
1839
1987
|
const declaredGroupSchema = z.strictObject({
|
|
1840
1988
|
type: z.literal("group"),
|
|
1841
|
-
name: z.string()
|
|
1989
|
+
name: z.string(),
|
|
1842
1990
|
mode: declaredModeSchema,
|
|
1843
|
-
options: evalOptionsDataSchema
|
|
1991
|
+
options: evalOptionsDataSchema,
|
|
1844
1992
|
get children() {
|
|
1845
1993
|
return z.array(declaredNodeSchema);
|
|
1846
1994
|
}
|
|
1847
|
-
})
|
|
1995
|
+
});
|
|
1848
1996
|
const declaredNodeSchema = z.discriminatedUnion("type", [
|
|
1849
1997
|
declaredEvalSchema,
|
|
1850
1998
|
declaredGroupSchema,
|
|
1851
1999
|
declaredTodoSchema
|
|
1852
|
-
])
|
|
1853
|
-
const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) })
|
|
2000
|
+
]);
|
|
2001
|
+
const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) });
|
|
1854
2002
|
const NODE_SCHEMAS = {
|
|
1855
2003
|
eval: declaredEvalSchema,
|
|
1856
2004
|
group: declaredGroupSchema,
|
|
@@ -1881,18 +2029,15 @@ function configLevel(config) {
|
|
|
1881
2029
|
questions: [],
|
|
1882
2030
|
timeout: void 0,
|
|
1883
2031
|
tags: [],
|
|
2032
|
+
revision: void 0,
|
|
1884
2033
|
pin: [],
|
|
1885
2034
|
sources: {}
|
|
1886
2035
|
};
|
|
1887
2036
|
}
|
|
1888
|
-
/**
|
|
1889
|
-
* The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
|
|
1890
|
-
* `order`). Throws a UsageError for options the plan refuses.
|
|
1891
|
-
*/
|
|
2037
|
+
/** The tests an eval registers. Throws a UsageError for options the plan refuses. */
|
|
1892
2038
|
function planEval(declaration, ctx) {
|
|
1893
|
-
const { name,
|
|
1894
|
-
const
|
|
1895
|
-
const where = `evaluate("${[...groups, name].join(" › ")}")`;
|
|
2039
|
+
const { name, groups: enclosing, options: own } = declaration;
|
|
2040
|
+
const where = `evaluate("${[...enclosing.map((g) => g.name), name].join(" › ")}")`;
|
|
1896
2041
|
checkOptions(own, where);
|
|
1897
2042
|
const levels = [...enclosing.map((g) => ({
|
|
1898
2043
|
options: g.options,
|
|
@@ -1904,25 +2049,54 @@ function planEval(declaration, ctx) {
|
|
|
1904
2049
|
const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
|
|
1905
2050
|
profile: ctx.profile,
|
|
1906
2051
|
runs: ctx.runs,
|
|
1907
|
-
skipped: mode === "skip" || mode === "todo"
|
|
2052
|
+
skipped: declaration.mode === "skip" || declaration.mode === "todo",
|
|
2053
|
+
pooled: declaration.family !== void 0
|
|
1908
2054
|
});
|
|
2055
|
+
const { harnesses } = resolvedEval;
|
|
2056
|
+
if (!matchesTags(resolvedEval.resolved.tags, ctx.tags)) return {
|
|
2057
|
+
tests: null,
|
|
2058
|
+
harnesses
|
|
2059
|
+
};
|
|
2060
|
+
return {
|
|
2061
|
+
tests: planTests(declaration, ctx, where, resolvedEval),
|
|
2062
|
+
harnesses
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
function planTests(declaration, ctx, where, resolvedEval) {
|
|
2066
|
+
const { name, mode, groups: enclosing } = declaration;
|
|
2067
|
+
const groups = enclosing.map((g) => g.name);
|
|
1909
2068
|
const { resolved } = resolvedEval;
|
|
1910
|
-
|
|
2069
|
+
const source = {
|
|
2070
|
+
...declaration.source,
|
|
2071
|
+
options: optionsHash([...enclosing.map((g) => g.options), declaration.options])
|
|
2072
|
+
};
|
|
1911
2073
|
const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
|
|
1912
2074
|
const gateways = ctx.profile?.gateway ?? ctx.config.gateway;
|
|
1913
2075
|
const gatewaySet = ctx.config.gatewaySet || ctx.profile?.gateway !== void 0;
|
|
1914
2076
|
const routes = /* @__PURE__ */ new Map();
|
|
1915
2077
|
const gatewayNotes = [];
|
|
2078
|
+
let vertex;
|
|
1916
2079
|
const routed = resolvedEval.skip ? resolved.harness : resolved.harness.map((harness) => {
|
|
1917
2080
|
const route = resolveGateway(gateways, harness, where);
|
|
1918
2081
|
const hid = harnessId(harness);
|
|
1919
2082
|
if (gatewaySet || harness.gateway !== void 0) {
|
|
1920
2083
|
routes.set(hid, route);
|
|
1921
|
-
gatewayNotes.push(`gateway: ${hid} → ${route.name}`);
|
|
2084
|
+
gatewayNotes.push(`gateway: ${hid} → ${route.name}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
|
|
1922
2085
|
}
|
|
1923
2086
|
if (route.name === "direct") return harness;
|
|
1924
|
-
|
|
1925
|
-
|
|
2087
|
+
if (harness.kind !== "fake") try {
|
|
2088
|
+
if (route.name === "vercel") {
|
|
2089
|
+
const problem = vercelCredentialProblem();
|
|
2090
|
+
if (problem) throw new UsageError(problem);
|
|
2091
|
+
}
|
|
2092
|
+
if (route.name === "vertex") {
|
|
2093
|
+
const entry = gateways.find((g) => g.kind === "vertex");
|
|
2094
|
+
vertex ??= vertexSettings(entry?.kind === "vertex" ? entry : {});
|
|
2095
|
+
}
|
|
2096
|
+
} catch (error) {
|
|
2097
|
+
if (error instanceof UsageError) throw new UsageError(`${where} on ${hid}: ${error.message}`);
|
|
2098
|
+
throw error;
|
|
2099
|
+
}
|
|
1926
2100
|
return {
|
|
1927
2101
|
...harness,
|
|
1928
2102
|
gateway: route.name
|
|
@@ -1931,9 +2105,16 @@ function planEval(declaration, ctx) {
|
|
|
1931
2105
|
const plannedAttempts = resolvedEval.skip ? [] : expand({
|
|
1932
2106
|
...resolved,
|
|
1933
2107
|
harness: routed
|
|
1934
|
-
});
|
|
2108
|
+
}, vertex);
|
|
2109
|
+
let judge;
|
|
1935
2110
|
if (plannedAttempts.length) try {
|
|
1936
|
-
resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
|
|
2111
|
+
const gateway = resolveModelGateway(resolved.judge.model, resolved.judge.gateway, { gateways }, "the judge model");
|
|
2112
|
+
const { model } = resolved.judge;
|
|
2113
|
+
if (typeof model === "string") judge = {
|
|
2114
|
+
model,
|
|
2115
|
+
...gateway ? { gateway } : {}
|
|
2116
|
+
};
|
|
2117
|
+
if (gatewaySet && fellBackToDirect(gateway, resolved.judge.gateway, { gateways })) gatewayNotes.push(`gateway: judge ${model} → direct (fallback)`);
|
|
1937
2118
|
} catch (error) {
|
|
1938
2119
|
if (error instanceof UsageError) throw new UsageError(`${where}: ${error.message}`);
|
|
1939
2120
|
throw error;
|
|
@@ -1941,6 +2122,8 @@ function planEval(declaration, ctx) {
|
|
|
1941
2122
|
if (ctx.config.secrets.unbrokered === "error") {
|
|
1942
2123
|
const [first] = unbrokered(resolved.env, plannedAttempts);
|
|
1943
2124
|
if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
|
|
2125
|
+
const [plain] = plainHeaders(plannedAttempts);
|
|
2126
|
+
if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
|
|
1944
2127
|
}
|
|
1945
2128
|
const pluginNotes = [];
|
|
1946
2129
|
const pluginLabels = /* @__PURE__ */ new Map();
|
|
@@ -1966,6 +2149,7 @@ function planEval(declaration, ctx) {
|
|
|
1966
2149
|
configFile: ctx.config.file,
|
|
1967
2150
|
file: path
|
|
1968
2151
|
});
|
|
2152
|
+
if (gatewayNotes.length && openRouterFallbacks(gateways)) gatewayNotes.push("gateway: openrouter() fallbacks on: reeval's own calls may fail over to another upstream provider; the harnesses' calls take OpenRouter's own routing");
|
|
1969
2153
|
plan.notes.push(...gatewayNotes, ...pluginNotes);
|
|
1970
2154
|
const setByOf = (planned) => ({
|
|
1971
2155
|
sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
|
|
@@ -1978,6 +2162,8 @@ function planEval(declaration, ctx) {
|
|
|
1978
2162
|
name,
|
|
1979
2163
|
groups,
|
|
1980
2164
|
tags: resolved.tags,
|
|
2165
|
+
...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
|
|
2166
|
+
source,
|
|
1981
2167
|
cell: first.cell,
|
|
1982
2168
|
n: 1,
|
|
1983
2169
|
label: "",
|
|
@@ -2014,6 +2200,8 @@ function planEval(declaration, ctx) {
|
|
|
2014
2200
|
name,
|
|
2015
2201
|
groups,
|
|
2016
2202
|
tags: resolved.tags,
|
|
2203
|
+
...resolved.revision !== void 0 ? { revision: resolved.revision } : {},
|
|
2204
|
+
source,
|
|
2017
2205
|
cell: planned.cell,
|
|
2018
2206
|
n: planned.n,
|
|
2019
2207
|
label: planned.label,
|
|
@@ -2035,13 +2223,14 @@ function planEval(declaration, ctx) {
|
|
|
2035
2223
|
} : plan,
|
|
2036
2224
|
...skip ? { skip } : {},
|
|
2037
2225
|
...family ? { family } : {},
|
|
2038
|
-
...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {}
|
|
2226
|
+
...routes.has(harnessId(planned.harness)) ? { gateway: routes.get(harnessId(planned.harness)) } : {},
|
|
2227
|
+
...judge ? { judge } : {}
|
|
2039
2228
|
};
|
|
2040
2229
|
return {
|
|
2041
2230
|
name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
|
|
2042
2231
|
mode: skip ? "skip" : mode,
|
|
2043
2232
|
attempt,
|
|
2044
|
-
timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
|
|
2233
|
+
timeout: skip ? void 0 : Math.min(timeoutMs + setupAllowance(planned) + GRACE_MS, MAX_TIMER_MS),
|
|
2045
2234
|
gate: skip ? void 0 : {
|
|
2046
2235
|
limits: attemptLimits(planned, ctx.concurrency),
|
|
2047
2236
|
rank: {
|
|
@@ -2221,16 +2410,54 @@ function registerTodo(name) {
|
|
|
2221
2410
|
const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
|
|
2222
2411
|
if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
|
|
2223
2412
|
}
|
|
2224
|
-
/**
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2413
|
+
/**
|
|
2414
|
+
* The evals declared so far, by id, with their name paths and whether each is a family's case,
|
|
2415
|
+
* so a second id is refused.
|
|
2416
|
+
*/
|
|
2417
|
+
const declared = /* @__PURE__ */ new Map();
|
|
2418
|
+
/** The pooled families declared so far, by id: their name path, length and the cases seen. */
|
|
2419
|
+
const families = /* @__PURE__ */ new Map();
|
|
2420
|
+
/** Two name paths with one id differ only where a name holds the ` › ` the id joins them with. */
|
|
2421
|
+
const separatorHint = (path, earlier) => path.length === earlier.length ? "" : ` " › " joins the group and eval names in an eval's id, so a name containing it can be a grouped eval's id.`;
|
|
2422
|
+
/** A run's facts name an eval and a family alike by id, so an eval can't take a family's. */
|
|
2423
|
+
const namedLikeFamily = (eval_, family) => new UsageError(`evaluate("${eval_.join(" › ")}") has the same id as the pooled evaluate.each("${family.join(" › ")}") in this file: a family's id is its name template. Give the eval or the template another name.${separatorHint(eval_, family)}`);
|
|
2424
|
+
function checkUnique(where, groups, name, hint, inFamily) {
|
|
2425
|
+
const path = [...groups, name];
|
|
2426
|
+
const id = evalId(collectingFile() ?? "", path);
|
|
2427
|
+
const earlier = declared.get(id);
|
|
2428
|
+
if (earlier) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}${separatorHint(path, earlier.path)}`);
|
|
2429
|
+
const family = inFamily ? void 0 : families.get(id);
|
|
2430
|
+
if (family) throw namedLikeFamily(path, family.path);
|
|
2431
|
+
declared.set(id, {
|
|
2432
|
+
path,
|
|
2433
|
+
inFamily
|
|
2434
|
+
});
|
|
2230
2435
|
}
|
|
2231
|
-
|
|
2436
|
+
/**
|
|
2437
|
+
* A family's cases pool by its id, the groups and the template, so each case may be declared
|
|
2438
|
+
* once and every case must agree on the list's length.
|
|
2439
|
+
*/
|
|
2440
|
+
function checkFamily(groups, family) {
|
|
2441
|
+
const path = [...groups, family.name];
|
|
2442
|
+
const id = evalId(collectingFile() ?? "", path);
|
|
2443
|
+
const earlier = families.get(id);
|
|
2444
|
+
if (!earlier) {
|
|
2445
|
+
const plain = declared.get(id);
|
|
2446
|
+
if (plain && !plain.inFamily) throw namedLikeFamily(plain.path, path);
|
|
2447
|
+
families.set(id, {
|
|
2448
|
+
path,
|
|
2449
|
+
cases: family.cases,
|
|
2450
|
+
seen: /* @__PURE__ */ new Set([family.case])
|
|
2451
|
+
});
|
|
2452
|
+
return;
|
|
2453
|
+
}
|
|
2454
|
+
if (earlier.path.length !== path.length || earlier.cases !== family.cases || earlier.seen.has(family.case)) throw new UsageError(`evaluate.each("${path.join(" › ")}") is declared twice in this file. Each pooled evaluate.each needs its own name, or their cases are pooled into one family. Give each its own name template, or pass evaluate.each(cases, { pool: false }) to keep the cases apart.${separatorHint(path, earlier.path)}`);
|
|
2455
|
+
earlier.seen.add(family.case);
|
|
2456
|
+
}
|
|
2457
|
+
function register(mode, name, own, fn, source, hint = "", family) {
|
|
2232
2458
|
const groups = groupStack.map((g) => g.name);
|
|
2233
|
-
|
|
2459
|
+
if (family) checkFamily(groups, family);
|
|
2460
|
+
checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint, !!family);
|
|
2234
2461
|
const body = `b${bodies.size + 1}`;
|
|
2235
2462
|
bodies.set(body, fn);
|
|
2236
2463
|
declare({
|
|
@@ -2239,9 +2466,10 @@ function register(mode, name, own, fn, hint = "", family) {
|
|
|
2239
2466
|
mode,
|
|
2240
2467
|
options: data(own),
|
|
2241
2468
|
body,
|
|
2469
|
+
source,
|
|
2242
2470
|
...family ? { family } : {}
|
|
2243
2471
|
});
|
|
2244
|
-
const tests = planEval({
|
|
2472
|
+
const { tests, harnesses } = planEval({
|
|
2245
2473
|
name,
|
|
2246
2474
|
mode,
|
|
2247
2475
|
groups: groupStack.map(({ name, options }) => ({
|
|
@@ -2249,22 +2477,25 @@ function register(mode, name, own, fn, hint = "", family) {
|
|
|
2249
2477
|
options
|
|
2250
2478
|
})),
|
|
2251
2479
|
options: own,
|
|
2480
|
+
source,
|
|
2252
2481
|
...family ? { family } : {}
|
|
2253
2482
|
}, {
|
|
2254
2483
|
...getRunContext(),
|
|
2255
2484
|
file: collectingFile(),
|
|
2256
2485
|
order: registered
|
|
2257
2486
|
});
|
|
2487
|
+
declareHarnesses(harnesses);
|
|
2258
2488
|
if (!tests) return;
|
|
2259
2489
|
registered++;
|
|
2260
2490
|
for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
2261
2491
|
}
|
|
2262
2492
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2263
2493
|
let registered = 0;
|
|
2264
|
-
const defineEval = (mode) => (name, a, b) => {
|
|
2494
|
+
const defineEval = (mode, declared) => (name, a, b) => {
|
|
2265
2495
|
checkOnly(mode, "evaluate");
|
|
2266
2496
|
const [own, fn] = splitArgs("evaluate", name, a, b);
|
|
2267
|
-
|
|
2497
|
+
const where = `evaluate("${name}")`;
|
|
2498
|
+
register(mode, name, withPaths(own, where), fn, declared ?? { body: bodyHash(fn, where, collectingFile()) });
|
|
2268
2499
|
};
|
|
2269
2500
|
/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
|
|
2270
2501
|
function formatName(template, item, index) {
|
|
@@ -2290,13 +2521,19 @@ const evalEach = (cases, options) => (name, a, b) => {
|
|
|
2290
2521
|
const own = withPaths(written, `evaluate.each("${name}")`);
|
|
2291
2522
|
const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
|
|
2292
2523
|
const pooled = pool && cases.length >= 2;
|
|
2524
|
+
const file = collectingFile();
|
|
2525
|
+
const body = bodyHash(fn, `evaluate.each("${name}")`, file);
|
|
2293
2526
|
cases.forEach((item, i) => {
|
|
2294
2527
|
const family = pooled ? {
|
|
2295
2528
|
name,
|
|
2296
2529
|
case: i,
|
|
2297
2530
|
cases: cases.length
|
|
2298
2531
|
} : void 0;
|
|
2299
|
-
|
|
2532
|
+
const source = {
|
|
2533
|
+
body,
|
|
2534
|
+
case: caseHash(item, `evaluate.each("${name}") case ${i}`, file)
|
|
2535
|
+
};
|
|
2536
|
+
register("run", formatName(name, item, i), own, (c) => fn(c, item), source, hint, family);
|
|
2300
2537
|
});
|
|
2301
2538
|
};
|
|
2302
2539
|
const evaluate = Object.assign(defineEval("run"), {
|
|
@@ -2326,8 +2563,8 @@ function declareNodes(nodes, bodyFor, base) {
|
|
|
2326
2563
|
} else if (node.family) {
|
|
2327
2564
|
const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
|
|
2328
2565
|
checkOnly(node.mode, "evaluate");
|
|
2329
|
-
register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
|
|
2330
|
-
} else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
2566
|
+
register(node.mode, node.name, options, bodyFor(node.body), node.source, "", node.family);
|
|
2567
|
+
} else defineEval(node.mode, node.source)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
|
|
2331
2568
|
}
|
|
2332
2569
|
/**
|
|
2333
2570
|
* Options from their data forms, with host paths absolute against `base`: environments become
|
|
@@ -2364,4 +2601,4 @@ function optionsFrom(written, where, base) {
|
|
|
2364
2601
|
//#endregion
|
|
2365
2602
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2366
2603
|
|
|
2367
|
-
//# sourceMappingURL=evaluate-
|
|
2604
|
+
//# sourceMappingURL=evaluate-BPP9s3s6.js.map
|