reeval 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/bin/mcp-timer.cjs +224 -0
- package/dist/{adapter-Bwy3QxAK.js → adapter-DigeVFWC.js} +3 -3
- package/dist/{adapter-Bwy3QxAK.js.map → adapter-DigeVFWC.js.map} +1 -1
- package/dist/{agent-DpE80WPz.js → agent-CatUEo5h.js} +21 -6
- package/dist/{agent-DpE80WPz.js.map → agent-CatUEo5h.js.map} +1 -1
- package/dist/{assemble-D9GWr-HS.js → assemble-CF063ms6.js} +19 -17
- package/dist/assemble-CF063ms6.js.map +1 -0
- package/dist/assemble-CJoD_Rzr.js +1 -0
- package/dist/bridge/index.mjs +2 -1
- package/dist/{capture-BQlGHiMC.js → capture-BiD7lVaM.js} +25 -17
- package/dist/{capture-BQlGHiMC.js.map → capture-BiD7lVaM.js.map} +1 -1
- package/dist/capture-Dp3q7K7Y.js +1 -0
- package/dist/{cli-ClpPT8kn.js → cli-B5zQCR2r.js} +161 -116
- package/dist/cli-B5zQCR2r.js.map +1 -0
- package/dist/{cli-spec-qmEnQ25_.js → cli-spec-BXIBUlaK.js} +34 -15
- package/dist/cli-spec-BXIBUlaK.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{command-TgbzPizb.js → command-BMHXHqMT.js} +15 -14
- package/dist/command-BMHXHqMT.js.map +1 -0
- package/dist/command-C94CQNh8.js +91 -0
- package/dist/command-C94CQNh8.js.map +1 -0
- package/dist/command-CVtgyqlM.js +335 -0
- package/dist/command-CVtgyqlM.js.map +1 -0
- package/dist/{command-DbAe1Rzb.js → command-CaZgCTIw.js} +14 -17
- package/dist/command-CaZgCTIw.js.map +1 -0
- package/dist/{command-BlM0y8jN.js → command-DTgBZR0k.js} +6 -6
- package/dist/{command-BlM0y8jN.js.map → command-DTgBZR0k.js.map} +1 -1
- package/dist/{commands-CthqwkAn.js → commands-BXzIDB3X.js} +1 -1
- package/dist/commands-CjpGBUWr.js +666 -0
- package/dist/commands-CjpGBUWr.js.map +1 -0
- package/dist/{config-DrVSEVN0.js → config-Dv5MG7Rp.js} +27 -48
- package/dist/config-Dv5MG7Rp.js.map +1 -0
- package/dist/{context-C1mxXh80.js → context-Dgn48BqT.js} +13 -2
- package/dist/context-Dgn48BqT.js.map +1 -0
- package/dist/{contrasts-5zsj32gk.js → contrasts-CcoMLC_y.js} +3 -3
- package/dist/{contrasts-5zsj32gk.js.map → contrasts-CcoMLC_y.js.map} +1 -1
- package/dist/db-BeqRiWLl.js +602 -0
- package/dist/db-BeqRiWLl.js.map +1 -0
- package/dist/db-CJDRVQal.js +2 -0
- package/dist/{dist-CuY4ehEk.js → dist-CMjp3qqF.js} +2 -2
- package/dist/{dist-CuY4ehEk.js.map → dist-CMjp3qqF.js.map} +1 -1
- package/dist/{docker-DOvjji9I.js → docker-BbTlMmYi.js} +2 -2
- package/dist/{docker-DOvjji9I.js.map → docker-BbTlMmYi.js.map} +1 -1
- package/dist/{docker-D1HZDS6q.js → docker-CCXdbREk.js} +20 -4
- package/dist/{docker-D1HZDS6q.js.map → docker-CCXdbREk.js.map} +1 -1
- package/dist/{docker-DNmMPOBb.js → docker-DC69dKxB.js} +1 -1
- package/dist/{doctor-D2MbNO8L.js → doctor-CqttrZcs.js} +57 -22
- package/dist/doctor-CqttrZcs.js.map +1 -0
- package/dist/{egress-2-qzPvGi.js → egress-BfK_O9ce.js} +435 -30
- package/dist/egress-BfK_O9ce.js.map +1 -0
- package/dist/env-BQqj93gu.js +21 -0
- package/dist/env-BQqj93gu.js.map +1 -0
- package/dist/{environment-Cca6a4wB.js → environment-BBz1Yes9.js} +224 -10
- package/dist/environment-BBz1Yes9.js.map +1 -0
- package/dist/{evaluate-DKIl54Hb.js → evaluate-CXQ1o-pd.js} +539 -559
- package/dist/evaluate-CXQ1o-pd.js.map +1 -0
- package/dist/{evaluate-DMyaGQEj.d.ts → evaluate-yjZKUexW.d.ts} +2 -2
- package/dist/facts-DtENlf9s.js +898 -0
- package/dist/facts-DtENlf9s.js.map +1 -0
- package/dist/{files-BpUYBwGu.js → files-B-eUpCZ1.js} +30 -7
- package/dist/files-B-eUpCZ1.js.map +1 -0
- package/dist/files-BsQbYz2i.js +65 -0
- package/dist/files-BsQbYz2i.js.map +1 -0
- package/dist/{files-BXHZFWmx.js → files-COHQLEHE.js} +6 -17
- package/dist/files-COHQLEHE.js.map +1 -0
- package/dist/files-DqCBW7i8.js +1 -0
- package/dist/files-Ha-jsQXf.js +1 -0
- package/dist/generate-Cs2TMH_k.js +1647 -0
- package/dist/generate-Cs2TMH_k.js.map +1 -0
- package/dist/{git-Crl3b87o.js → git-DhykwgKF.js} +2 -2
- package/dist/{git-Crl3b87o.js.map → git-DhykwgKF.js.map} +1 -1
- package/dist/{harness-CEuntrla.js → harness-BAjw1fcr.js} +2 -2
- package/dist/{harness-CEuntrla.js.map → harness-BAjw1fcr.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{header-DHQv_ZrF.js → header-kSmWatcO.js} +10 -3
- package/dist/{header-DHQv_ZrF.js.map → header-kSmWatcO.js.map} +1 -1
- package/dist/{images-Bi3aGxb9.js → images-BmJRuTpm.js} +61 -13
- package/dist/images-BmJRuTpm.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +4 -4
- package/dist/init-Bwc9byQr.js +543 -0
- package/dist/init-Bwc9byQr.js.map +1 -0
- package/dist/init-D7Y1ny1z.js +1 -0
- package/dist/insights-B2Xmhz9o.js +64 -0
- package/dist/insights-B2Xmhz9o.js.map +1 -0
- package/dist/{invocations-BYDPe3fV.js → invocations-CQTbP092.js} +3 -3
- package/dist/{invocations-BYDPe3fV.js.map → invocations-CQTbP092.js.map} +1 -1
- package/dist/{judge-DW_GDOmB.js → judge-Dt6KW6_O.js} +5 -3
- package/dist/judge-Dt6KW6_O.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +8 -7
- package/dist/judge.js.map +1 -1
- package/dist/{live-jlfszAbb.js → live-DYd2QEyR.js} +2 -2
- package/dist/{live-jlfszAbb.js.map → live-DYd2QEyR.js.map} +1 -1
- package/dist/{model-D0IcOQao.js → model-B-mel-Gv.js} +43 -18
- package/dist/model-B-mel-Gv.js.map +1 -0
- package/dist/model-BGKYVpoj.js +1 -0
- package/dist/{model-DaVCY3rb.js → model-C7tSwT7f.js} +2 -2
- package/dist/{model-DaVCY3rb.js.map → model-C7tSwT7f.js.map} +1 -1
- package/dist/{model-DaX4bNLZ.js → model-CWlAqmbU.js} +135 -40
- package/dist/model-CWlAqmbU.js.map +1 -0
- package/dist/model-DHY-dfoO.js +1 -0
- package/dist/{node-B3ydHo6t.js → node-D0mtjRAO.js} +3 -3
- package/dist/{node-B3ydHo6t.js.map → node-D0mtjRAO.js.map} +1 -1
- package/dist/{options-C0d0AzeR.js → options-CPUW97_7.js} +7 -9
- package/dist/options-CPUW97_7.js.map +1 -0
- package/dist/{pairs-CnInRu-G.js → pairs-DP6XOJby.js} +2 -2
- package/dist/{pairs-CnInRu-G.js.map → pairs-DP6XOJby.js.map} +1 -1
- package/dist/{patch-B1bnfhN-.js → patch-D4oluWzH.js} +2 -2
- package/dist/{patch-B1bnfhN-.js.map → patch-D4oluWzH.js.map} +1 -1
- package/dist/{plan-DgQxp9yy.js → plan-OkwUcJQ9.js} +21 -8
- package/dist/plan-OkwUcJQ9.js.map +1 -0
- package/dist/plugins-CUWMnSNb.js +654 -0
- package/dist/plugins-CUWMnSNb.js.map +1 -0
- package/dist/{preflight-DlQvbySb.js → preflight-B6pY89bF.js} +1 -1
- package/dist/{preflight-BRzD30L5.js → preflight-Bjm7_ylE.js} +14 -10
- package/dist/preflight-Bjm7_ylE.js.map +1 -0
- package/dist/{prepare-B_y-AFMv.js → prepare-CQmK3SBn.js} +70 -23
- package/dist/prepare-CQmK3SBn.js.map +1 -0
- package/dist/prepare-DbdgtZKe.js +2 -0
- package/dist/{provenance-DQ0AXn6c.js → provenance-B3Y17oCC.js} +3 -3
- package/dist/{provenance-DQ0AXn6c.js.map → provenance-B3Y17oCC.js.map} +1 -1
- package/dist/providers-CXN2RFz-.js +1 -0
- package/dist/{providers-DoTq1jHW.js → providers-eoniqK6w.js} +24 -10
- package/dist/providers-eoniqK6w.js.map +1 -0
- package/dist/{query-BO-ux7RB.js → query-CLRZmWBT.js} +3 -3
- package/dist/{query-BO-ux7RB.js.map → query-CLRZmWBT.js.map} +1 -1
- package/dist/query-CmBDYt0_.js +1 -0
- package/dist/{questions-C2dym2ce.js → questions-1XOulevV.js} +6 -6
- package/dist/{questions-C2dym2ce.js.map → questions-1XOulevV.js.map} +1 -1
- package/dist/{record-BKmMvZ6O.js → record-kk9M1hj-.js} +8 -4
- package/dist/record-kk9M1hj-.js.map +1 -0
- package/dist/remarks-1521zwfW.js +38 -0
- package/dist/remarks-1521zwfW.js.map +1 -0
- package/dist/remarks-BhAV1H4V.js +1 -0
- package/dist/remarks-DlJCqWRs.js +278 -0
- package/dist/remarks-DlJCqWRs.js.map +1 -0
- package/dist/{remote-CPPW86Q9.js → remote-cE2VWfkJ.js} +3 -3
- package/dist/{remote-CPPW86Q9.js.map → remote-cE2VWfkJ.js.map} +1 -1
- package/dist/resolve-C4zeccCH.js +724 -0
- package/dist/resolve-C4zeccCH.js.map +1 -0
- package/dist/resolve-D30yZYmB.js +1 -0
- package/dist/{results-BE2f-1XS.js → results-Drtt0uFl.js} +33 -8
- package/dist/results-Drtt0uFl.js.map +1 -0
- package/dist/{resume-Ca1c8vix.js → resume-BEFazUfy.js} +4 -4
- package/dist/{resume-Ca1c8vix.js.map → resume-BEFazUfy.js.map} +1 -1
- package/dist/rules-BFzQqPIm.js +317 -0
- package/dist/rules-BFzQqPIm.js.map +1 -0
- package/dist/run-BeMy92Us.js +88 -0
- package/dist/run-BeMy92Us.js.map +1 -0
- package/dist/{sandbox-DLrsvu7U.js → sandbox-CQaJj7n9.js} +2 -2
- package/dist/{sandbox-DLrsvu7U.js.map → sandbox-CQaJj7n9.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-V4-2w0l8.js +1 -0
- package/dist/{server-TUY5xA_A.js → server-xK0les4f.js} +8 -8
- package/dist/{server-TUY5xA_A.js.map → server-xK0les4f.js.map} +1 -1
- package/dist/source-BDow_oW8.js +1 -0
- package/dist/{source-DJGS0a0u.js → source-Q1hEKbCs.js} +2 -2
- package/dist/{source-DJGS0a0u.js.map → source-Q1hEKbCs.js.map} +1 -1
- package/dist/src-B9L9yGKI.js +1 -0
- package/dist/{src-CLoOMVWI.js → src-Bcah6ZkT.js} +1 -1
- package/dist/{src-DvCg0NBs.js → src-BhmaorOU.js} +28 -6
- package/dist/{src-DvCg0NBs.js.map → src-BhmaorOU.js.map} +1 -1
- package/dist/{src-CF17Ug08.js → src-C0CeKiLU.js} +19 -4
- package/dist/{src-CF17Ug08.js.map → src-C0CeKiLU.js.map} +1 -1
- package/dist/src-C0xowZ3C.js +2 -0
- package/dist/src-CHIfIKUu.js +2 -0
- package/dist/src-DLtyz4w5.js +2 -0
- package/dist/src-DPIjmReZ.js +2 -0
- package/dist/{src-BxfCKA3_.js → src-DXNh5mTr.js} +1 -1
- package/dist/{src-FGXhkEkH.js → src-DiLMkIr9.js} +142 -13
- package/dist/src-DiLMkIr9.js.map +1 -0
- package/dist/{src-ftu5wPSt.js → src-LekQrr_S.js} +209 -29
- package/dist/src-LekQrr_S.js.map +1 -0
- package/dist/src-bCnsA6Vp.js +2 -0
- package/dist/{src-u8_ixn_F.js → src-cbCb-MPy.js} +20 -4
- package/dist/src-cbCb-MPy.js.map +1 -0
- package/dist/{src-DvK0EIZx.js → src-qGoFkuJF.js} +55 -6
- package/dist/{src-DvK0EIZx.js.map → src-qGoFkuJF.js.map} +1 -1
- package/dist/{src-DSYYPEOj.js → src-ynBVosfX.js} +1 -1
- package/dist/state-BQR_BCm4.js +1 -0
- package/dist/{state-DfpHox6b.js → state-BmZ8QVZ8.js} +12 -12
- package/dist/{state-DfpHox6b.js.map → state-BmZ8QVZ8.js.map} +1 -1
- package/dist/studio-app/assets/{index-CBkhNAv8.js → index-CnYJijyI.js} +9 -9
- package/dist/studio-app/index.html +1 -1
- package/dist/suite-BaHPmAGy.js +1 -0
- package/dist/{suite-Cw_7G2RM.js → suite-C8sw2pox.js} +80 -28
- package/dist/suite-C8sw2pox.js.map +1 -0
- package/dist/{summary-BUKfU-9l.js → summary-0oQNGskv.js} +273 -47
- package/dist/summary-0oQNGskv.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +2 -2
- package/dist/{text-gEcKjAyp.js → text-CrVy5ioL.js} +18 -2
- package/dist/text-CrVy5ioL.js.map +1 -0
- package/dist/{transcript-BNNANU8A.js → transcript-BTxJpW4R.js} +2 -2
- package/dist/{transcript-BNNANU8A.js.map → transcript-BTxJpW4R.js.map} +1 -1
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-BYdlwMwo.d.ts → types-GraZtfHN.d.ts} +848 -34
- package/dist/usage-BDo2q_yu.js +1 -0
- package/dist/{command-Bj39y-Wm.js → validate-BdoXUuL_.js} +15 -372
- package/dist/validate-BdoXUuL_.js.map +1 -0
- package/dist/view/index.d.ts +20 -257
- package/dist/view/index.js +7 -6
- package/dist/{views-B8w2J-gD.js → views-BoUoX4_A.js} +8 -98
- package/dist/views-BoUoX4_A.js.map +1 -0
- package/dist/views-noVQNCUP.js +2 -0
- package/dist/{worker-entry-DNnveLqx.js → worker-entry-D0-GErmU.js} +2 -2
- package/dist/{worker-entry-DNnveLqx.js.map → worker-entry-D0-GErmU.js.map} +1 -1
- package/dist/{workspace-C_X0tdv6.js → workspace-B90YDU_a.js} +1 -1
- package/dist/{workspace-C4RA3srV.js → workspace-tjZFkjxC.js} +4 -4
- package/dist/{workspace-C4RA3srV.js.map → workspace-tjZFkjxC.js.map} +1 -1
- package/package.json +3 -1
- package/skills/eval-rubric/SKILL.md +130 -0
- package/dist/agent-D483ru7D.js +0 -334
- package/dist/agent-D483ru7D.js.map +0 -1
- package/dist/assemble-D5KGrwnJ.js +0 -1
- package/dist/assemble-D9GWr-HS.js.map +0 -1
- package/dist/capture-CYGLpWxR.js +0 -1
- package/dist/cli-ClpPT8kn.js.map +0 -1
- package/dist/cli-spec-qmEnQ25_.js.map +0 -1
- package/dist/command-Bj39y-Wm.js.map +0 -1
- package/dist/command-C-cVjlny.js +0 -1
- package/dist/command-DbAe1Rzb.js.map +0 -1
- package/dist/command-TgbzPizb.js.map +0 -1
- package/dist/commands-YUwlyTbr.js +0 -228
- package/dist/commands-YUwlyTbr.js.map +0 -1
- package/dist/config-DrVSEVN0.js.map +0 -1
- package/dist/context-C1mxXh80.js.map +0 -1
- package/dist/db-CKg5NtiW.js +0 -2
- package/dist/db-DfhipHEF.js +0 -1416
- package/dist/db-DfhipHEF.js.map +0 -1
- package/dist/doctor-D2MbNO8L.js.map +0 -1
- package/dist/egress-2-qzPvGi.js.map +0 -1
- package/dist/environment-Cca6a4wB.js.map +0 -1
- package/dist/evaluate-DKIl54Hb.js.map +0 -1
- package/dist/files-BXHZFWmx.js.map +0 -1
- package/dist/files-BpUYBwGu.js.map +0 -1
- package/dist/files-Bw8lWEjg.js +0 -1
- package/dist/files-D6J0iSdx.js +0 -1
- package/dist/images-Bi3aGxb9.js.map +0 -1
- package/dist/init-Cl3f3NBT.js +0 -360
- package/dist/init-Cl3f3NBT.js.map +0 -1
- package/dist/init-fhOXED8F.js +0 -1
- package/dist/judge-DW_GDOmB.js.map +0 -1
- package/dist/model-CXhrziBe.js +0 -1
- package/dist/model-D0IcOQao.js.map +0 -1
- package/dist/model-DaX4bNLZ.js.map +0 -1
- package/dist/model-_bxNrk2h.js +0 -1
- package/dist/options-C0d0AzeR.js.map +0 -1
- package/dist/plan-DgQxp9yy.js.map +0 -1
- package/dist/preflight-BRzD30L5.js.map +0 -1
- package/dist/prepare-B_y-AFMv.js.map +0 -1
- package/dist/prepare-DCKTLHN3.js +0 -2
- package/dist/providers-B9yTdIqF.js +0 -1
- package/dist/providers-DoTq1jHW.js.map +0 -1
- package/dist/query-CRQfsw0e.js +0 -1
- package/dist/record-BKmMvZ6O.js.map +0 -1
- package/dist/results-BE2f-1XS.js.map +0 -1
- package/dist/server-Dp8eOeK5.js +0 -1
- package/dist/source-DdbLGR1M.js +0 -1
- package/dist/src-CcrQbpG8.js +0 -2
- package/dist/src-DCFtJRp0.js +0 -2
- package/dist/src-DYO3DdnQ.js +0 -1
- package/dist/src-FGXhkEkH.js.map +0 -1
- package/dist/src-WD-zgQYZ.js +0 -2
- package/dist/src-_oI9AePb.js +0 -2
- package/dist/src-ftu5wPSt.js.map +0 -1
- package/dist/src-qsKCqv7a.js +0 -2
- package/dist/src-u8_ixn_F.js.map +0 -1
- package/dist/state-DpAxYyED.js +0 -1
- package/dist/suite-Cw_7G2RM.js.map +0 -1
- package/dist/suite-DamAEnNF.js +0 -1
- package/dist/summary-BUKfU-9l.js.map +0 -1
- package/dist/summary-Baa4rTHS.js +0 -192
- package/dist/summary-Baa4rTHS.js.map +0 -1
- package/dist/text-gEcKjAyp.js.map +0 -1
- package/dist/tools-DLHgetpx.js +0 -249
- package/dist/tools-DLHgetpx.js.map +0 -1
- package/dist/views-B8w2J-gD.js.map +0 -1
- package/dist/views-DblnI3kA.js +0 -2
|
@@ -1,32 +1,34 @@
|
|
|
1
1
|
import { t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
2
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
|
|
3
3
|
import { n as dataProblem, p as parseDuration } from "./shared-CA61BuPJ.js";
|
|
4
|
-
import { D as decideState, E as badJudgeArtifacts, O as filePartsOf, T as artifactRefsOf, _ as unbrokeredError, a as evalPlan, d as refuseIllegalCompare, f as sandboxId, g as unbrokered, h as secretVarsIn, i as checkOptions, k as resolveJudgeArtifacts, l as matchesTags, m as sandboxSetBy, n as cascade, o as expand, p as sandboxProvider, s as harnessId, t as brokeredIn, v as mergeJudge, w as judgeProblem, y as resolveStateOptions } from "./options-
|
|
5
|
-
import {
|
|
6
|
-
import { c as relativeFile, i as evalSlug,
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
9
|
-
import {
|
|
4
|
+
import { D as decideState, E as badJudgeArtifacts, O as filePartsOf, T as artifactRefsOf, _ as unbrokeredError, a as evalPlan, d as refuseIllegalCompare, f as sandboxId, g as unbrokered, h as secretVarsIn, i as checkOptions, k as resolveJudgeArtifacts, l as matchesTags, m as sandboxSetBy, n as cascade, o as expand, p as sandboxProvider, s as harnessId, t as brokeredIn, v as mergeJudge, w as judgeProblem, y as resolveStateOptions } from "./options-CPUW97_7.js";
|
|
5
|
+
import { E as sandboxPaths, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, j as workspaceOf, m as filesOf } from "./environment-BBz1Yes9.js";
|
|
6
|
+
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
7
|
+
import { c as judgeModelName, d as isApiError, u as runJudge } from "./judge-Dt6KW6_O.js";
|
|
8
|
+
import "./config-Dv5MG7Rp.js";
|
|
9
|
+
import { t as inCI } from "./env-BQqj93gu.js";
|
|
10
|
+
import { t as MCP_TIMER, ut as shellQuote } from "./src-LekQrr_S.js";
|
|
11
|
+
import { o as redactSecrets } from "./redact-DJ9JCShu.js";
|
|
12
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-CUWMnSNb.js";
|
|
10
13
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
11
|
-
import { _ as
|
|
12
|
-
import { S as placeholder, nt as shellQuote, tt as run } from "./src-ftu5wPSt.js";
|
|
14
|
+
import { S as writePluginFiles, _ as expectedBlockedFor, a as unenforceable, b as refusedMcp, f as agentMcp, h as claudeMcpServers, m as brokeringFor, n as allowedHosts, o as createFakeSandbox, r as egressLabel, x as refusedPlugins, y as layoutOf } from "./egress-BfK_O9ce.js";
|
|
13
15
|
import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile } from "./runtime-PX_oXiCS.js";
|
|
14
16
|
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CGPV3BZY.js";
|
|
15
17
|
import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
|
|
16
|
-
import {
|
|
17
|
-
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-
|
|
18
|
-
import { t as contextTaker } from "./context-
|
|
18
|
+
import { i as formatDuration, n as clip, t as capMiddle } from "./text-CrVy5ioL.js";
|
|
19
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BiD7lVaM.js";
|
|
20
|
+
import { t as contextTaker } from "./context-Dgn48BqT.js";
|
|
21
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-C4zeccCH.js";
|
|
19
22
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
20
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
21
|
-
import { n as judgeDiff } from "./patch-
|
|
22
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
23
|
+
import { i as preparedWorkspace } from "./workspace-tjZFkjxC.js";
|
|
24
|
+
import { n as judgeDiff } from "./patch-D4oluWzH.js";
|
|
25
|
+
import { t as transcriptForJudge } from "./transcript-BTxJpW4R.js";
|
|
23
26
|
import { r as withLease } from "./client-fcwgwOAD.js";
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import { parse } from "yaml";
|
|
27
|
+
import { dirname, join } from "node:path";
|
|
28
|
+
import { parse, stringify } from "yaml";
|
|
27
29
|
import { z } from "zod";
|
|
28
30
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
29
|
-
import { mkdir, readFile, rename,
|
|
31
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
30
32
|
//#region src/attempt/assertions.ts
|
|
31
33
|
const MESSAGE_LIMIT = 1024;
|
|
32
34
|
const LABEL_LIMIT = 60;
|
|
@@ -133,6 +135,11 @@ function traceEnvironment(debug, spec, env) {
|
|
|
133
135
|
setup: env.setup,
|
|
134
136
|
skills: env.skills.map((s) => s.name),
|
|
135
137
|
workspaceSkills: env.workspaceSkills.map((s) => s.name),
|
|
138
|
+
plugins: env.plugins.map((p) => ({
|
|
139
|
+
name: p.name,
|
|
140
|
+
parts: p.parts,
|
|
141
|
+
files: p.files.length
|
|
142
|
+
})),
|
|
136
143
|
mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
|
|
137
144
|
transport: m.transport,
|
|
138
145
|
command: m.command,
|
|
@@ -152,545 +159,36 @@ function traceEnvironment(debug, spec, env) {
|
|
|
152
159
|
}, "resolved environment");
|
|
153
160
|
}
|
|
154
161
|
//#endregion
|
|
155
|
-
//#region src/environment/resolve.ts
|
|
156
|
-
const SKILL_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
157
|
-
const DEFAULT_SETUP_TIMEOUT = "5m";
|
|
158
|
-
function broker(secrets, name, value, hosts, where = ["header"], phase = "agent") {
|
|
159
|
-
const at = badCharacterAt(value);
|
|
160
|
-
if (at && where.includes("header")) throw new InfraError(`${name} has whitespace or a line break in it (character ${at}), so it can't go in a request header. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
161
|
-
const sub = {
|
|
162
|
-
name,
|
|
163
|
-
placeholder: placeholder(),
|
|
164
|
-
value,
|
|
165
|
-
hosts: [...hosts],
|
|
166
|
-
in: [...where],
|
|
167
|
-
phase
|
|
168
|
-
};
|
|
169
|
-
secrets.brokered.push(sub);
|
|
170
|
-
return sub.placeholder;
|
|
171
|
-
}
|
|
172
|
-
/** The host value, or its placeholder when it's brokered to `secret.hosts` (else `hosts`). */
|
|
173
|
-
function hostValue(name, secret, where, secrets, hosts = secret.hosts) {
|
|
174
|
-
const from = secret.from ?? name;
|
|
175
|
-
const value = process.env[from];
|
|
176
|
-
if (!value) {
|
|
177
|
-
if (secret.optional) {
|
|
178
|
-
secrets.unset.push(name);
|
|
179
|
-
return;
|
|
180
|
-
}
|
|
181
|
-
const source = from === name ? name : `${name} (from ${from})`;
|
|
182
|
-
throw new InfraError(`Secret ${source} for ${where} isn't set. Set it in the environment or .env.local.`);
|
|
183
|
-
}
|
|
184
|
-
registerSecret(value);
|
|
185
|
-
if (secret.to === "checks") secrets.checks.push(name);
|
|
186
|
-
else secrets.names.push(name);
|
|
187
|
-
if (!hosts) return value;
|
|
188
|
-
return broker(secrets, name, value, hosts, secret.in, secret.to === "checks" ? "checks" : "agent");
|
|
189
|
-
}
|
|
190
|
-
/** Each host variable in the template, or with `hosts` one placeholder per variable. */
|
|
191
|
-
function template(value, where, secrets, hosts) {
|
|
192
|
-
const placeholders = /* @__PURE__ */ new Map();
|
|
193
|
-
return value.template.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, name) => {
|
|
194
|
-
const v = process.env[name];
|
|
195
|
-
if (!v) throw new InfraError(`secret.template for ${where} needs ${name}, which isn't set. Set it in the environment or .env.local.`);
|
|
196
|
-
registerSecret(v);
|
|
197
|
-
if (!hosts) return v;
|
|
198
|
-
const p = placeholders.get(name) ?? broker(secrets, name, v, hosts);
|
|
199
|
-
placeholders.set(name, p);
|
|
200
|
-
return p;
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
/** A `SKILL.md` split into its YAML frontmatter's `name` and `description` and its body. */
|
|
204
|
-
function parseSkillMd(text, where) {
|
|
205
|
-
const match = /^---\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)([\s\S]*)$/.exec(text);
|
|
206
|
-
if (!match) return { body: text };
|
|
207
|
-
let front;
|
|
208
|
-
try {
|
|
209
|
-
front = parse(match[1] ?? "");
|
|
210
|
-
} catch (error) {
|
|
211
|
-
const reason = error.message.split("\n")[0];
|
|
212
|
-
throw new InfraError(`${where}: the SKILL.md frontmatter isn't valid YAML (${reason}). Quote values that contain ": ".`);
|
|
213
|
-
}
|
|
214
|
-
if (front != null && (typeof front !== "object" || Array.isArray(front))) throw new InfraError(`${where}: the SKILL.md frontmatter must be a YAML mapping, e.g. name: my-skill and description: … on their own lines`);
|
|
215
|
-
const field = (key) => {
|
|
216
|
-
const value = front?.[key];
|
|
217
|
-
if (value == null) return void 0;
|
|
218
|
-
if (typeof value !== "string") throw new InfraError(`${where}: the SKILL.md ${key} must be a string (got ${typeof value})`);
|
|
219
|
-
return value.trim() || void 0;
|
|
220
|
-
};
|
|
221
|
-
return {
|
|
222
|
-
name: field("name"),
|
|
223
|
-
description: field("description"),
|
|
224
|
-
body: match[2] ?? ""
|
|
225
|
-
};
|
|
226
|
-
}
|
|
227
|
-
/** A gitignore-style test for `DEFAULT_SKILL_EXCLUDE`-shaped patterns against a skill-relative path. */
|
|
228
|
-
function excludeMatcher(patterns) {
|
|
229
|
-
const rules = patterns.map((pattern) => {
|
|
230
|
-
const dirOnly = pattern.endsWith("/");
|
|
231
|
-
const body = pattern.replace(/\/+$/, "");
|
|
232
|
-
const anchored = body.includes("/");
|
|
233
|
-
const glob = body.replace(/^\/+/, "").split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*");
|
|
234
|
-
return {
|
|
235
|
-
dirOnly,
|
|
236
|
-
re: new RegExp(`${anchored ? "^" : "(?:^|/)"}${glob}$`)
|
|
237
|
-
};
|
|
238
|
-
});
|
|
239
|
-
return (path, isDir) => rules.some((r) => (isDir || !r.dirOnly) && r.re.test(path));
|
|
240
|
-
}
|
|
241
|
-
function parseSkill(dir, source, exclude) {
|
|
242
|
-
const front = parseSkillMd(readFileSync(join(dir, "SKILL.md"), "utf8"), `Skill ${basename(dir)} in ${source}`);
|
|
243
|
-
const name = front.name || basename(dir);
|
|
244
|
-
checkSkill(name, front.description, source);
|
|
245
|
-
const skip = excludeMatcher(exclude);
|
|
246
|
-
const paths = [];
|
|
247
|
-
const excluded = [];
|
|
248
|
-
const walk = (rel) => {
|
|
249
|
-
for (const entry of readdirSync(join(dir, rel), { withFileTypes: true })) {
|
|
250
|
-
const path = rel ? `${rel}/${entry.name}` : entry.name;
|
|
251
|
-
const isDir = entry.isDirectory();
|
|
252
|
-
if (entry.name === ".git") continue;
|
|
253
|
-
if (skip(path, isDir)) excluded.push(isDir ? `${path}/` : path);
|
|
254
|
-
else if (isDir) walk(path);
|
|
255
|
-
else if (entry.isFile() && path !== "SKILL.md") paths.push(path);
|
|
256
|
-
}
|
|
257
|
-
};
|
|
258
|
-
walk("");
|
|
259
|
-
const files = paths.sort().map((path) => ({
|
|
260
|
-
path,
|
|
261
|
-
content: readFileSync(join(dir, path), "utf8")
|
|
262
|
-
}));
|
|
263
|
-
return {
|
|
264
|
-
skill: {
|
|
265
|
-
name,
|
|
266
|
-
description: front.description ?? "",
|
|
267
|
-
content: front.body,
|
|
268
|
-
files
|
|
269
|
-
},
|
|
270
|
-
excluded: excluded.sort()
|
|
271
|
-
};
|
|
272
|
-
}
|
|
273
|
-
function checkSkill(name, description, source) {
|
|
274
|
-
if (!SKILL_NAME.test(name)) throw new InfraError(`Skill "${name}" in ${source}: the name must be kebab-case, e.g. my-skill (lower-case letters, digits and -)`);
|
|
275
|
-
if (!description) throw new InfraError(`Skill "${name}" in ${source} has no description in its SKILL.md frontmatter. Add a description: line saying when to use it.`);
|
|
276
|
-
}
|
|
277
|
-
function loadSkills(dir, source, only, exclude = DEFAULT_SKILL_EXCLUDE) {
|
|
278
|
-
if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new InfraError(`Skill source ${source} isn't a directory. Point it at a skill's directory, or at a directory of skills.`);
|
|
279
|
-
const dirs = existsSync(join(dir, "SKILL.md")) ? [dir] : readdirSync(dir, { withFileTypes: true }).filter((e) => e.isDirectory() && existsSync(join(dir, e.name, "SKILL.md"))).map((e) => join(dir, e.name)).sort();
|
|
280
|
-
if (dirs.length === 0) throw new InfraError(`Skill source ${source} has no SKILL.md (and no subdirectories with one)`);
|
|
281
|
-
const skills = dirs.map((d) => parseSkill(d, source, exclude));
|
|
282
|
-
if (!only) return skills;
|
|
283
|
-
const missing = only.filter((n) => !skills.some((s) => s.skill.name === n));
|
|
284
|
-
if (missing.length) throw new InfraError(`only: ${JSON.stringify(missing)} not found in ${source} (found: ${skills.map((s) => s.skill.name).join(", ")})`);
|
|
285
|
-
return skills.filter((s) => only.includes(s.skill.name));
|
|
286
|
-
}
|
|
287
|
-
async function git(args, cwd, what) {
|
|
288
|
-
const r = await run("git", args, { cwd });
|
|
289
|
-
if (r.exitCode !== 0) throw new InfraError(`${what} failed: ${r.stderr.trim()}`);
|
|
290
|
-
return r.stdout.trim();
|
|
291
|
-
}
|
|
292
|
-
/** One clone per source and ref per worker process; the commit is recorded. */
|
|
293
|
-
const clones = /* @__PURE__ */ new Map();
|
|
294
|
-
function cloneSkills(root, url, ref) {
|
|
295
|
-
const key = `${url}#${ref ?? ""}`;
|
|
296
|
-
let clone = clones.get(key);
|
|
297
|
-
if (!clone) {
|
|
298
|
-
clone = (async () => {
|
|
299
|
-
const dir = join(root, ".reeval", "cache", "skills", `${sha256(key).slice(0, 16)}-${process.pid}`);
|
|
300
|
-
await rm(dir, {
|
|
301
|
-
recursive: true,
|
|
302
|
-
force: true
|
|
303
|
-
});
|
|
304
|
-
await mkdir(dir, { recursive: true });
|
|
305
|
-
const what = `Cloning ${url}${ref ? ` at ${ref}` : ""}`;
|
|
306
|
-
await git(["init", "-q"], dir, what);
|
|
307
|
-
await git([
|
|
308
|
-
"fetch",
|
|
309
|
-
"-q",
|
|
310
|
-
"--depth",
|
|
311
|
-
"1",
|
|
312
|
-
url,
|
|
313
|
-
ref ?? "HEAD"
|
|
314
|
-
], dir, what);
|
|
315
|
-
await git([
|
|
316
|
-
"-c",
|
|
317
|
-
"advice.detachedHead=false",
|
|
318
|
-
"checkout",
|
|
319
|
-
"-q",
|
|
320
|
-
"FETCH_HEAD"
|
|
321
|
-
], dir, what);
|
|
322
|
-
return {
|
|
323
|
-
dir,
|
|
324
|
-
commit: await git(["rev-parse", "HEAD"], dir, what)
|
|
325
|
-
};
|
|
326
|
-
})();
|
|
327
|
-
clones.set(key, clone);
|
|
328
|
-
clone.catch(() => clones.delete(key));
|
|
329
|
-
}
|
|
330
|
-
return clone;
|
|
331
|
-
}
|
|
332
|
-
const packs = /* @__PURE__ */ new Map();
|
|
333
|
-
/** `npm pack` into the cache, unpacked; returns the package directory. */
|
|
334
|
-
function unpackNpm(root, spec) {
|
|
335
|
-
let pack = packs.get(spec);
|
|
336
|
-
if (!pack) {
|
|
337
|
-
pack = (async () => {
|
|
338
|
-
const dir = join(root, ".reeval", "cache", "skills", `npm-${sha256(spec).slice(0, 16)}-${process.pid}`);
|
|
339
|
-
await rm(dir, {
|
|
340
|
-
recursive: true,
|
|
341
|
-
force: true
|
|
342
|
-
});
|
|
343
|
-
await mkdir(dir, { recursive: true });
|
|
344
|
-
const packed = await run("npm", [
|
|
345
|
-
"pack",
|
|
346
|
-
spec,
|
|
347
|
-
"--silent",
|
|
348
|
-
"--pack-destination",
|
|
349
|
-
dir
|
|
350
|
-
], { cwd: dir });
|
|
351
|
-
const tarball = packed.stdout.trim().split("\n").at(-1);
|
|
352
|
-
if (packed.exitCode !== 0 || !tarball) throw new InfraError(`npm pack ${spec} failed: ${packed.stderr.trim()}`);
|
|
353
|
-
const untar = await run("tar", ["-xzf", tarball], { cwd: dir });
|
|
354
|
-
if (untar.exitCode !== 0) throw new InfraError(`Unpacking ${spec} failed: ${untar.stderr.trim()}`);
|
|
355
|
-
return join(dir, "package");
|
|
356
|
-
})();
|
|
357
|
-
packs.set(spec, pack);
|
|
358
|
-
pack.catch(() => packs.delete(spec));
|
|
359
|
-
}
|
|
360
|
-
return pack;
|
|
361
|
-
}
|
|
362
|
-
async function resolveSkills(root, s) {
|
|
363
|
-
switch (s.source) {
|
|
364
|
-
case "local": {
|
|
365
|
-
const label = `local:${relative(root, s.path).split(sep).join("/") || "."}`;
|
|
366
|
-
return {
|
|
367
|
-
skills: loadSkills(s.path, label, s.only, s.exclude),
|
|
368
|
-
source: label,
|
|
369
|
-
commit: null
|
|
370
|
-
};
|
|
371
|
-
}
|
|
372
|
-
case "git": {
|
|
373
|
-
const { url } = parseGitRepo(s.repo);
|
|
374
|
-
const { dir, commit } = await cloneSkills(root, url, s.ref);
|
|
375
|
-
const path = s.path ?? (!existsSync(join(dir, "SKILL.md")) && existsSync(join(dir, "skills")) ? "skills" : "");
|
|
376
|
-
const label = `${s.repo}${s.ref ? `#${s.ref}` : ""}${path ? `:${path}` : ""}`;
|
|
377
|
-
return {
|
|
378
|
-
skills: loadSkills(join(dir, path), label, s.only, s.exclude),
|
|
379
|
-
source: label,
|
|
380
|
-
commit
|
|
381
|
-
};
|
|
382
|
-
}
|
|
383
|
-
case "registry": {
|
|
384
|
-
const [repo = "", name = ""] = s.id.split("@");
|
|
385
|
-
const { dir, commit } = await cloneSkills(root, `https://github.com/${repo}.git`, s.version);
|
|
386
|
-
const label = `registry:${s.id}${s.version ? `#${s.version}` : ""}`;
|
|
387
|
-
return {
|
|
388
|
-
skills: loadSkills(existsSync(join(dir, "skills")) ? join(dir, "skills") : dir, label, [name]),
|
|
389
|
-
source: label,
|
|
390
|
-
commit
|
|
391
|
-
};
|
|
392
|
-
}
|
|
393
|
-
case "npm": {
|
|
394
|
-
const dir = await unpackNpm(root, s.spec);
|
|
395
|
-
const label = `npm:${isAbsolute(s.spec) ? relative(root, s.spec).split(sep).join("/") : s.spec}:${s.path}`;
|
|
396
|
-
return {
|
|
397
|
-
skills: loadSkills(join(dir, s.path), label, s.only, s.exclude),
|
|
398
|
-
source: label,
|
|
399
|
-
commit: null
|
|
400
|
-
};
|
|
401
|
-
}
|
|
402
|
-
case "inline": {
|
|
403
|
-
checkSkill(s.name, s.description, "skill.inline");
|
|
404
|
-
const files = Object.entries(s.files).sort(([a], [b]) => a < b ? -1 : 1).map(([path, content]) => ({
|
|
405
|
-
path,
|
|
406
|
-
content
|
|
407
|
-
}));
|
|
408
|
-
return {
|
|
409
|
-
skills: [{
|
|
410
|
-
skill: {
|
|
411
|
-
name: s.name,
|
|
412
|
-
description: s.description,
|
|
413
|
-
content: s.content,
|
|
414
|
-
files
|
|
415
|
-
},
|
|
416
|
-
excluded: []
|
|
417
|
-
}],
|
|
418
|
-
source: "inline",
|
|
419
|
-
commit: null
|
|
420
|
-
};
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
const skillHash = (s) => sha256(JSON.stringify(s));
|
|
425
|
-
const readHostFile = (f, where) => {
|
|
426
|
-
if (!existsSync(f.path)) throw new InfraError(`${f.path} for ${where} doesn't exist. file() paths resolve against the file that calls it, or reeval.yaml's directory.`);
|
|
427
|
-
return readFileSync(f.path);
|
|
428
|
-
};
|
|
429
|
-
/**
|
|
430
|
-
* Layers the environment (root first; a later layer wins on shared keys, and each such key is
|
|
431
|
-
* an override), over the options `env` merged down the cascade. `where` names the attempt, and
|
|
432
|
-
* `rulesFile` is the workspace file the harness's agent reads `rules` from.
|
|
433
|
-
*/
|
|
434
|
-
async function resolveEnvironment(environment, optionsEnv, root, where, rulesFile, egress = "allow-all", prepared = () => void 0) {
|
|
435
|
-
const envWhere = `environment ${environment.name} (${where})`;
|
|
436
|
-
const overrides = /* @__PURE__ */ new Set();
|
|
437
|
-
const env = { ...optionsEnv };
|
|
438
|
-
const files = /* @__PURE__ */ new Map();
|
|
439
|
-
const mcp = /* @__PURE__ */ new Map();
|
|
440
|
-
const skills = /* @__PURE__ */ new Map();
|
|
441
|
-
let rules;
|
|
442
|
-
const setup = [];
|
|
443
|
-
const seen = {
|
|
444
|
-
env: /* @__PURE__ */ new Set(),
|
|
445
|
-
files: /* @__PURE__ */ new Set(),
|
|
446
|
-
mcp: /* @__PURE__ */ new Set()
|
|
447
|
-
};
|
|
448
|
-
let egressSet = false;
|
|
449
|
-
let workspace;
|
|
450
|
-
let sandboxSet = false;
|
|
451
|
-
for (const { spec } of environment.layers) {
|
|
452
|
-
if (spec.sandbox !== void 0) {
|
|
453
|
-
if (sandboxSet) overrides.add("sandbox");
|
|
454
|
-
sandboxSet = true;
|
|
455
|
-
}
|
|
456
|
-
if (spec.workspace !== void 0) {
|
|
457
|
-
if (workspace) overrides.add("workspace");
|
|
458
|
-
workspace = spec.workspace;
|
|
459
|
-
}
|
|
460
|
-
if (spec.egress !== void 0) {
|
|
461
|
-
if (egressSet) overrides.add("egress");
|
|
462
|
-
egressSet = true;
|
|
463
|
-
}
|
|
464
|
-
for (const [k, v] of Object.entries(spec.env ?? {})) {
|
|
465
|
-
if (seen.env.has(k)) overrides.add(`env.${k}`);
|
|
466
|
-
seen.env.add(k);
|
|
467
|
-
env[k] = v;
|
|
468
|
-
}
|
|
469
|
-
for (const [path, v] of Object.entries(spec.files ?? {})) {
|
|
470
|
-
if (seen.files.has(path)) overrides.add(`files.${path}`);
|
|
471
|
-
seen.files.add(path);
|
|
472
|
-
files.set(path, v);
|
|
473
|
-
}
|
|
474
|
-
if (spec.rules !== void 0) {
|
|
475
|
-
if (rules !== void 0) overrides.add("rules");
|
|
476
|
-
rules = spec.rules;
|
|
477
|
-
}
|
|
478
|
-
for (const [name, server] of Object.entries(spec.mcp ?? {})) {
|
|
479
|
-
if (seen.mcp.has(name)) overrides.add(`mcp.${name}`);
|
|
480
|
-
seen.mcp.add(name);
|
|
481
|
-
mcp.set(name, server);
|
|
482
|
-
}
|
|
483
|
-
for (const step of spec.setup ?? []) {
|
|
484
|
-
const s = typeof step === "string" ? { run: step } : step;
|
|
485
|
-
setup.push({
|
|
486
|
-
run: s.run,
|
|
487
|
-
timeoutMs: parseDuration(s.timeout ?? DEFAULT_SETUP_TIMEOUT),
|
|
488
|
-
...s.cwd ? { cwd: s.cwd } : {}
|
|
489
|
-
});
|
|
490
|
-
}
|
|
491
|
-
const layerSkills = /* @__PURE__ */ new Set();
|
|
492
|
-
for (const source of spec.skills ?? []) {
|
|
493
|
-
const found = await resolveSkills(root, source);
|
|
494
|
-
for (const loaded of found.skills) {
|
|
495
|
-
const name = loaded.skill.name;
|
|
496
|
-
if (skills.has(name) && !layerSkills.has(name)) overrides.add(`skills.${name}`);
|
|
497
|
-
layerSkills.add(name);
|
|
498
|
-
skills.delete(name);
|
|
499
|
-
skills.set(name, {
|
|
500
|
-
...loaded,
|
|
501
|
-
placement: source.placement,
|
|
502
|
-
source: found.source,
|
|
503
|
-
commit: found.commit
|
|
504
|
-
});
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
const secrets = {
|
|
509
|
-
names: [],
|
|
510
|
-
checks: [],
|
|
511
|
-
unset: [],
|
|
512
|
-
brokered: []
|
|
513
|
-
};
|
|
514
|
-
const vars = {};
|
|
515
|
-
const checkVars = {};
|
|
516
|
-
for (const [name, value] of Object.entries(env)) {
|
|
517
|
-
if (!isSecret(value)) {
|
|
518
|
-
vars[name] = value;
|
|
519
|
-
continue;
|
|
520
|
-
}
|
|
521
|
-
const resolved = hostValue(name, value, envWhere, secrets);
|
|
522
|
-
if (resolved === void 0) continue;
|
|
523
|
-
if (value.to === "checks") checkVars[name] = resolved;
|
|
524
|
-
else vars[name] = resolved;
|
|
525
|
-
}
|
|
526
|
-
const secretOf = (value, key, hosts) => {
|
|
527
|
-
if (typeof value === "string") return value;
|
|
528
|
-
if (isSecretTemplate(value)) return template(value, envWhere, secrets, hosts);
|
|
529
|
-
return hostValue(key, value, envWhere, secrets, value.hosts ?? hosts);
|
|
530
|
-
};
|
|
531
|
-
const resolvedMcp = {};
|
|
532
|
-
for (const [name, s] of mcp) if (s.transport === "stdio") {
|
|
533
|
-
const e = {};
|
|
534
|
-
for (const [k, v] of Object.entries(s.env)) {
|
|
535
|
-
const r = secretOf(v, k);
|
|
536
|
-
if (r !== void 0) e[k] = r;
|
|
537
|
-
}
|
|
538
|
-
resolvedMcp[name] = {
|
|
539
|
-
transport: "stdio",
|
|
540
|
-
command: s.command,
|
|
541
|
-
args: s.args,
|
|
542
|
-
env: e
|
|
543
|
-
};
|
|
544
|
-
} else {
|
|
545
|
-
const h = {};
|
|
546
|
-
const hosts = mcpHosts(s.url);
|
|
547
|
-
for (const [k, v] of Object.entries(s.headers)) {
|
|
548
|
-
const r = secretOf(v, k, hosts);
|
|
549
|
-
if (r !== void 0) h[k] = r;
|
|
550
|
-
}
|
|
551
|
-
resolvedMcp[name] = {
|
|
552
|
-
transport: s.transport,
|
|
553
|
-
url: s.url,
|
|
554
|
-
headers: h
|
|
555
|
-
};
|
|
556
|
-
}
|
|
557
|
-
const fileList = [...files].map(([path, v]) => ({
|
|
558
|
-
path,
|
|
559
|
-
content: isHostFile(v) ? readHostFile(v, envWhere) : v,
|
|
560
|
-
...isHostFile(v) && statSync(v.path).mode & 64 ? { executable: true } : {}
|
|
561
|
-
}));
|
|
562
|
-
const rulesContent = rules === void 0 ? void 0 : isHostFile(rules) ? Buffer.from(readHostFile(rules, envWhere)).toString("utf8") : rules;
|
|
563
|
-
const skillList = [...skills.values()];
|
|
564
|
-
const skillRecords = skillList.map(({ skill, placement, source, commit, excluded }) => ({
|
|
565
|
-
name: skill.name,
|
|
566
|
-
source,
|
|
567
|
-
commit,
|
|
568
|
-
sha256: skillHash(skill),
|
|
569
|
-
placement,
|
|
570
|
-
excluded
|
|
571
|
-
}));
|
|
572
|
-
const fileRecords = fileList.map((f) => ({
|
|
573
|
-
path: f.path,
|
|
574
|
-
sha256: sha256(f.content)
|
|
575
|
-
}));
|
|
576
|
-
const executable = fileList.flatMap((f) => f.executable ? [f.path] : []);
|
|
577
|
-
const rulesRecord = rulesContent === void 0 ? void 0 : {
|
|
578
|
-
path: rulesFile,
|
|
579
|
-
sha256: sha256(rulesContent)
|
|
580
|
-
};
|
|
581
|
-
if (rulesContent !== void 0) fileList.push({
|
|
582
|
-
path: rulesFile,
|
|
583
|
-
content: rulesContent
|
|
584
|
-
});
|
|
585
|
-
const secretNames = [.../* @__PURE__ */ new Set([...secrets.names, ...secrets.checks])].sort();
|
|
586
|
-
const brokered = secrets.brokered.map((b) => JSON.stringify([
|
|
587
|
-
b.name,
|
|
588
|
-
b.hosts,
|
|
589
|
-
b.in,
|
|
590
|
-
b.phase
|
|
591
|
-
])).sort();
|
|
592
|
-
const brokeredNames = [...new Set(secrets.brokered.map((b) => b.name))].sort();
|
|
593
|
-
const varNames = Object.keys(vars).filter((n) => !secrets.names.includes(n)).sort();
|
|
594
|
-
const literalVars = Object.fromEntries(varNames.map((n) => [n, vars[n]]));
|
|
595
|
-
const mcpShape = Object.fromEntries(Object.entries(resolvedMcp).map(([name, s]) => s.transport === "stdio" ? [name, {
|
|
596
|
-
command: s.command,
|
|
597
|
-
args: s.args,
|
|
598
|
-
env: Object.keys(s.env).sort()
|
|
599
|
-
}] : [name, {
|
|
600
|
-
transport: s.transport,
|
|
601
|
-
url: s.url,
|
|
602
|
-
headers: Object.keys(s.headers).sort()
|
|
603
|
-
}]));
|
|
604
|
-
const tree = workspace && prepared(workspace);
|
|
605
|
-
if (workspace && !tree) throw new InfraError(`The workspace git(${JSON.stringify(workspaceLabel(workspace))}) for ${envWhere} wasn't prepared before the run started`);
|
|
606
|
-
const workspaceRecord = workspace && tree ? {
|
|
607
|
-
source: isAbsolute(workspace.url) ? relative(root, workspace.url).split(sep).join("/") || "." : workspace.repo,
|
|
608
|
-
ref: workspace.ref ?? null,
|
|
609
|
-
path: workspace.path ?? null,
|
|
610
|
-
commit: tree.commit,
|
|
611
|
-
tree: tree.tree,
|
|
612
|
-
...workspace.init ? { init: true } : {}
|
|
613
|
-
} : void 0;
|
|
614
|
-
const digest = `sha256:${sha256(JSON.stringify({
|
|
615
|
-
setup,
|
|
616
|
-
skills: skillRecords.map((s) => [
|
|
617
|
-
s.name,
|
|
618
|
-
s.sha256,
|
|
619
|
-
s.placement
|
|
620
|
-
]),
|
|
621
|
-
mcp: mcpShape,
|
|
622
|
-
files: fileRecords,
|
|
623
|
-
...executable.length ? { executable } : {},
|
|
624
|
-
rules: rulesRecord?.sha256,
|
|
625
|
-
vars: literalVars,
|
|
626
|
-
secrets: secretNames,
|
|
627
|
-
checks: [...secrets.checks].sort(),
|
|
628
|
-
unset: [...secrets.unset].sort(),
|
|
629
|
-
brokered,
|
|
630
|
-
...egress === "allow-all" ? {} : { egress: typeof egress === "string" ? egress : [...egress.allow].sort() },
|
|
631
|
-
...workspaceRecord ? { workspace: workspaceRecord.init ? [workspaceRecord.tree, "init"] : workspaceRecord.tree } : {}
|
|
632
|
-
}))}`;
|
|
633
|
-
return {
|
|
634
|
-
record: {
|
|
635
|
-
name: environment.name,
|
|
636
|
-
parent: environment.parent,
|
|
637
|
-
overrides: [...overrides],
|
|
638
|
-
digest,
|
|
639
|
-
setup: setup.map(stepCommand),
|
|
640
|
-
skills: skillRecords,
|
|
641
|
-
mcp: Object.keys(resolvedMcp).sort(),
|
|
642
|
-
files: rulesRecord ? [...fileRecords, rulesRecord] : fileRecords,
|
|
643
|
-
vars: varNames,
|
|
644
|
-
secrets: secretNames,
|
|
645
|
-
brokered: brokeredNames,
|
|
646
|
-
unset: [...secrets.unset].sort(),
|
|
647
|
-
egress,
|
|
648
|
-
...workspaceRecord ? { workspace: workspaceRecord } : {}
|
|
649
|
-
},
|
|
650
|
-
setup,
|
|
651
|
-
skills: skillList.filter((s) => s.placement === "harness").map((s) => s.skill),
|
|
652
|
-
workspaceSkills: skillList.filter((s) => s.placement === "workspace").map((s) => s.skill),
|
|
653
|
-
mcp: resolvedMcp,
|
|
654
|
-
files: fileList,
|
|
655
|
-
vars,
|
|
656
|
-
checkVars,
|
|
657
|
-
substitutions: secrets.brokered,
|
|
658
|
-
...workspace && tree ? { workspace: {
|
|
659
|
-
tar: tree.tar,
|
|
660
|
-
label: workspaceLabel(workspace),
|
|
661
|
-
commit: tree.commit,
|
|
662
|
-
...workspace.init ? { init: true } : {}
|
|
663
|
-
} } : {}
|
|
664
|
-
};
|
|
665
|
-
}
|
|
666
|
-
/** A skill as a `SKILL.md` file plus its other files, for writing into a directory. */
|
|
667
|
-
function skillFiles(skill) {
|
|
668
|
-
return [{
|
|
669
|
-
path: "SKILL.md",
|
|
670
|
-
content: `${`---\nname: ${skill.name}\ndescription: ${JSON.stringify(skill.description)}\n---\n`}${skill.content}`
|
|
671
|
-
}, ...skill.files];
|
|
672
|
-
}
|
|
673
|
-
//#endregion
|
|
674
162
|
//#region src/harness/fake.ts
|
|
675
163
|
const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
|
|
676
164
|
/**
|
|
677
165
|
* Writes canned files and returns a canned message on every turn, after running its `script`'s
|
|
678
166
|
* real sandbox actions. Harness-placed skills are written under `~/` where its layout's adapter
|
|
679
|
-
* puts them (`~/.agents/skills` by default, as Cursor's does).
|
|
167
|
+
* puts them (`~/.agents/skills` by default, as Cursor's does). Plugins are already copied under
|
|
168
|
+
* `~/.reeval/plugins/`; where its layout takes plugin items flat, their skills arrive in `skills`.
|
|
169
|
+
* Where it takes them namespaced, it writes its layout's plugin files (Claude Code's settings)
|
|
170
|
+
* and leaves the plugins' MCP servers to the plugins, as that agent does.
|
|
680
171
|
*/
|
|
681
172
|
const createFakeHarness = (descriptor) => {
|
|
682
173
|
let starts = 0;
|
|
683
|
-
return { async start(sandbox, _signal, { skills, workspaceSkills, mcp }) {
|
|
174
|
+
return { async start(sandbox, _signal, { skills, workspaceSkills, mcp: all, plugins }) {
|
|
684
175
|
if (descriptor.failWith) throw new InfraError(descriptor.failWith);
|
|
685
176
|
const hangs = descriptor.startHangs;
|
|
686
177
|
if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
|
|
687
|
-
const
|
|
178
|
+
const layout = layoutOf(descriptor);
|
|
179
|
+
const mcp = agentMcp(layout, all, plugins);
|
|
180
|
+
for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
|
|
181
|
+
const dir = `~/${layout.harnessSkillDir}`;
|
|
688
182
|
for (const skill of skills) {
|
|
689
183
|
for (const f of skillFiles(skill)) await sandbox.writeFile(`${dir}/${skill.name}/${f.path}`, f.content);
|
|
690
184
|
debugLog("env", `skill ${skill.name} placed at ${dir}/${skill.name}/ by the fake agent`);
|
|
691
185
|
}
|
|
692
186
|
const names = [...skills.map((s) => s.name), ...workspaceSkills];
|
|
693
|
-
const loaded = [
|
|
187
|
+
const loaded = [
|
|
188
|
+
names.length ? `skills: ${names.join(", ")}` : "",
|
|
189
|
+
plugins.length ? `plugins: ${plugins.map((p) => `${p.name} (${p.parts.join(", ")})`).join(", ")}` : "",
|
|
190
|
+
Object.keys(mcp).length ? `mcp: ${Object.keys(mcp).join(", ")}` : ""
|
|
191
|
+
].filter(Boolean);
|
|
694
192
|
let calls = 0;
|
|
695
193
|
let turns = 0;
|
|
696
194
|
const skilled = names.length > 0 && descriptor.withSkill;
|
|
@@ -879,7 +377,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
879
377
|
" trap '' PIPE",
|
|
880
378
|
` printf '%s\\n' ${messages.join(" ")} 2>/dev/null`,
|
|
881
379
|
" i=0",
|
|
882
|
-
` while [ $i -lt
|
|
380
|
+
` while [ $i -lt 900 ] && [ ! -f "$d/code" ] && ! grep -Eq '"id": ?2[,}]' "$d/out"; do sleep 0.1; i=$((i + 1)); done`,
|
|
883
381
|
`} | { ${command} > "$d/out" 2> "$d/err"; echo $? > "$d/code"; }`,
|
|
884
382
|
"code=$(cat \"$d/code\")",
|
|
885
383
|
"cat \"$d/out\"; cat \"$d/err\" >&2; rm -rf \"$d\"; exit $code"
|
|
@@ -912,7 +410,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
912
410
|
//#region src/harness/index.ts
|
|
913
411
|
async function createHarness(descriptor) {
|
|
914
412
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
915
|
-
const { createAgentHarness } = await import("./agent-
|
|
413
|
+
const { createAgentHarness } = await import("./agent-CatUEo5h.js");
|
|
916
414
|
return createAgentHarness(descriptor);
|
|
917
415
|
}
|
|
918
416
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
@@ -926,7 +424,10 @@ const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir
|
|
|
926
424
|
*/
|
|
927
425
|
async function prepareEnvironment(a, root, runId, where, signal) {
|
|
928
426
|
const { spec, record } = a;
|
|
929
|
-
const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w)
|
|
427
|
+
const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w), {
|
|
428
|
+
harness: spec.harness,
|
|
429
|
+
allowed: allowedHosts(spec.egress)
|
|
430
|
+
});
|
|
930
431
|
a.env = env;
|
|
931
432
|
record.environment = env.record;
|
|
932
433
|
const { secrets, brokered } = env.record;
|
|
@@ -939,13 +440,29 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
939
440
|
if (debug) traceEnvironment(debug, spec, env);
|
|
940
441
|
record.image = null;
|
|
941
442
|
if (spec.sandbox.kind === "docker") {
|
|
942
|
-
const { preparedFor } = await import("./preflight-
|
|
943
|
-
const { attemptImage, dockerProvider } = await import("./providers-
|
|
443
|
+
const { preparedFor } = await import("./preflight-B6pY89bF.js");
|
|
444
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-CXN2RFz-.js");
|
|
944
445
|
signal.throwIfAborted();
|
|
945
|
-
const
|
|
446
|
+
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
447
|
+
const builds = builtPlugins(spec.environment.layers);
|
|
448
|
+
const prepared = preparedFor(root, runId, provider, spec.sandbox, spec.harness, builds);
|
|
946
449
|
if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
|
|
947
450
|
record.image = attemptImage(prepared);
|
|
948
451
|
a.image = record.image.ref;
|
|
452
|
+
const built = preparedImage(prepared).plugins ?? [];
|
|
453
|
+
for (const [i, plugin] of env.plugins.entries()) {
|
|
454
|
+
if (plugin.declared.build === void 0) continue;
|
|
455
|
+
const found = built.find((b) => b.id === pluginBuildId(plugin.declared));
|
|
456
|
+
if (!found) throw new InfraError(`The ${plugin.name} plugin's build for ${where} wasn't prepared before the run started`);
|
|
457
|
+
if (found.commit !== plugin.commit) throw new InfraError(`The ${plugin.name} plugin's ref moved during the run: it was built at ${found.commit?.slice(0, 12)} and is now ${plugin.commit?.slice(0, 12)}. Pin ref to a commit or tag.`);
|
|
458
|
+
plugin.built = {
|
|
459
|
+
dir: found.dir,
|
|
460
|
+
tag: found.tag
|
|
461
|
+
};
|
|
462
|
+
const pluginRecord = env.record.plugins?.[i];
|
|
463
|
+
if (pluginRecord) pluginRecord.build = found.tag;
|
|
464
|
+
debugLog("image", `plugin ${plugin.name} built in ${found.tag}`);
|
|
465
|
+
}
|
|
949
466
|
const { ref, id, base, install, agentCli } = record.image;
|
|
950
467
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
951
468
|
} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
@@ -1240,7 +757,7 @@ async function createSandbox(descriptor, options) {
|
|
|
1240
757
|
switch (descriptor.kind) {
|
|
1241
758
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
1242
759
|
case "docker": {
|
|
1243
|
-
const { createDockerSandbox } = await import("./docker-
|
|
760
|
+
const { createDockerSandbox } = await import("./docker-DC69dKxB.js");
|
|
1244
761
|
return await createDockerSandbox(descriptor, options);
|
|
1245
762
|
}
|
|
1246
763
|
}
|
|
@@ -1319,11 +836,12 @@ async function recordOom(a, sandbox) {
|
|
|
1319
836
|
}
|
|
1320
837
|
/**
|
|
1321
838
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
1322
|
-
* Refused hosts the harness always tries
|
|
839
|
+
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
840
|
+
* summarised in one `-v` line.
|
|
1323
841
|
*/
|
|
1324
842
|
function recordEgress(a, sandbox) {
|
|
1325
843
|
const traffic = sandbox?.traffic?.();
|
|
1326
|
-
const expected = expectedBlockedFor(a.spec.harness);
|
|
844
|
+
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0);
|
|
1327
845
|
const refused = traffic?.blocked ?? [];
|
|
1328
846
|
const isExpected = (h) => !!expected?.hosts.includes(h.host);
|
|
1329
847
|
const expectedBlocked = refused.filter(isExpected);
|
|
@@ -1374,6 +892,118 @@ function turnErrors(transcripts) {
|
|
|
1374
892
|
});
|
|
1375
893
|
}
|
|
1376
894
|
//#endregion
|
|
895
|
+
//#region src/harness/unpack.ts
|
|
896
|
+
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
897
|
+
const FRONTMATTER = /^\uFEFF?---\r?\n/;
|
|
898
|
+
const FRONTMATTER_BLOCK = /^\uFEFF?---\r?\n(?:---|([\s\S]*?)\r?\n---)[ \t]*(?:\r?\n|$)/;
|
|
899
|
+
const RULE = /^rules\/([^/]+)\.mdc?$/;
|
|
900
|
+
const MARKDOWN = (dir) => new RegExp(`^${dir}/([^/]+)\\.md$`);
|
|
901
|
+
/** A subagent's or command's Markdown with its frontmatter rewritten by `format`. */
|
|
902
|
+
function reformatted(text, part, what, format) {
|
|
903
|
+
const block = FRONTMATTER_BLOCK.exec(text);
|
|
904
|
+
let front = {};
|
|
905
|
+
if (block) {
|
|
906
|
+
try {
|
|
907
|
+
front = parse(block[1] ?? "") ?? {};
|
|
908
|
+
} catch {
|
|
909
|
+
throw new InfraError(`${what}'s frontmatter isn't valid YAML`);
|
|
910
|
+
}
|
|
911
|
+
if (!isRecord(front)) throw new InfraError(`${what}'s frontmatter isn't a YAML mapping`);
|
|
912
|
+
}
|
|
913
|
+
const body = block ? text.slice(block[0].length).replace(/^\r?\n/, "") : text.replace(/^\uFEFF/, "");
|
|
914
|
+
const out = format(part, front);
|
|
915
|
+
const yaml = Object.keys(out).length ? stringify(out, { lineWidth: 0 }) : "";
|
|
916
|
+
return yaml ? `---\n${yaml}---\n\n${body}` : body;
|
|
917
|
+
}
|
|
918
|
+
/** Cursor's handlers for one plugin's `hooks/hooks.json`, by Cursor's event name. */
|
|
919
|
+
function hookHandlers(plugin, text, hooks) {
|
|
920
|
+
const what = `the ${plugin.name} plugin's hooks/hooks.json`;
|
|
921
|
+
let raw;
|
|
922
|
+
try {
|
|
923
|
+
raw = JSON.parse(text);
|
|
924
|
+
} catch {
|
|
925
|
+
throw new InfraError(`${what} isn't valid JSON`);
|
|
926
|
+
}
|
|
927
|
+
const events = isRecord(raw) && isRecord(raw.hooks) ? raw.hooks : {};
|
|
928
|
+
const out = {};
|
|
929
|
+
for (const [event, groups] of Object.entries(events)) {
|
|
930
|
+
const named = hooks.events[event];
|
|
931
|
+
if (!named) throw new InfraError(`${what} has ${event} hooks, which the agent has no event for`);
|
|
932
|
+
for (const group of Array.isArray(groups) ? groups.filter(isRecord) : []) {
|
|
933
|
+
const matcher = hooks.matcher(event, typeof group.matcher === "string" ? group.matcher : void 0);
|
|
934
|
+
if (matcher === null) continue;
|
|
935
|
+
for (const h of Array.isArray(group.hooks) ? group.hooks.filter(isRecord) : []) {
|
|
936
|
+
if ((h.type ?? "command") !== "command" || typeof h.command !== "string") throw new InfraError(`${what} has a ${event} hook that isn't a command`);
|
|
937
|
+
out[named] ??= [];
|
|
938
|
+
out[named].push({
|
|
939
|
+
type: "command",
|
|
940
|
+
command: h.command.replaceAll(PLUGIN_ROOT, plugin.dir),
|
|
941
|
+
...typeof h.timeout === "number" ? { timeout: h.timeout } : {},
|
|
942
|
+
...matcher === void 0 ? {} : { matcher }
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
return out;
|
|
948
|
+
}
|
|
949
|
+
/**
|
|
950
|
+
* The workspace files that give a flat agent the plugins' rules, subagents, commands and hooks,
|
|
951
|
+
* by path relative to the workspace: each part `unpack` names and the plugin keeps, with
|
|
952
|
+
* `${CLAUDE_PLUGIN_ROOT}` filled in with the plugin's `dir`. The hooks of every plugin go in one
|
|
953
|
+
* file, after the handlers of `hooksFile` (the workspace's own, or null).
|
|
954
|
+
*/
|
|
955
|
+
function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
956
|
+
const out = [];
|
|
957
|
+
let hooks;
|
|
958
|
+
for (const plugin of plugins) {
|
|
959
|
+
const keeps = (part, dir) => dir !== void 0 && plugin.parts.includes(part);
|
|
960
|
+
for (const f of plugin.files) {
|
|
961
|
+
const text = () => Buffer.from(f.content).toString("utf8").replaceAll(PLUGIN_ROOT, plugin.dir);
|
|
962
|
+
const rule = RULE.exec(f.path);
|
|
963
|
+
if (rule && keeps("rules", unpack.rules)) {
|
|
964
|
+
const body = text();
|
|
965
|
+
out.push({
|
|
966
|
+
path: `${unpack.rules}/${plugin.name}-${rule[1]}.mdc`,
|
|
967
|
+
content: FRONTMATTER.test(body) ? body : `---\nalwaysApply: true\n---\n\n${body}`
|
|
968
|
+
});
|
|
969
|
+
continue;
|
|
970
|
+
}
|
|
971
|
+
for (const [part, dir] of [["agents", unpack.agents], ["commands", unpack.commands]]) {
|
|
972
|
+
const name = MARKDOWN(part).exec(f.path)?.[1];
|
|
973
|
+
if (!name || !keeps(part, dir)) continue;
|
|
974
|
+
const what = `the ${plugin.name} plugin's ${part === "agents" ? "subagent" : "command"} ${f.path}`;
|
|
975
|
+
out.push({
|
|
976
|
+
path: `${dir}/${name}.md`,
|
|
977
|
+
content: unpack.frontmatter ? reformatted(text(), part, what, unpack.frontmatter) : text()
|
|
978
|
+
});
|
|
979
|
+
}
|
|
980
|
+
if (f.path === "hooks/hooks.json" && unpack.hooks && plugin.parts.includes("hooks")) {
|
|
981
|
+
if (!hooks) {
|
|
982
|
+
let own = {};
|
|
983
|
+
try {
|
|
984
|
+
own = hooksFile === null ? {} : JSON.parse(hooksFile);
|
|
985
|
+
} catch {
|
|
986
|
+
throw new InfraError(`The workspace's ${unpack.hooks.file} isn't valid JSON, so reeval can't add the plugins' hooks to it`);
|
|
987
|
+
}
|
|
988
|
+
hooks = isRecord(own) ? {
|
|
989
|
+
version: 1,
|
|
990
|
+
...own
|
|
991
|
+
} : { version: 1 };
|
|
992
|
+
}
|
|
993
|
+
const merged = isRecord(hooks.hooks) ? { ...hooks.hooks } : {};
|
|
994
|
+
const raw = Buffer.from(f.content).toString("utf8");
|
|
995
|
+
for (const [event, handlers] of Object.entries(hookHandlers(plugin, raw, unpack.hooks))) merged[event] = [...Array.isArray(merged[event]) ? merged[event] : [], ...handlers];
|
|
996
|
+
hooks.hooks = merged;
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
if (hooks && unpack.hooks) out.push({
|
|
1001
|
+
path: unpack.hooks.file,
|
|
1002
|
+
content: `${JSON.stringify(hooks, null, 2)}\n`
|
|
1003
|
+
});
|
|
1004
|
+
return out;
|
|
1005
|
+
}
|
|
1006
|
+
//#endregion
|
|
1377
1007
|
//#region src/sandbox/phases.ts
|
|
1378
1008
|
const strip = ({ phase: _, ...sub }) => sub;
|
|
1379
1009
|
/**
|
|
@@ -1403,6 +1033,70 @@ function brokerPhases(box, subs) {
|
|
|
1403
1033
|
};
|
|
1404
1034
|
}
|
|
1405
1035
|
//#endregion
|
|
1036
|
+
//#region src/attempt/mcp-startup.ts
|
|
1037
|
+
/**
|
|
1038
|
+
* The timer and its log beside the workspace: `/reeval/bin/mcp-timer.cjs` in Docker. `.cjs`, since
|
|
1039
|
+
* a sandbox's directories may be in a `"type": "module"` package.
|
|
1040
|
+
*/
|
|
1041
|
+
const TIMER = "../bin/mcp-timer.cjs";
|
|
1042
|
+
const LOG = "../mcp-startup.jsonl";
|
|
1043
|
+
/** The attempt's error when an MCP server never answered `initialize`. */
|
|
1044
|
+
const mcpTimeoutMessage = (server) => `MCP server ${server} didn't answer initialize within 60s`;
|
|
1045
|
+
/**
|
|
1046
|
+
* Puts the timer in the sandbox and runs every stdio server through it, so whichever process
|
|
1047
|
+
* starts a server (the agent's CLI, once a session or once a turn, or the fake agent) logs when it
|
|
1048
|
+
* answered. Returns the servers unchanged, and no log, when none is stdio, or with `untimed`'s
|
|
1049
|
+
* reason when the sandbox has no `node` (an `image` used as is), rather than break them.
|
|
1050
|
+
*/
|
|
1051
|
+
async function timeMcpServers(box, servers, signal) {
|
|
1052
|
+
if (!Object.values(servers).some((s) => s.transport === "stdio")) return { servers };
|
|
1053
|
+
const found = await box.exec("command -v node", { signal });
|
|
1054
|
+
const node = found.exitCode === 0 ? found.stdout.trim().split("\n")[0] : "";
|
|
1055
|
+
if (!node) return {
|
|
1056
|
+
servers,
|
|
1057
|
+
untimed: "the sandbox has no node to run the timer"
|
|
1058
|
+
};
|
|
1059
|
+
await box.writeFile(TIMER, await readFile(MCP_TIMER, "utf8"));
|
|
1060
|
+
await box.writeFile(LOG, "");
|
|
1061
|
+
const timer = `${box.workdir.replace(/[\\/][^\\/]*$/, "")}/bin/mcp-timer.cjs`;
|
|
1062
|
+
const wrapped = Object.fromEntries(Object.entries(servers).map(([name, s]) => [name, s.transport === "stdio" ? {
|
|
1063
|
+
...s,
|
|
1064
|
+
command: node,
|
|
1065
|
+
args: [
|
|
1066
|
+
timer,
|
|
1067
|
+
"--name",
|
|
1068
|
+
name,
|
|
1069
|
+
"--",
|
|
1070
|
+
s.command,
|
|
1071
|
+
...s.args
|
|
1072
|
+
]
|
|
1073
|
+
} : s]));
|
|
1074
|
+
let seen = 0;
|
|
1075
|
+
return {
|
|
1076
|
+
servers: wrapped,
|
|
1077
|
+
log: { async read() {
|
|
1078
|
+
const complete = (await box.readFile(LOG).catch(() => null) ?? "").split("\n").slice(0, -1);
|
|
1079
|
+
const fresh = complete.slice(seen);
|
|
1080
|
+
seen = complete.length;
|
|
1081
|
+
return fresh.flatMap((line) => {
|
|
1082
|
+
try {
|
|
1083
|
+
const entry = JSON.parse(line);
|
|
1084
|
+
return typeof entry.server === "string" ? [entry] : [];
|
|
1085
|
+
} catch {
|
|
1086
|
+
return [];
|
|
1087
|
+
}
|
|
1088
|
+
});
|
|
1089
|
+
} }
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
/** Each server start in `entries`, with how long it took to answer `initialize`; null when it didn't. */
|
|
1093
|
+
const mcpStartupOf = (entries) => entries.map((e) => ({
|
|
1094
|
+
server: e.server,
|
|
1095
|
+
ms: e.initializedAt === void 0 ? null : Math.max(0, e.initializedAt - e.spawnedAt)
|
|
1096
|
+
}));
|
|
1097
|
+
/** The first server in `entries` that never answered `initialize`. */
|
|
1098
|
+
const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
1099
|
+
//#endregion
|
|
1406
1100
|
//#region src/attempt/state.ts
|
|
1407
1101
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1408
1102
|
const SETUP_BUDGET_MS = 12e5;
|
|
@@ -1563,6 +1257,131 @@ async function copyWorkspace(box, workspace, signal) {
|
|
|
1563
1257
|
debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
|
|
1564
1258
|
}
|
|
1565
1259
|
/**
|
|
1260
|
+
* A built plugin's directory, copied from where its layer left it in the image, less the parts
|
|
1261
|
+
* `only` leaves out, with the manifest as reeval read it.
|
|
1262
|
+
*/
|
|
1263
|
+
async function copyBuiltPlugin(a, box, plugin, dir) {
|
|
1264
|
+
const dropped = droppedPaths(plugin.parts).map((p) => shellQuote(`${dir}/${p}`));
|
|
1265
|
+
const r = await box.exec(`mkdir -p ${shellQuote(dir)} && cp -a ${shellQuote(plugin.built.dir)}/. ${shellQuote(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
|
|
1266
|
+
if (r.exitCode !== 0) throw new InfraError(`Couldn't copy the ${plugin.name} plugin's build from ${plugin.built.tag}: ${tailOf(r.stderr || r.stdout)}`);
|
|
1267
|
+
const manifest = plugin.files.find((f) => f.path === PLUGIN_MANIFEST);
|
|
1268
|
+
if (manifest) await box.writeFile(`${dir}/${PLUGIN_MANIFEST}`, manifest.content);
|
|
1269
|
+
}
|
|
1270
|
+
/**
|
|
1271
|
+
* Copies each plugin to `PLUGIN_DIR/<name>/` before the workspace snapshot (a built one from its
|
|
1272
|
+
* layer in the image, any other from the host), and points its MCP servers' `${CLAUDE_PLUGIN_ROOT}`
|
|
1273
|
+
* at that directory, for the adapter to load from there.
|
|
1274
|
+
*/
|
|
1275
|
+
async function copyPlugins(a, box, env) {
|
|
1276
|
+
const out = [];
|
|
1277
|
+
for (const plugin of env.plugins) {
|
|
1278
|
+
const at = `${PLUGIN_DIR}/${plugin.name}`;
|
|
1279
|
+
const dir = `${box.home}/${at.slice(2)}`;
|
|
1280
|
+
const { built } = plugin;
|
|
1281
|
+
if (built) await copyBuiltPlugin(a, box, {
|
|
1282
|
+
...plugin,
|
|
1283
|
+
built
|
|
1284
|
+
}, dir);
|
|
1285
|
+
else for (const f of plugin.files) await box.writeFile(`${at}/${f.path}`, f.content);
|
|
1286
|
+
const executable = built ? [] : plugin.files.flatMap((f) => f.executable ? [`${dir}/${f.path}`] : []);
|
|
1287
|
+
if (executable.length) {
|
|
1288
|
+
const r = await box.exec(`chmod +x ${executable.map(shellQuote).join(" ")}`, { signal: a.signal });
|
|
1289
|
+
if (r.exitCode !== 0) throw new InfraError(`Couldn't make the ${plugin.name} plugin's ${executable.length} executable files executable: ${tailOf(r.stderr || r.stdout)}`);
|
|
1290
|
+
}
|
|
1291
|
+
const root = (v) => v.replaceAll(PLUGIN_ROOT, dir);
|
|
1292
|
+
for (const key of plugin.servers) {
|
|
1293
|
+
const server = env.mcp[key];
|
|
1294
|
+
if (server?.transport !== "stdio") continue;
|
|
1295
|
+
env.mcp[key] = {
|
|
1296
|
+
...server,
|
|
1297
|
+
command: root(server.command),
|
|
1298
|
+
args: server.args.map(root),
|
|
1299
|
+
env: {
|
|
1300
|
+
...Object.fromEntries(Object.entries(server.env).map(([k, v]) => [k, root(v)])),
|
|
1301
|
+
[PLUGIN_ROOT_VAR]: dir
|
|
1302
|
+
}
|
|
1303
|
+
};
|
|
1304
|
+
}
|
|
1305
|
+
out.push({
|
|
1306
|
+
name: plugin.name,
|
|
1307
|
+
dir,
|
|
1308
|
+
parts: plugin.parts,
|
|
1309
|
+
servers: plugin.servers
|
|
1310
|
+
});
|
|
1311
|
+
debugLog("env", `plugin ${plugin.name} → ${at}/ (${built ? `built in ${built.tag}` : `${plugin.files.length} files`}; ${plugin.parts.join(", ")})`);
|
|
1312
|
+
}
|
|
1313
|
+
return out;
|
|
1314
|
+
}
|
|
1315
|
+
/**
|
|
1316
|
+
* The parts a flat agent has unpacked, as a built plugin's copy in the sandbox has them: its
|
|
1317
|
+
* build's output, which `files` (the source before the build) may not match. Builds run on Docker
|
|
1318
|
+
* only, so the sandbox has `find`.
|
|
1319
|
+
*/
|
|
1320
|
+
async function unpackedFromCopy(a, box, dir) {
|
|
1321
|
+
const r = await box.exec(`cd ${shellQuote(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
|
|
1322
|
+
const out = [];
|
|
1323
|
+
for (const path of r.stdout.split("\n").filter(Boolean)) {
|
|
1324
|
+
const content = await box.readFile(`${dir}/${path}`);
|
|
1325
|
+
if (content !== null) out.push({
|
|
1326
|
+
path,
|
|
1327
|
+
content: Buffer.from(content)
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
return out;
|
|
1331
|
+
}
|
|
1332
|
+
/**
|
|
1333
|
+
* Unpacks the plugins' rules, subagents, commands and hooks into the workspace where a flat agent
|
|
1334
|
+
* reads them, before the snapshot, so the diff leaves them out. A file the workspace already has
|
|
1335
|
+
* is an infra error, except the hooks file, whose own handlers the plugins' join.
|
|
1336
|
+
*/
|
|
1337
|
+
async function writeUnpackedPlugins(a, box, env, plugins, unpack) {
|
|
1338
|
+
const placed = [];
|
|
1339
|
+
for (const p of plugins) {
|
|
1340
|
+
const resolved = env.plugins.find((r) => r.name === p.name);
|
|
1341
|
+
const files = resolved?.built ? await unpackedFromCopy(a, box, p.dir) : resolved?.files ?? [];
|
|
1342
|
+
placed.push({
|
|
1343
|
+
...p,
|
|
1344
|
+
files
|
|
1345
|
+
});
|
|
1346
|
+
}
|
|
1347
|
+
const files = unpackPlugins(placed, unpack, unpack.hooks ? await box.readFile(unpack.hooks.file) : null);
|
|
1348
|
+
const fresh = files.filter((f) => f.path !== unpack.hooks?.file);
|
|
1349
|
+
if (fresh.length) {
|
|
1350
|
+
const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
|
|
1351
|
+
if (taken.length) throw new InfraError(`The workspace already has ${taken.join(", ")}, which a plugin would replace: rename the plugin's file, or leave the plugin out of this cell`);
|
|
1352
|
+
}
|
|
1353
|
+
for (const f of files) {
|
|
1354
|
+
await box.writeFile(f.path, f.content);
|
|
1355
|
+
debugLog("env", `plugin part → ${f.path} (unpacked for the agent)`);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
/**
|
|
1359
|
+
* For an agent that starts a plugin's MCP servers from the plugin itself: rewrites each placed
|
|
1360
|
+
* plugin's `.mcp.json` to the servers as reeval resolved them, through the MCP timer, so they're
|
|
1361
|
+
* timed like the environment's. A manifest that declares `mcpServers` is pointed at that file.
|
|
1362
|
+
* Only namespaced layouts, which refuse a plugin server reading a host secret, so no secret value
|
|
1363
|
+
* lands in the file.
|
|
1364
|
+
*/
|
|
1365
|
+
async function writePluginMcp(box, env, plugins, servers) {
|
|
1366
|
+
for (const placed of plugins) {
|
|
1367
|
+
if (!placed.servers.length) continue;
|
|
1368
|
+
const own = Object.fromEntries(placed.servers.flatMap((key) => {
|
|
1369
|
+
const s = servers[key];
|
|
1370
|
+
return s ? [[key.slice(placed.name.length + 1), s]] : [];
|
|
1371
|
+
}));
|
|
1372
|
+
const at = `${PLUGIN_DIR}/${placed.name}`;
|
|
1373
|
+
const json = (v) => `${JSON.stringify(v, null, 2)}\n`;
|
|
1374
|
+
await box.writeFile(`${at}/.mcp.json`, json({ mcpServers: claudeMcpServers(own) }));
|
|
1375
|
+
const manifest = env.plugins.find((p) => p.name === placed.name)?.files.find((f) => f.path === PLUGIN_MANIFEST);
|
|
1376
|
+
const parsed = manifest ? JSON.parse(Buffer.from(manifest.content).toString("utf8")) : {};
|
|
1377
|
+
if (parsed.mcpServers !== void 0) await box.writeFile(`${at}/${PLUGIN_MANIFEST}`, json({
|
|
1378
|
+
...parsed,
|
|
1379
|
+
mcpServers: "./.mcp.json"
|
|
1380
|
+
}));
|
|
1381
|
+
debugLog("env", `plugin ${placed.name}'s MCP servers → ${at}/.mcp.json, through the timer`);
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
/**
|
|
1566
1385
|
* Starts the harness session within its `startTimeout`. A start that doesn't become ready in time
|
|
1567
1386
|
* is aborted (and closed, should it finish later) and started once more: a stuck bootstrap is
|
|
1568
1387
|
* usually transient, and nothing has run yet, so the retry can't change what the attempt
|
|
@@ -1594,7 +1413,7 @@ async function startHarness(a, harness, box, options) {
|
|
|
1594
1413
|
}
|
|
1595
1414
|
/**
|
|
1596
1415
|
* Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
|
|
1597
|
-
* environment's files
|
|
1416
|
+
* environment's files, workspace skills and plugins, runs its setup steps, snapshots the workspace for
|
|
1598
1417
|
* the diff, and starts the harness session with the harness-placed skills and MCP servers.
|
|
1599
1418
|
*/
|
|
1600
1419
|
async function startSession(a, runId) {
|
|
@@ -1608,7 +1427,7 @@ async function startSession(a, runId) {
|
|
|
1608
1427
|
env: env.vars,
|
|
1609
1428
|
image: a.image,
|
|
1610
1429
|
brokering: brokeringFor(spec.harness),
|
|
1611
|
-
expectedBlocked: expectedBlockedFor(spec.harness)?.hosts,
|
|
1430
|
+
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0)?.hosts,
|
|
1612
1431
|
substitutions: env.substitutions,
|
|
1613
1432
|
egress: spec.egress
|
|
1614
1433
|
});
|
|
@@ -1640,19 +1459,28 @@ async function startSession(a, runId) {
|
|
|
1640
1459
|
for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
|
|
1641
1460
|
debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
|
|
1642
1461
|
}
|
|
1462
|
+
const plugins = await copyPlugins(a, box, env);
|
|
1643
1463
|
const layout = layoutOf(spec.harness);
|
|
1464
|
+
if (layout.pluginUnpack && plugins.length) await writeUnpackedPlugins(a, box, env, plugins, layout.pluginUnpack);
|
|
1644
1465
|
for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
|
|
1645
1466
|
for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
|
|
1646
1467
|
await runSetup(a, box, env);
|
|
1647
1468
|
await snapshot(box, a.signal);
|
|
1648
1469
|
debugLog("env", "workspace snapshot taken: the diff starts here");
|
|
1470
|
+
const timed = await timeMcpServers(box, env.mcp, a.signal);
|
|
1471
|
+
if (timed.log) {
|
|
1472
|
+
a.mcpStartup = timed.log;
|
|
1473
|
+
debugLog("env", "stdio MCP servers start through reeval's MCP timer");
|
|
1474
|
+
} else if (timed.untimed) debugLog("env", `stdio MCP servers start untimed: ${timed.untimed}`);
|
|
1475
|
+
if (layout.pluginPlacement === "namespaced") await writePluginMcp(box, env, plugins, timed.servers);
|
|
1649
1476
|
const harness = await createHarness(spec.harness);
|
|
1650
1477
|
const harnessStart = performance.now();
|
|
1651
1478
|
debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
|
|
1652
1479
|
const session = await startHarness(a, harness, box, {
|
|
1653
1480
|
skills: env.skills,
|
|
1654
1481
|
workspaceSkills: env.workspaceSkills.map((s) => s.name),
|
|
1655
|
-
mcp:
|
|
1482
|
+
mcp: timed.servers,
|
|
1483
|
+
plugins
|
|
1656
1484
|
});
|
|
1657
1485
|
debugLog("harness", `${harnessId(spec.harness)} session ready in ${formatDuration(performance.now() - harnessStart)}`);
|
|
1658
1486
|
a.session = session;
|
|
@@ -1709,6 +1537,46 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1709
1537
|
cancel
|
|
1710
1538
|
};
|
|
1711
1539
|
}
|
|
1540
|
+
/** How often a turn reads the MCP timer's log for a server that never answered `initialize`. */
|
|
1541
|
+
const MCP_POLL_MS = 2e3;
|
|
1542
|
+
/**
|
|
1543
|
+
* Reads the MCP timer's log through a turn. `signal` aborts as soon as a server has timed out, so
|
|
1544
|
+
* an agent stuck waiting for it stops; `end` takes the turn's last entries.
|
|
1545
|
+
*/
|
|
1546
|
+
function watchMcpStartup(log) {
|
|
1547
|
+
const stop = new AbortController();
|
|
1548
|
+
const entries = [];
|
|
1549
|
+
let timedOut;
|
|
1550
|
+
let at = 0;
|
|
1551
|
+
let reading;
|
|
1552
|
+
const poll = async () => {
|
|
1553
|
+
entries.push(...await log.read());
|
|
1554
|
+
timedOut ??= timedOutServer(entries);
|
|
1555
|
+
if (timedOut && !stop.signal.aborted) {
|
|
1556
|
+
at = Date.now();
|
|
1557
|
+
stop.abort(new InfraError(mcpTimeoutMessage(timedOut)));
|
|
1558
|
+
}
|
|
1559
|
+
};
|
|
1560
|
+
const timer = setInterval(() => {
|
|
1561
|
+
reading ??= poll().finally(() => {
|
|
1562
|
+
reading = void 0;
|
|
1563
|
+
});
|
|
1564
|
+
}, MCP_POLL_MS);
|
|
1565
|
+
return {
|
|
1566
|
+
signal: stop.signal,
|
|
1567
|
+
close: () => clearInterval(timer),
|
|
1568
|
+
async end() {
|
|
1569
|
+
clearInterval(timer);
|
|
1570
|
+
await reading;
|
|
1571
|
+
await poll();
|
|
1572
|
+
return {
|
|
1573
|
+
entries,
|
|
1574
|
+
timedOut,
|
|
1575
|
+
at: at || Date.now()
|
|
1576
|
+
};
|
|
1577
|
+
}
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1712
1580
|
/**
|
|
1713
1581
|
* One turn. Sends the prompt to the started session, records the turn as it goes (an
|
|
1714
1582
|
* errored entry first, so a turn that throws is still in the record), takes the agent's context
|
|
@@ -1739,11 +1607,48 @@ function turnSender(a, takeContext, checks) {
|
|
|
1739
1607
|
turn: turnNumber,
|
|
1740
1608
|
prompt
|
|
1741
1609
|
}, `turn ${turnNumber} prompt`);
|
|
1742
|
-
const
|
|
1610
|
+
const watch = a.mcpStartup && watchMcpStartup(a.mcpStartup);
|
|
1611
|
+
const signal = watch ? AbortSignal.any([a.signal, watch.signal]) : a.signal;
|
|
1612
|
+
const { cutoff, cancel } = cutOff(signal, entry.startedAt);
|
|
1743
1613
|
const sendStarted = Date.now();
|
|
1744
|
-
const
|
|
1614
|
+
const sending = agent.send(prompt, signal).catch((error) => {
|
|
1615
|
+
if (!watch?.signal.aborted || a.signal.aborted) throw error;
|
|
1616
|
+
return {
|
|
1617
|
+
text: "",
|
|
1618
|
+
status: "aborted",
|
|
1619
|
+
transcript: [],
|
|
1620
|
+
usage: null,
|
|
1621
|
+
model: null,
|
|
1622
|
+
unreported: [],
|
|
1623
|
+
startedAt: entry.startedAt,
|
|
1624
|
+
durationMs: Date.now() - sendStarted
|
|
1625
|
+
};
|
|
1626
|
+
});
|
|
1627
|
+
const { raw, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
1628
|
+
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
1629
|
+
throw error;
|
|
1630
|
+
}).finally(() => {
|
|
1631
|
+
cancel();
|
|
1632
|
+
watch?.close();
|
|
1633
|
+
});
|
|
1745
1634
|
const sendWallMs = Date.now() - sendStarted;
|
|
1746
|
-
|
|
1635
|
+
const startup = await watch?.end();
|
|
1636
|
+
const timedOut = startup?.timedOut && mcpTimeoutMessage(startup.timedOut);
|
|
1637
|
+
const seenAt = startup?.at ?? Date.now();
|
|
1638
|
+
const result = timedOut ? {
|
|
1639
|
+
...sent,
|
|
1640
|
+
status: "error",
|
|
1641
|
+
transcript: [...sent.transcript, {
|
|
1642
|
+
type: "error",
|
|
1643
|
+
message: timedOut,
|
|
1644
|
+
ts: new Date(seenAt).toISOString(),
|
|
1645
|
+
atMs: Math.max(0, seenAt - Date.parse(sent.startedAt))
|
|
1646
|
+
}].sort((x, y) => x.atMs - y.atMs)
|
|
1647
|
+
} : sent;
|
|
1648
|
+
Object.assign(entry, result, raw ? { raw } : {}, {
|
|
1649
|
+
durationMs: Math.max(result.durationMs, sendWallMs),
|
|
1650
|
+
...startup ? { mcpStartup: mcpStartupOf(startup.entries) } : {}
|
|
1651
|
+
});
|
|
1747
1652
|
for (const event of result.transcript) debugTrace("harness", "transcript", {
|
|
1748
1653
|
turn: turnNumber,
|
|
1749
1654
|
...event
|
|
@@ -1762,6 +1667,12 @@ function turnSender(a, takeContext, checks) {
|
|
|
1762
1667
|
});
|
|
1763
1668
|
const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
|
|
1764
1669
|
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
1670
|
+
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
1671
|
+
if (timedOut) {
|
|
1672
|
+
const error = new InfraError(timedOut);
|
|
1673
|
+
a.startFailure ??= { error };
|
|
1674
|
+
throw error;
|
|
1675
|
+
}
|
|
1765
1676
|
a.signal.throwIfAborted();
|
|
1766
1677
|
const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
|
|
1767
1678
|
a.record.diff = diff;
|
|
@@ -1993,11 +1904,18 @@ function planEval(declaration, ctx) {
|
|
|
1993
1904
|
const [first] = unbrokered(resolved.env, plannedAttempts);
|
|
1994
1905
|
if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
|
|
1995
1906
|
}
|
|
1907
|
+
const pluginNotes = [];
|
|
1908
|
+
const pluginLabels = /* @__PURE__ */ new Map();
|
|
1996
1909
|
for (const planned of plannedAttempts) {
|
|
1997
1910
|
const why = unenforceable(planned.egress, planned.sandbox, where);
|
|
1998
1911
|
if (why) throw new UsageError(why);
|
|
1912
|
+
const on = `${where} on ${harnessId(planned.harness)}`;
|
|
1999
1913
|
const mcp = refusedMcp(planned.harness, planned.environment.layers);
|
|
2000
|
-
if (mcp) throw new UsageError(`${
|
|
1914
|
+
if (mcp) throw new UsageError(`${on}: ${mcp}`);
|
|
1915
|
+
const plugins = planPlugins(planned, ctx.config.root);
|
|
1916
|
+
if (plugins.refusal) throw new UsageError(`${on}: ${plugins.refusal}`);
|
|
1917
|
+
for (const note of plugins.notes) if (!pluginNotes.includes(note)) pluginNotes.push(note);
|
|
1918
|
+
if (plugins.labels.length) pluginLabels.set(planned, plugins.labels);
|
|
2001
1919
|
}
|
|
2002
1920
|
const path = ctx.file ? relativeFile(ctx.config.root, ctx.file) : null;
|
|
2003
1921
|
const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
|
|
@@ -2010,6 +1928,7 @@ function planEval(declaration, ctx) {
|
|
|
2010
1928
|
configFile: ctx.config.file,
|
|
2011
1929
|
file: path
|
|
2012
1930
|
});
|
|
1931
|
+
plan.notes.push(...pluginNotes);
|
|
2013
1932
|
const setByOf = (planned) => ({
|
|
2014
1933
|
sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
|
|
2015
1934
|
harness: harnessBy
|
|
@@ -2045,7 +1964,11 @@ function planEval(declaration, ctx) {
|
|
|
2045
1964
|
if (!cells.includes(planned.cell.key)) cells.push(planned.cell.key);
|
|
2046
1965
|
const workspace = workspaceOf(planned.environment.layers);
|
|
2047
1966
|
const files = filesOf(planned.environment.layers);
|
|
2048
|
-
const
|
|
1967
|
+
const builds = builtPlugins(planned.environment.layers);
|
|
1968
|
+
const setBy = builds.length ? {
|
|
1969
|
+
...setByOf(planned),
|
|
1970
|
+
pluginBuilds: `environment ${planned.environment.name}${path ? ` in ${path}` : ""}`
|
|
1971
|
+
} : setByOf(planned);
|
|
2049
1972
|
const id = evalId(rankFile, [...groups, name]);
|
|
2050
1973
|
const key = attemptKey(id, planned.cell.key, planned.n);
|
|
2051
1974
|
const skip = ctx.completed?.includes(key) ? "already completed in this run" : void 0;
|
|
@@ -2066,6 +1989,8 @@ function planEval(declaration, ctx) {
|
|
|
2066
1989
|
egress: planned.egress,
|
|
2067
1990
|
...workspace ? { workspace } : {},
|
|
2068
1991
|
...files.length ? { files } : {},
|
|
1992
|
+
...pluginLabels.has(planned) ? { plugins: pluginLabels.get(planned) } : {},
|
|
1993
|
+
...builds.length ? { pluginBuilds: builds } : {},
|
|
2069
1994
|
plan: skip ? {
|
|
2070
1995
|
...plan,
|
|
2071
1996
|
skip
|
|
@@ -2105,6 +2030,61 @@ function planEval(declaration, ctx) {
|
|
|
2105
2030
|
};
|
|
2106
2031
|
});
|
|
2107
2032
|
}
|
|
2033
|
+
/**
|
|
2034
|
+
* What the plan can tell of an attempt's plugins before any is fetched: a build outside Docker, a
|
|
2035
|
+
* local plugin's content, and the harness's refusal, as a refusal; the commands that fetch a
|
|
2036
|
+
* package at start, as notes under `allow-all`; and a label for each plugin. Git and npm plugins
|
|
2037
|
+
* are read when the attempt resolves its environment, which refuses them the same way.
|
|
2038
|
+
*/
|
|
2039
|
+
function planPlugins(planned, root) {
|
|
2040
|
+
const { layers } = planned.environment;
|
|
2041
|
+
const sources = layeredPlugins(layers);
|
|
2042
|
+
if (sources.length === 0) return {
|
|
2043
|
+
notes: [],
|
|
2044
|
+
labels: []
|
|
2045
|
+
};
|
|
2046
|
+
const read = [];
|
|
2047
|
+
const unread = [];
|
|
2048
|
+
const labels = [];
|
|
2049
|
+
for (const source of sources) {
|
|
2050
|
+
const call = pluginCall(source, root);
|
|
2051
|
+
const p = localPlugin(source, root);
|
|
2052
|
+
if (p instanceof PluginProblem) return {
|
|
2053
|
+
refusal: p.message,
|
|
2054
|
+
notes: [],
|
|
2055
|
+
labels
|
|
2056
|
+
};
|
|
2057
|
+
const build = buildRefusal(source, planned.sandbox, p?.name, root);
|
|
2058
|
+
if (build) return {
|
|
2059
|
+
refusal: build,
|
|
2060
|
+
notes: [],
|
|
2061
|
+
labels
|
|
2062
|
+
};
|
|
2063
|
+
if (p) read.push(p);
|
|
2064
|
+
else unread.push(call);
|
|
2065
|
+
labels.push(p ? `${p.name} (${p.kept.join(", ")})` : pluginSourceLabel(source, root));
|
|
2066
|
+
}
|
|
2067
|
+
const twice = duplicateLocalPlugin(layers, root);
|
|
2068
|
+
if (twice) return {
|
|
2069
|
+
refusal: twice,
|
|
2070
|
+
notes: [],
|
|
2071
|
+
labels
|
|
2072
|
+
};
|
|
2073
|
+
const own = {
|
|
2074
|
+
skills: localSkills(layers, root),
|
|
2075
|
+
mcp: [...new Set(layers.flatMap((l) => Object.keys(l.spec.mcp ?? {})))]
|
|
2076
|
+
};
|
|
2077
|
+
const refusal = refusedPlugins(planned.harness, read, own, unread);
|
|
2078
|
+
if (refusal) return {
|
|
2079
|
+
refusal,
|
|
2080
|
+
notes: [],
|
|
2081
|
+
labels
|
|
2082
|
+
};
|
|
2083
|
+
return {
|
|
2084
|
+
...fetchesAtStart(read, allowedHosts(planned.egress)),
|
|
2085
|
+
labels
|
|
2086
|
+
};
|
|
2087
|
+
}
|
|
2108
2088
|
/** The run-wide cap, plus the sandbox provider's and harness's own caps when they set one. */
|
|
2109
2089
|
function attemptLimits(planned, concurrency) {
|
|
2110
2090
|
const limits = { attempts: concurrency };
|
|
@@ -2345,4 +2325,4 @@ function optionsFrom(written, where, base) {
|
|
|
2345
2325
|
//#endregion
|
|
2346
2326
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2347
2327
|
|
|
2348
|
-
//# sourceMappingURL=evaluate-
|
|
2328
|
+
//# sourceMappingURL=evaluate-CXQ1o-pd.js.map
|