reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -1,46 +1,51 @@
|
|
|
1
1
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-
|
|
3
|
-
import {
|
|
4
|
-
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
5
|
-
import { a as TURN_EDITED, f as limitsLabel, r as withLease } from "./client-KM4C5r5_.js";
|
|
6
|
-
import { d as sandboxKind, n as capabilitiesOf, s as providerLimit } from "./kinds-Bh2I35id.js";
|
|
7
|
-
import { O as sandboxPaths, P as workspaceOf, f as filesOf, i as callerDir, j as traitsOf, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-cPMCOk-h.js";
|
|
2
|
+
import { g as parseDuration, m as MAX_TIMER_MS, n as dataProblem, t as closest } from "./shared-DtZwIAfy.js";
|
|
3
|
+
import { Gt as OPAQUE_OUTPUT, Yt as shellQuote$1, u as MCP_TIMER } from "./src-CWfy8ct8.js";
|
|
8
4
|
import { t as evalSlug } from "./slug-IHh3f770.js";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import { l as
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
5
|
+
import { c as relativeFile, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-B-JKTCP8.js";
|
|
6
|
+
import { _ as redactSplitSecrets, d as maskedVariables, f as redactFields, g as redactSecrets, o as findSecrets, p as redactJson, y as redactValue } from "./redact-t-L3VGy8.js";
|
|
7
|
+
import { B as providerLimit, I as capabilitiesOf, S as pluginBuildId, W as sandboxKind, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, k as pluginSourceLabel, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-cC0sqoDA.js";
|
|
8
|
+
import { a as plural, i as formatDuration, n as clip, o as withoutBom, t as capMiddle } from "./text-B8-W3Cvn.js";
|
|
9
|
+
import { h as vercelCredentialProblem, m as vercelBaseUrlProblem, p as vercelBaseUrl } from "./routes-Cp4gAjBZ.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-BEWxHoxH.js";
|
|
11
|
+
import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
|
|
12
|
+
import { C as writePluginFiles, S as refusedSharedMachine, b as refusedMcp, c as agentMcp, d as brokeringFor, f as claudeMcpServers, h as fakeBrokering, l as agentOf, m as expectedBlockedFor, t as AGENTS, v as keyRejected, x as refusedPlugins, y as layoutOf } from "./agents-KBTYhSSh.js";
|
|
13
|
+
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
14
|
+
import { d as limitsLabel, f as runMatchLabel, r as withLease, s as judgeFailedMessage, u as TURN_EDITED } from "./client-DMEYR3MA.js";
|
|
15
|
+
import { A as sandboxPaths, I as workspaceOf, N as traitsOf, f as filesOf, i as callerDir, l as environmentFrom, p as globFilter, u as environmentPaths } from "./environment-C0Gg8kyN.js";
|
|
16
|
+
import { $ as plainHeaders, $t as decideState, E as resolveGateway, Ft as mergeJudge, G as majorityOf, H as expand, It as resolveStateOptions, J as plainEnv, Jt as NO_JUDGE_INPUT, K as matchesTags, Lt as observedHere, M as brokeredIn, N as cascade, Qt as badJudgeArtifacts, Rt as observedName, S as resolveModelGateway, Tt as phaseLabel, U as harnessId, V as evalPlan, Xt as judgeProblem, Y as plainEnvError, Yt as inputsOn, Z as plainHeaderError, Zt as artifactRefsOf, at as sandboxId, b as gatewayScope, bt as gatewayPolicy, dt as unbrokeredError, en as filePartsOf, ht as allowedHosts, it as runsOf, kt as unenforceable, lt as secretVarsIn, ot as sandboxProvider, qt as INPUT_KEYS, rt as refuseIllegalCompare, st as sandboxSetBy, tn as resolveJudgeArtifacts, ut as unbrokered, xt as isDirect, y as fellBackToDirect, yt as egressLabel, z as checkOptions, zt as observer } from "./config-C0RKt00M.js";
|
|
17
|
+
import { f as runJudge, g as isTransient, h as isApiError, l as judgeModelName } from "./judge-DwzqucLS.js";
|
|
15
18
|
import { t as inCI } from "./env-8x1O8I1M.js";
|
|
16
|
-
import {
|
|
17
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BfnhazOv.js";
|
|
18
|
-
import { _ as unverifiedHarness, c as brokeringFor, d as expectedBlockedFor, g as refusedPlugins, h as refusedMcp, l as claudeMcpServers, m as layoutOf, o as agentMcp, p as keyRejected, t as AGENTS, v as writePluginFiles } from "./agents-DXuFxtGC.js";
|
|
19
|
-
import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
|
|
20
|
-
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-Xm9oeC7o.js";
|
|
19
|
+
import { a as registerTest, i as registerGroup, n as declarePlanned, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CPUH0Vk5.js";
|
|
21
20
|
import { c as formatBytes } from "./debug-CIXzB4wK.js";
|
|
22
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
23
|
-
import { t as attemptDebugDir } from "./files-
|
|
24
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
25
|
-
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-
|
|
26
|
-
import { t as contextTaker } from "./context-
|
|
27
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
21
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-_dUsblVx.js";
|
|
22
|
+
import { t as attemptDebugDir } from "./files-g1watgRC.js";
|
|
23
|
+
import { r as openRouterFallbacks } from "./openrouter-models-B1sIOTD2.js";
|
|
24
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as snapshot, f as workspaceBlobs, l as changedBetween, o as asCheck, p as workspaceFiles, r as serveCommand, s as createChecks, t as capturePage, u as diffSinceSnapshot } from "./capture-BkbPsfej.js";
|
|
25
|
+
import { t as contextTaker } from "./context-DrEAtUyg.js";
|
|
26
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-DorSHDBs.js";
|
|
27
|
+
import { n as fakeBootstrapScript } from "./fake-bootstrap-VpisF1WE.js";
|
|
28
|
+
import { n as providerStop, t as ProviderStopError } from "./provider-stop-CxzWW7Ft.js";
|
|
29
|
+
import { t as PrepareError } from "./provider-ieWOn4YS.js";
|
|
30
|
+
import { i as preparedWorkspace } from "./workspace-DwTe1zo6.js";
|
|
30
31
|
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
31
|
-
import { n as judgeDiff } from "./patch-
|
|
32
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import { a as
|
|
32
|
+
import { n as judgeDiff } from "./patch-BR8rdl7r.js";
|
|
33
|
+
import { t as transcriptForJudge } from "./transcript-BXWZiMuC.js";
|
|
34
|
+
import { n as modelCallEvents } from "./model-calls-B0BoQfJ4.js";
|
|
35
|
+
import { t as resourcesLine } from "./resources-DqdwihhT.js";
|
|
36
|
+
import { a as writeTar, n as readTar, t as extractTar } from "./tarball-C8BSTAMk.js";
|
|
37
|
+
import { a as optionsHash, n as caseHash, r as caseJson, t as bodyHash } from "./eval-source-CsW5VIXS.js";
|
|
38
|
+
import { t as environmentInputs } from "./inputs-5UIDpjz-.js";
|
|
39
|
+
import { i as withDefaultEffort } from "./default-effort-D70cTG00.js";
|
|
40
|
+
import { a as traitFilters, i as pickTraits, n as controlTraitKeys, r as filtersLabel } from "./traits-CCD8XsmI.js";
|
|
37
41
|
import { z } from "zod";
|
|
38
42
|
import { dirname, join } from "node:path";
|
|
39
|
-
import {
|
|
40
|
-
import { randomBytes } from "node:crypto";
|
|
41
|
-
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
43
|
+
import { randomBytes, randomUUID } from "node:crypto";
|
|
42
44
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
45
|
+
import { parse, stringify } from "yaml";
|
|
43
46
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
47
|
+
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
48
|
+
import { resolveBridgePort } from "@ai-sdk/harness/utils";
|
|
44
49
|
// src/attempt/assertions.ts
|
|
45
50
|
const MESSAGE_LIMIT = 1024;
|
|
46
51
|
const LABEL_LIMIT = 60;
|
|
@@ -70,7 +75,7 @@ function assertionRecords(raw) {
|
|
|
70
75
|
const seen = /* @__PURE__ */ new Map();
|
|
71
76
|
const used = /* @__PURE__ */ new Set();
|
|
72
77
|
return raw.map((r) => {
|
|
73
|
-
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
78
|
+
const named = r.matcher === "toStayWithin" ? limitsLabel(r.args[0]) : r.matcher === "toHaveRun" && runMatchLabel(r.args[0]) !== void 0 ? runMatchLabel(r.args[0]) : r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
|
|
74
79
|
const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
|
|
75
80
|
let n = seen.get(base) ?? 0;
|
|
76
81
|
let label;
|
|
@@ -103,12 +108,25 @@ function checkFailure(raw) {
|
|
|
103
108
|
if (!failed) return void 0;
|
|
104
109
|
const error = failed.error instanceof Error ? failed.error : new Error(text(failed.error));
|
|
105
110
|
if (failed.soft) return error;
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
}
|
|
111
|
+
const message = `${error.message} ${CAUGHT_HINT}`;
|
|
112
|
+
const stack = error.stack?.replace(error.message, () => message);
|
|
113
|
+
if (error instanceof DOMException) return Object.defineProperties(new Error(message, { cause: error }), {
|
|
114
|
+
name: own(error.name),
|
|
115
|
+
stack: own(stack)
|
|
116
|
+
});
|
|
117
|
+
return Object.setPrototypeOf(Object.defineProperties(/* @__PURE__ */ new Error(), {
|
|
118
|
+
...Object.getOwnPropertyDescriptors(error),
|
|
119
|
+
name: own(error.name),
|
|
120
|
+
message: own(message),
|
|
121
|
+
stack: own(stack)
|
|
122
|
+
}), Object.getPrototypeOf(error));
|
|
123
|
+
}
|
|
124
|
+
const own = (value) => ({
|
|
125
|
+
value,
|
|
126
|
+
writable: true,
|
|
127
|
+
configurable: true,
|
|
128
|
+
enumerable: false
|
|
129
|
+
});
|
|
112
130
|
// src/attempt/debug.ts
|
|
113
131
|
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
114
132
|
function tracePlan(debug, spec, ctx) {
|
|
@@ -206,12 +224,17 @@ const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n"
|
|
|
206
224
|
*/
|
|
207
225
|
const createFakeHarness = (descriptor) => {
|
|
208
226
|
let starts = 0;
|
|
209
|
-
return { async start(sandbox,
|
|
227
|
+
return { async start(sandbox, signal, { skills, workspaceSkills, mcp: all, plugins }) {
|
|
210
228
|
if (descriptor.failWith) throw new InfraError(descriptor.failWith);
|
|
211
229
|
const hangs = descriptor.startHangs;
|
|
212
230
|
if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
|
|
231
|
+
if (descriptor.bootstrap) {
|
|
232
|
+
const r = await sandbox.exec(fakeBootstrapScript(descriptor.bootstrap), { signal });
|
|
233
|
+
if (r.exitCode !== 0) throw new InfraError(`The fake agent's bootstrap exited ${r.exitCode}: ${redactSecrets(r.stderr || r.stdout).trim().slice(-500)}`);
|
|
234
|
+
}
|
|
213
235
|
const layout = layoutOf(descriptor);
|
|
214
236
|
const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
|
|
237
|
+
if (descriptor.brokers && AGENTS[descriptor.layout ?? "cursor"].bridgePort) await reachBridge(sandbox, signal);
|
|
215
238
|
const mcp = agentMcp(layout, all, plugins);
|
|
216
239
|
for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
|
|
217
240
|
const dir = `~/${layout.harnessSkillDir}`;
|
|
@@ -231,9 +254,11 @@ const createFakeHarness = (descriptor) => {
|
|
|
231
254
|
const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
|
|
232
255
|
const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
|
|
233
256
|
const flaked = Math.random() < (descriptor.flakeRate ?? 0);
|
|
257
|
+
let running;
|
|
234
258
|
return {
|
|
235
259
|
async send(prompt, signal) {
|
|
236
260
|
const startedAt = Date.now();
|
|
261
|
+
const brokeredBefore = sandbox.brokeredStatuses?.().length ?? 0;
|
|
237
262
|
const script = descriptor.script ?? [];
|
|
238
263
|
const steps = script[Math.min(turns++, script.length - 1)] ?? [];
|
|
239
264
|
const at = () => {
|
|
@@ -261,10 +286,19 @@ const createFakeHarness = (descriptor) => {
|
|
|
261
286
|
durationMs: descriptor.turnMs ?? Date.now() - startedAt,
|
|
262
287
|
transcript,
|
|
263
288
|
usage: descriptor.usage ?? null,
|
|
289
|
+
...!descriptor.costUsd ? {} : { cost: {
|
|
290
|
+
usd: descriptor.costUsd,
|
|
291
|
+
basis: "reported",
|
|
292
|
+
source: "fake"
|
|
293
|
+
} },
|
|
264
294
|
model: descriptor.servedModel ?? null,
|
|
265
295
|
unreported: [],
|
|
266
296
|
...descriptor.turnMs === void 0 ? {} : { agentMs: descriptor.turnMs }
|
|
267
297
|
});
|
|
298
|
+
running = (note) => ({
|
|
299
|
+
...done("", "aborted"),
|
|
300
|
+
transcript: [...transcript, note]
|
|
301
|
+
});
|
|
268
302
|
const write = async (path, content) => {
|
|
269
303
|
const time = at();
|
|
270
304
|
const before = await sandbox.readFile(path);
|
|
@@ -294,8 +328,9 @@ const createFakeHarness = (descriptor) => {
|
|
|
294
328
|
tool: tool.tool,
|
|
295
329
|
input: tool.input,
|
|
296
330
|
output: tool.output,
|
|
331
|
+
outputView: "model",
|
|
297
332
|
ok: tool.ok === void 0 ? true : tool.ok,
|
|
298
|
-
durationMs: 0
|
|
333
|
+
durationMs: tool.durationMs === void 0 ? 0 : tool.durationMs
|
|
299
334
|
});
|
|
300
335
|
if (flaked) {
|
|
301
336
|
const text = "I ran out of ideas and left the workspace unchanged.";
|
|
@@ -323,6 +358,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
323
358
|
tool: "shell",
|
|
324
359
|
input: { command: step.exec },
|
|
325
360
|
output: r.stdout,
|
|
361
|
+
outputView: "model",
|
|
326
362
|
ok: r.exitCode === 0,
|
|
327
363
|
durationMs: Date.now() - startedAt - time.atMs
|
|
328
364
|
});
|
|
@@ -340,6 +376,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
340
376
|
tool: `mcp__${server}__${tool}`,
|
|
341
377
|
input: {},
|
|
342
378
|
output: r.output,
|
|
379
|
+
outputView: "model",
|
|
343
380
|
ok: r.ok,
|
|
344
381
|
durationMs: Date.now() - startedAt - time.atMs
|
|
345
382
|
});
|
|
@@ -350,7 +387,16 @@ const createFakeHarness = (descriptor) => {
|
|
|
350
387
|
type: "error",
|
|
351
388
|
message: step.error
|
|
352
389
|
});
|
|
353
|
-
|
|
390
|
+
const statuses = sandbox.brokeredStatuses?.().slice(brokeredBefore) ?? [];
|
|
391
|
+
const stopped = providerStop({
|
|
392
|
+
errors: [step.error],
|
|
393
|
+
status: "error",
|
|
394
|
+
statuses
|
|
395
|
+
});
|
|
396
|
+
return {
|
|
397
|
+
...done("", "error"),
|
|
398
|
+
...stopped && { providerStop: stopped }
|
|
399
|
+
};
|
|
354
400
|
} else if ("refuseKey" in step) {
|
|
355
401
|
const message = `${keyRejected(step.refuseKey)}. 401 Unauthorized (the fake agent's)`;
|
|
356
402
|
debugLog("harness", `turn failed: ${message}`);
|
|
@@ -379,6 +425,10 @@ const createFakeHarness = (descriptor) => {
|
|
|
379
425
|
});
|
|
380
426
|
return done(text, "completed");
|
|
381
427
|
},
|
|
428
|
+
stopped: async (note) => {
|
|
429
|
+
if (!running) throw new Error("No turn is running");
|
|
430
|
+
return running(note);
|
|
431
|
+
},
|
|
382
432
|
async close() {}
|
|
383
433
|
};
|
|
384
434
|
} };
|
|
@@ -388,7 +438,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
388
438
|
* endpoints, as a real adapter does, and returns the variable its `exec` steps get it in.
|
|
389
439
|
*/
|
|
390
440
|
async function brokerKey(descriptor, sandbox) {
|
|
391
|
-
const { credential, endpoints } =
|
|
441
|
+
const { credential, endpoints } = fakeBrokering(descriptor.layout);
|
|
392
442
|
const key = process.env[credential];
|
|
393
443
|
if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
|
|
394
444
|
const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
|
|
@@ -408,6 +458,40 @@ async function brokerKey(descriptor, sandbox) {
|
|
|
408
458
|
return { [credential]: placeholder };
|
|
409
459
|
}
|
|
410
460
|
/**
|
|
461
|
+
* Serves on the port the layout's adapter would put its bridge on, in the sandbox, and reaches it
|
|
462
|
+
* from the host through the session's endpoint, as that adapter's start does.
|
|
463
|
+
*/
|
|
464
|
+
async function reachBridge(sandbox, signal) {
|
|
465
|
+
const network = hasNetworkSession(sandbox) ? sandbox.network : void 0;
|
|
466
|
+
if (!network) throw new InfraError("The fake agent's bridge needs a sandbox session");
|
|
467
|
+
let port;
|
|
468
|
+
try {
|
|
469
|
+
port = resolveBridgePort({
|
|
470
|
+
sandboxSession: network,
|
|
471
|
+
portOverride: void 0
|
|
472
|
+
});
|
|
473
|
+
} catch (error) {
|
|
474
|
+
throw new InfraError(`Fake harness failed to start: ${error.message}`);
|
|
475
|
+
}
|
|
476
|
+
const server = await network.spawn({
|
|
477
|
+
command: `node -e ${shellQuote$1(`require('http').createServer((q, s) => s.end('bridge')).listen(${port}, '0.0.0.0', () => console.log('up'))`)}`,
|
|
478
|
+
abortSignal: signal
|
|
479
|
+
});
|
|
480
|
+
try {
|
|
481
|
+
const up = await server.stdout.getReader().read();
|
|
482
|
+
if (!new TextDecoder().decode(up.value).includes("up")) throw new InfraError(`The fake agent's bridge didn't listen on ${port}`);
|
|
483
|
+
const { url } = await network.getPortEndpoint({
|
|
484
|
+
port,
|
|
485
|
+
protocol: "http"
|
|
486
|
+
});
|
|
487
|
+
const reply = await (await fetch(url, { signal })).text();
|
|
488
|
+
if (reply !== "bridge") throw new InfraError(`${url} answered ${JSON.stringify(reply)}`);
|
|
489
|
+
debugLog("harness", `bridge port ${port} reached through ${url}`);
|
|
490
|
+
} finally {
|
|
491
|
+
await server.kill();
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
/**
|
|
411
495
|
* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
|
|
412
496
|
* arguments: initialize, then the call.
|
|
413
497
|
*/
|
|
@@ -445,8 +529,8 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
445
529
|
arguments: {}
|
|
446
530
|
}
|
|
447
531
|
}
|
|
448
|
-
].map((m) => shellQuote(JSON.stringify(m)));
|
|
449
|
-
const command = [server.command, ...server.args].map(shellQuote).join(" ");
|
|
532
|
+
].map((m) => shellQuote$1(JSON.stringify(m)));
|
|
533
|
+
const command = [server.command, ...server.args].map(shellQuote$1).join(" ");
|
|
450
534
|
const script = [
|
|
451
535
|
"d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
|
|
452
536
|
"{",
|
|
@@ -486,7 +570,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
486
570
|
/** The attempt's harness, for the sandbox it runs in; an agent the sandbox can't run is an infra error. */
|
|
487
571
|
async function createHarness(descriptor, sandbox, vertex) {
|
|
488
572
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
489
|
-
const { createAgentHarness } = await import("./agent-
|
|
573
|
+
const { createAgentHarness } = await import("./agent-B-6GFDqA.js");
|
|
490
574
|
return createAgentHarness(descriptor, sandbox, vertex);
|
|
491
575
|
}
|
|
492
576
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
@@ -501,7 +585,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
501
585
|
const { spec, record } = a;
|
|
502
586
|
const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w), {
|
|
503
587
|
harness: spec.harness,
|
|
504
|
-
allowed: allowedHosts(spec.egress)
|
|
588
|
+
allowed: allowedHosts(spec.egress),
|
|
589
|
+
...spec.egress.setup ? { setupEgress: spec.egress.setup.policy } : {}
|
|
505
590
|
});
|
|
506
591
|
a.env = env;
|
|
507
592
|
record.environment = env.record;
|
|
@@ -516,8 +601,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
516
601
|
record.image = null;
|
|
517
602
|
const kind = sandboxKind(spec.sandbox);
|
|
518
603
|
if (kind.capabilities.images) {
|
|
519
|
-
const { preparedFor } = await import("./preflight-
|
|
520
|
-
const { attemptImage, createProviders, preparedImage } = await import("./providers-
|
|
604
|
+
const { preparedFor } = await import("./preflight-zsAn3heN.js");
|
|
605
|
+
const { attemptImage, createProviders, preparedImage } = await import("./providers-BkLGzaSY.js");
|
|
521
606
|
signal.throwIfAborted();
|
|
522
607
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
523
608
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -542,14 +627,18 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
542
627
|
const { ref, id, base, install, agentCli } = record.image;
|
|
543
628
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
544
629
|
} else if (kind.capabilities.isolated) {
|
|
545
|
-
const { preparedFor } = await import("./preflight-
|
|
546
|
-
const { createProviders, vmImage } = await import("./providers-
|
|
630
|
+
const { preparedFor } = await import("./preflight-zsAn3heN.js");
|
|
631
|
+
const { bakedImage, createProviders, preparedHosts, vmImage } = await import("./providers-BkLGzaSY.js");
|
|
547
632
|
const provider = createProviders()[kind.provider(spec.sandbox)];
|
|
548
633
|
const prepared = provider && preparedFor(root, runId, provider, spec.sandbox, spec.harness);
|
|
549
634
|
const vm = prepared ? vmImage(prepared) : null;
|
|
550
635
|
if (vm) record.vmImage = vm;
|
|
636
|
+
const baked = prepared ? bakedImage(prepared) : null;
|
|
637
|
+
if (baked) a.image = baked;
|
|
638
|
+
const hosts = prepared ? preparedHosts(prepared) : null;
|
|
639
|
+
if (hosts) a.hosts = hosts;
|
|
551
640
|
const from = vm?.ref ?? ("image" in spec.sandbox ? spec.sandbox.image : void 0);
|
|
552
|
-
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from it`);
|
|
641
|
+
debugLog("image", `${from ?? "none recorded"}${vm?.digest ? ` (${vm.digest})` : ""} · ${kind.helpers[0]}() checked it before the run and starts each attempt from ${baked ? `${baked}, which has the harness baked in` : "it"}`);
|
|
553
642
|
} else debugLog("image", kind.capabilities.isolated ? `none: the ${spec.sandbox.kind} sandbox starts from its provider's base` : `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
554
643
|
}
|
|
555
644
|
// src/judge/size.ts
|
|
@@ -607,6 +696,19 @@ function stateHint(state, question) {
|
|
|
607
696
|
* no model's limit, so this follows every provider error.
|
|
608
697
|
*/
|
|
609
698
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
699
|
+
const count = (digits) => Number(digits.replaceAll(",", ""));
|
|
700
|
+
/**
|
|
701
|
+
* Whether the provider refused the call on a token limit this input alone is over, which no
|
|
702
|
+
* retry fixes: a request too large, OpenAI's `Limit 30000, Requested 45000`, or Anthropic's
|
|
703
|
+
* `of 30,000 input tokens per minute` against the input's estimated size.
|
|
704
|
+
*/
|
|
705
|
+
function overTokenLimit(message, state, question) {
|
|
706
|
+
if (/request too large/i.test(message)) return true;
|
|
707
|
+
const asked = /Limit:? ([\d,]+), Requested:? ([\d,]+)/i.exec(message);
|
|
708
|
+
if (asked) return count(asked[2]) > count(asked[1]);
|
|
709
|
+
const perMinute = /([\d,]+) (?:input )?tokens per minute/i.exec(message);
|
|
710
|
+
return !!perMinute && inputTokens(state, question) >= count(perMinute[1]);
|
|
711
|
+
}
|
|
610
712
|
// src/judge/state.ts
|
|
611
713
|
function turnState(t) {
|
|
612
714
|
return {
|
|
@@ -778,10 +880,16 @@ function judgeCaller(a, readFile) {
|
|
|
778
880
|
debugLog("judge", `asking ${name}: ${JSON.stringify(clip(redactSecrets(question), 160))}`);
|
|
779
881
|
if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
|
|
780
882
|
debugTrace("judge", "prompt", input, "judge prompt");
|
|
781
|
-
const failed = (message, retries) => {
|
|
883
|
+
const failed = (message, transient, retries) => {
|
|
782
884
|
a.judgeErrors.push({
|
|
783
885
|
call: order,
|
|
784
|
-
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}
|
|
886
|
+
message: `c.judge(${JSON.stringify(clip(redactSecrets(question), 80))}) failed: ${message}`,
|
|
887
|
+
transient,
|
|
888
|
+
matcher: judgeFailedMessage({
|
|
889
|
+
question,
|
|
890
|
+
model: name,
|
|
891
|
+
error: message
|
|
892
|
+
})
|
|
785
893
|
});
|
|
786
894
|
input.error = message;
|
|
787
895
|
if (retries?.length) input.retries = retries;
|
|
@@ -819,7 +927,7 @@ function judgeCaller(a, readFile) {
|
|
|
819
927
|
...unpinned,
|
|
820
928
|
gateway: entry || gateway
|
|
821
929
|
} : unpinned;
|
|
822
|
-
const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
930
|
+
const { verdict, retries, usage } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
|
|
823
931
|
...routed,
|
|
824
932
|
model,
|
|
825
933
|
threshold: atLeast
|
|
@@ -832,6 +940,7 @@ function judgeCaller(a, readFile) {
|
|
|
832
940
|
judgments.splice(placeFor(judgments.map((j) => j.call ?? 0), order), 0, {
|
|
833
941
|
...recorded,
|
|
834
942
|
...retries.length ? { retries } : {},
|
|
943
|
+
...usage ? { usage } : {},
|
|
835
944
|
call: order
|
|
836
945
|
});
|
|
837
946
|
const { score } = verdict;
|
|
@@ -853,7 +962,8 @@ function judgeCaller(a, readFile) {
|
|
|
853
962
|
if (a.signal.aborted) throw error;
|
|
854
963
|
const { retries } = error;
|
|
855
964
|
const message = error instanceof Error ? error.message : String(error);
|
|
856
|
-
|
|
965
|
+
const transient = isTransient(error) && !overTokenLimit(message, state, question);
|
|
966
|
+
return failed(isApiError(error) && !transient ? withInputSize(message, state, question) : message, transient, retries);
|
|
857
967
|
}
|
|
858
968
|
};
|
|
859
969
|
return async (question, given) => {
|
|
@@ -878,6 +988,191 @@ async function createSandbox(descriptor, options) {
|
|
|
878
988
|
throw new InfraError(`Failed to create ${kind.provider(descriptor)} sandbox: ${error.message}`);
|
|
879
989
|
}
|
|
880
990
|
}
|
|
991
|
+
// src/view/timeline.ts
|
|
992
|
+
const isToolLike = (e) => e.type === "file-edit" || e.type === "web-search" || e.type === "web-fetch" || e.type === "tool";
|
|
993
|
+
/**
|
|
994
|
+
* The events with each tool call's missing `durationMs` run to the next event's `atMs` (the
|
|
995
|
+
* turn's end for the last), flagged `durationInferred`. A main model call ends it too: the agent
|
|
996
|
+
* only calls the model again once the tool's result is back. A side call or a call that carries
|
|
997
|
+
* the whole turn doesn't.
|
|
998
|
+
*/
|
|
999
|
+
function inferDurations(events, turnMs) {
|
|
1000
|
+
return events.map((e, i) => {
|
|
1001
|
+
if (!isToolLike(e) || e.durationMs !== null) return e;
|
|
1002
|
+
const next = events.slice(i + 1).find((n) => n.atMs > e.atMs && (n.type !== "model-call" || !n.stream && n.role !== "side"));
|
|
1003
|
+
const end = Math.max(e.atMs, next ? next.atMs : turnMs);
|
|
1004
|
+
return {
|
|
1005
|
+
...e,
|
|
1006
|
+
durationMs: end - e.atMs,
|
|
1007
|
+
durationInferred: true
|
|
1008
|
+
};
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
|
|
1012
|
+
const OUTPUT_TOOL = /^(bash_?output|task_?output|await_?shell)$/i;
|
|
1013
|
+
const CURSOR_TERMINAL = /\.cursor\/projects\/[^"]*\/terminals\/[^"/]+\.txt/;
|
|
1014
|
+
const TERMINAL_COMMAND = /command:\s*("(?:[^"\\]|\\.)*")/;
|
|
1015
|
+
const record = (v) => v && typeof v === "object" ? v : {};
|
|
1016
|
+
/** A shell call's command, or null for another tool. */
|
|
1017
|
+
function commandOf(e) {
|
|
1018
|
+
if (e.type !== "tool" || !SHELL_TOOL.test(e.tool) || OUTPUT_TOOL.test(e.tool)) return null;
|
|
1019
|
+
const input = record(e.input);
|
|
1020
|
+
const command = input.command ?? input.cmd;
|
|
1021
|
+
return typeof command === "string" ? command.trim() : null;
|
|
1022
|
+
}
|
|
1023
|
+
const strings = (v) => typeof v === "string" ? [v] : v && typeof v === "object" ? Object.values(v).flatMap(strings) : [];
|
|
1024
|
+
const textOf = (v) => typeof v === "string" ? v : JSON.stringify(v) ?? "";
|
|
1025
|
+
/** The command a Cursor terminal file's header names, from a read of it. */
|
|
1026
|
+
function terminalCommand(e) {
|
|
1027
|
+
if (e.type !== "tool") return null;
|
|
1028
|
+
const match = strings(e.output).map((text) => TERMINAL_COMMAND.exec(text)).find(Boolean);
|
|
1029
|
+
if (!match?.[1]) return null;
|
|
1030
|
+
try {
|
|
1031
|
+
return String(JSON.parse(match[1])).trim();
|
|
1032
|
+
} catch {
|
|
1033
|
+
return null;
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
function backgrounded(e, named) {
|
|
1037
|
+
if (e.type !== "tool") return false;
|
|
1038
|
+
const input = record(e.input);
|
|
1039
|
+
if (input.run_in_background === true || input.is_background === true) return true;
|
|
1040
|
+
const command = commandOf(e);
|
|
1041
|
+
if (command === null) return false;
|
|
1042
|
+
return /(^|[^&])&$/.test(command) || /(^|[\s;&|])(nohup|setsid)\s/.test(command) || named.has(command);
|
|
1043
|
+
}
|
|
1044
|
+
function polls(e) {
|
|
1045
|
+
if (e.type !== "tool") return false;
|
|
1046
|
+
if (OUTPUT_TOOL.test(e.tool)) return true;
|
|
1047
|
+
const command = commandOf(e);
|
|
1048
|
+
if (command !== null) return /^(sleep|wait)(\s|$)/.test(command);
|
|
1049
|
+
return CURSOR_TERMINAL.test(textOf(e.input));
|
|
1050
|
+
}
|
|
1051
|
+
const endOf = (e) => e.atMs + (e.durationMs ?? 0);
|
|
1052
|
+
/** Work that ends a wait no poll has checked yet: an edit, or another command. */
|
|
1053
|
+
const works = (e) => e.type === "file-edit" || commandOf(e) !== null;
|
|
1054
|
+
/**
|
|
1055
|
+
* The background jobs the agent waited on, a heuristic over the transcript: a span opens at a
|
|
1056
|
+
* call run in the background or a poll and takes in each poll after it. Until its first poll,
|
|
1057
|
+
* lookups (reads, searches) leave it open and an edit or another command drops it; after it,
|
|
1058
|
+
* the next call that isn't a poll closes it. A background call no poll followed isn't waited on.
|
|
1059
|
+
*/
|
|
1060
|
+
function waitSpans(events) {
|
|
1061
|
+
const named = new Set(events.map(terminalCommand).filter((c) => c !== null));
|
|
1062
|
+
const spans = [];
|
|
1063
|
+
let open = null;
|
|
1064
|
+
const close = () => {
|
|
1065
|
+
if (open?.polls.length) spans.push(open);
|
|
1066
|
+
open = null;
|
|
1067
|
+
};
|
|
1068
|
+
events.forEach((e, i) => {
|
|
1069
|
+
if (!isToolLike(e) || e.subagent) return;
|
|
1070
|
+
if (backgrounded(e, named)) {
|
|
1071
|
+
if (open?.polls.length) close();
|
|
1072
|
+
open ??= {
|
|
1073
|
+
startMs: e.atMs,
|
|
1074
|
+
endMs: endOf(e),
|
|
1075
|
+
background: i,
|
|
1076
|
+
polls: []
|
|
1077
|
+
};
|
|
1078
|
+
} else if (polls(e)) {
|
|
1079
|
+
open ??= {
|
|
1080
|
+
startMs: e.atMs,
|
|
1081
|
+
endMs: e.atMs,
|
|
1082
|
+
background: null,
|
|
1083
|
+
polls: []
|
|
1084
|
+
};
|
|
1085
|
+
open.polls.push(i);
|
|
1086
|
+
open.endMs = Math.max(open.endMs, endOf(e));
|
|
1087
|
+
} else if (open?.polls.length || works(e)) close();
|
|
1088
|
+
});
|
|
1089
|
+
close();
|
|
1090
|
+
return spans;
|
|
1091
|
+
}
|
|
1092
|
+
const PRIORITY = [
|
|
1093
|
+
"waiting",
|
|
1094
|
+
"tool",
|
|
1095
|
+
"model"
|
|
1096
|
+
];
|
|
1097
|
+
/** The turn from 0 to `turnMs` as segments, each moment the first of waiting, tool, model and other. */
|
|
1098
|
+
function segments(events, turnMs) {
|
|
1099
|
+
const marks = [];
|
|
1100
|
+
const add = (kind, start, end) => {
|
|
1101
|
+
const s = Math.max(0, Math.min(turnMs, start));
|
|
1102
|
+
const e = Math.max(0, Math.min(turnMs, end));
|
|
1103
|
+
if (e <= s) return;
|
|
1104
|
+
marks.push({
|
|
1105
|
+
at: s,
|
|
1106
|
+
kind,
|
|
1107
|
+
delta: 1
|
|
1108
|
+
}, {
|
|
1109
|
+
at: e,
|
|
1110
|
+
kind,
|
|
1111
|
+
delta: -1
|
|
1112
|
+
});
|
|
1113
|
+
};
|
|
1114
|
+
for (const span of waitSpans(events)) add(0, span.startMs, span.endMs);
|
|
1115
|
+
for (const e of events) if (isToolLike(e)) add(1, e.atMs, endOf(e));
|
|
1116
|
+
else if (e.type === "model-call" && !e.stream) add(2, e.atMs, e.atMs + e.durationMs);
|
|
1117
|
+
marks.sort((a, b) => a.at - b.at);
|
|
1118
|
+
const depth = [
|
|
1119
|
+
0,
|
|
1120
|
+
0,
|
|
1121
|
+
0
|
|
1122
|
+
];
|
|
1123
|
+
const out = [];
|
|
1124
|
+
let at = 0;
|
|
1125
|
+
const push = (kind, end) => {
|
|
1126
|
+
if (end <= at) return;
|
|
1127
|
+
const last = out.at(-1);
|
|
1128
|
+
if (last?.kind === kind) last.endMs = end;
|
|
1129
|
+
else out.push({
|
|
1130
|
+
kind,
|
|
1131
|
+
startMs: at,
|
|
1132
|
+
endMs: end
|
|
1133
|
+
});
|
|
1134
|
+
at = end;
|
|
1135
|
+
};
|
|
1136
|
+
const current = () => PRIORITY[depth.findIndex((d) => d > 0)] ?? "other";
|
|
1137
|
+
for (const m of marks) {
|
|
1138
|
+
push(current(), m.at);
|
|
1139
|
+
depth[m.kind] = (depth[m.kind] ?? 0) + m.delta;
|
|
1140
|
+
}
|
|
1141
|
+
push(current(), turnMs);
|
|
1142
|
+
return out;
|
|
1143
|
+
}
|
|
1144
|
+
/** A turn's `timeSplit`, from its transcript (durations inferred) and length. */
|
|
1145
|
+
function timeSplit(events, turnMs) {
|
|
1146
|
+
const total = {
|
|
1147
|
+
waiting: 0,
|
|
1148
|
+
tool: 0,
|
|
1149
|
+
model: 0,
|
|
1150
|
+
other: 0
|
|
1151
|
+
};
|
|
1152
|
+
for (const s of segments(events, turnMs)) total[s.kind] += s.endMs - s.startMs;
|
|
1153
|
+
const seen = events.some((e) => e.type === "model-call" && !e.stream);
|
|
1154
|
+
const waitingMs = Math.round(total.waiting);
|
|
1155
|
+
const toolMs = Math.round(total.tool);
|
|
1156
|
+
const modelMs = Math.round(total.model);
|
|
1157
|
+
return {
|
|
1158
|
+
modelMs: seen ? modelMs : null,
|
|
1159
|
+
toolMs,
|
|
1160
|
+
waitingMs,
|
|
1161
|
+
otherMs: Math.max(0, Math.round(turnMs) - waitingMs - toolMs - modelMs)
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
/** The turns' splits summed; `modelMs` null when any turn's is, undefined with none. */
|
|
1165
|
+
function sumSplits(splits) {
|
|
1166
|
+
const known = splits.filter((s) => s !== void 0);
|
|
1167
|
+
if (known.length === 0) return void 0;
|
|
1168
|
+
const sum = (f) => known.reduce((n, s) => n + f(s), 0);
|
|
1169
|
+
return {
|
|
1170
|
+
modelMs: known.every((s) => s.modelMs !== null) ? sum((s) => s.modelMs ?? 0) : null,
|
|
1171
|
+
toolMs: sum((s) => s.toolMs),
|
|
1172
|
+
waitingMs: sum((s) => s.waitingMs),
|
|
1173
|
+
otherMs: sum((s) => s.otherMs)
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
881
1176
|
// src/attempt/record.ts
|
|
882
1177
|
function classify(error, signal) {
|
|
883
1178
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -909,6 +1204,34 @@ function isJudges(a, error, raw) {
|
|
|
909
1204
|
if (a.judgeThrown.has(message)) return true;
|
|
910
1205
|
return raw.some((r) => r.status !== "pass" && judged(r) && (r.error === error || messageOf(r.error) === message));
|
|
911
1206
|
}
|
|
1207
|
+
/** The error the attempt's failed judge calls fail it with once its body is done, if any failed. */
|
|
1208
|
+
function judgeErrorsMessage(a) {
|
|
1209
|
+
const messages = a.judgeErrors.toSorted((x, y) => x.call - y.call).map((e) => e.message);
|
|
1210
|
+
const [only, ...more] = messages;
|
|
1211
|
+
if (!only) return void 0;
|
|
1212
|
+
return more.length ? `${more.length + 1} judge calls failed:\n${messages.join("\n")}` : only;
|
|
1213
|
+
}
|
|
1214
|
+
/** A matcher's message without the `label: ` its `expect` puts first. */
|
|
1215
|
+
const unlabeled = (r) => {
|
|
1216
|
+
const message = messageOf(r.error);
|
|
1217
|
+
const prefix = r.label === void 0 ? "" : `${r.label}: `;
|
|
1218
|
+
return message.startsWith(prefix) ? message.slice(prefix.length) : message;
|
|
1219
|
+
};
|
|
1220
|
+
/**
|
|
1221
|
+
* Whether judge calls the provider couldn't answer (a rate limit, a 5xx, overload, the network,
|
|
1222
|
+
* after the retries) are all that failed the attempt: every failed call was transient, and every
|
|
1223
|
+
* check that didn't hold is a `toPass()` on one of them, not polled. A verdict that answered, a
|
|
1224
|
+
* non-transient judge error (a bad key or model, an input over the provider's limit) or any other
|
|
1225
|
+
* check makes it the attempt's own failure. By message, since another language's runner hands back
|
|
1226
|
+
* its own copy of the error.
|
|
1227
|
+
*/
|
|
1228
|
+
function judgeOutage(a, error, raw) {
|
|
1229
|
+
if (!a.judgeErrors.length || !a.judgeErrors.every((e) => e.transient)) return false;
|
|
1230
|
+
const message = messageOf(error);
|
|
1231
|
+
if (a.judgeThrown.has(message) && message !== judgeErrorsMessage(a)) return false;
|
|
1232
|
+
const unanswered = new Set(a.judgeErrors.map((e) => e.matcher));
|
|
1233
|
+
return raw.every((r) => r.status === "pass" || judged(r) && !r.matcher.startsWith("poll.") && unanswered.has(unlabeled(r)));
|
|
1234
|
+
}
|
|
912
1235
|
/**
|
|
913
1236
|
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
914
1237
|
* (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
|
|
@@ -921,7 +1244,8 @@ function attemptError(a, error, raw) {
|
|
|
921
1244
|
}
|
|
922
1245
|
/**
|
|
923
1246
|
* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
|
|
924
|
-
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's.
|
|
1247
|
+
* assertion, which is the judge's when `error`, from `attemptError`, is a judge's. A judge's that
|
|
1248
|
+
* only a provider outage caused is an infra error, so `--retry-errors` and `--resume` run it again.
|
|
925
1249
|
*/
|
|
926
1250
|
function recordFailure(a, error, timedOut, raw) {
|
|
927
1251
|
const { record } = a;
|
|
@@ -941,9 +1265,17 @@ function recordFailure(a, error, timedOut, raw) {
|
|
|
941
1265
|
setBy: setBy ?? []
|
|
942
1266
|
};
|
|
943
1267
|
}
|
|
944
|
-
if (
|
|
1268
|
+
if (error instanceof ProviderStopError && !timedOut) record.error.provider = { ...error.stop };
|
|
1269
|
+
if (record.error.kind === "assertion" && isJudges(a, error, raw)) record.error.kind = judgeOutage(a, error, raw) ? "infra" : "judge";
|
|
945
1270
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
946
1271
|
}
|
|
1272
|
+
/** What the sandbox gave the attempt and what it used, into the record, before the sandbox goes. */
|
|
1273
|
+
async function recordResources(a) {
|
|
1274
|
+
const resources = await a.meter?.stop({ gone: a.signal.aborted }).catch(() => void 0);
|
|
1275
|
+
if (!resources) return;
|
|
1276
|
+
a.record.resources = resources;
|
|
1277
|
+
debugLog("teardown", `resources ${resourcesLine(resources)}`);
|
|
1278
|
+
}
|
|
947
1279
|
/**
|
|
948
1280
|
* A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
|
|
949
1281
|
* whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
|
|
@@ -965,6 +1297,8 @@ async function recordOom(a, sandbox) {
|
|
|
965
1297
|
a.record.outcome = "infra_error";
|
|
966
1298
|
return new InfraError(message);
|
|
967
1299
|
}
|
|
1300
|
+
/** The gateway tags entries with the phase it was switched to, which is always one of the attempt's. */
|
|
1301
|
+
const phased = (entry) => entry;
|
|
968
1302
|
/**
|
|
969
1303
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
970
1304
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
@@ -974,17 +1308,23 @@ function recordEgress(a, sandbox) {
|
|
|
974
1308
|
const traffic = sandbox?.traffic?.();
|
|
975
1309
|
const expected = expectedBlockedFor(a.spec.harness, (a.env?.plugins.length ?? 0) > 0, a.spec.vertex);
|
|
976
1310
|
const refused = traffic?.blocked ?? [];
|
|
977
|
-
const isExpected = (h) => !!expected?.hosts.includes(h.host);
|
|
1311
|
+
const isExpected = (h) => h.phase !== "setup" && !!expected?.hosts.includes(h.host);
|
|
978
1312
|
const expectedBlocked = refused.filter(isExpected);
|
|
1313
|
+
const { setup } = a.spec.egress;
|
|
1314
|
+
const direct = isDirect(a.spec.egress.policy) || !!setup && isDirect(setup.policy);
|
|
979
1315
|
a.record.egress = {
|
|
980
1316
|
policy: a.spec.egress.policy,
|
|
981
1317
|
allowed: allowedHosts(a.spec.egress),
|
|
1318
|
+
...setup ? { setup: {
|
|
1319
|
+
policy: setup.policy,
|
|
1320
|
+
allowed: allowedHosts(setup)
|
|
1321
|
+
} } : {},
|
|
982
1322
|
recorded: !!traffic,
|
|
983
|
-
...traffic &&
|
|
984
|
-
contacted: traffic?.contacted ?? [],
|
|
985
|
-
blocked: refused.filter((h) => !isExpected(h)),
|
|
986
|
-
expectedBlocked,
|
|
987
|
-
unresolved: traffic?.unresolved ?? []
|
|
1323
|
+
...traffic && direct ? { partial: true } : {},
|
|
1324
|
+
contacted: (traffic?.contacted ?? []).map(phased),
|
|
1325
|
+
blocked: refused.filter((h) => !isExpected(h)).map(phased),
|
|
1326
|
+
expectedBlocked: expectedBlocked.map(phased),
|
|
1327
|
+
unresolved: (traffic?.unresolved ?? []).map(phased)
|
|
988
1328
|
};
|
|
989
1329
|
if (expected && expectedBlocked.length) {
|
|
990
1330
|
const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
|
|
@@ -999,6 +1339,25 @@ function recordEgress(a, sandbox) {
|
|
|
999
1339
|
if (cost !== void 0) a.record.reportedCostUsd = cost;
|
|
1000
1340
|
}
|
|
1001
1341
|
/**
|
|
1342
|
+
* Each turn's model calls and inferred tool durations in its transcript, and where its time
|
|
1343
|
+
* went. After teardown, so a stream that outlived its turn (Cursor's) has ended.
|
|
1344
|
+
*/
|
|
1345
|
+
function recordTimeline(a, sandbox) {
|
|
1346
|
+
const calls = sandbox?.modelCalls?.() ?? [];
|
|
1347
|
+
for (const turn of a.record.turns) {
|
|
1348
|
+
const events = inferDurations([...turn.transcript, ...modelCallEvents(calls, {
|
|
1349
|
+
startedAt: Date.parse(turn.startedAt),
|
|
1350
|
+
durationMs: turn.durationMs,
|
|
1351
|
+
model: turn.model,
|
|
1352
|
+
steps: a.turnSteps.get(turn) ?? []
|
|
1353
|
+
})].sort((x, y) => x.atMs - y.atMs), turn.durationMs);
|
|
1354
|
+
turn.transcript = events;
|
|
1355
|
+
turn.timeSplit = timeSplit(events, turn.durationMs);
|
|
1356
|
+
}
|
|
1357
|
+
const split = sumSplits(a.record.turns.map((t) => t.timeSplit));
|
|
1358
|
+
if (split) a.record.timeSplit = split;
|
|
1359
|
+
}
|
|
1360
|
+
/**
|
|
1002
1361
|
* The record as its staging file holds it: redacted, with a secret split across a turn's raw
|
|
1003
1362
|
* stream parts masked first, since masking each part on its own leaves pieces of it to rebuild.
|
|
1004
1363
|
*/
|
|
@@ -1028,7 +1387,7 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
1028
1387
|
verdict: j.verdict,
|
|
1029
1388
|
score: j.score,
|
|
1030
1389
|
threshold: j.threshold,
|
|
1031
|
-
question: clip(j.question, 200)
|
|
1390
|
+
question: clip(redactSecrets(j.question), 200)
|
|
1032
1391
|
})),
|
|
1033
1392
|
turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
|
|
1034
1393
|
...failedChecks(record),
|
|
@@ -1169,18 +1528,32 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
1169
1528
|
// src/sandbox/phases.ts
|
|
1170
1529
|
const strip = ({ phase: _, ...sub }) => sub;
|
|
1171
1530
|
/**
|
|
1172
|
-
* Which brokered secrets the gateway swaps in when
|
|
1173
|
-
* at least one check or setup step is running, so a
|
|
1174
|
-
* the placeholder.
|
|
1531
|
+
* Which brokered secrets the gateway swaps in when, and which egress policy it enforces when.
|
|
1532
|
+
* `to: 'checks'` secrets are active only while at least one check or setup step is running, so a
|
|
1533
|
+
* request the agent makes between them gets the placeholder. A setup policy of its own holds from
|
|
1534
|
+
* the sandbox's start until `agent()`; every check runs under the agent's.
|
|
1175
1535
|
*/
|
|
1176
|
-
function brokerPhases(box, subs) {
|
|
1536
|
+
function brokerPhases(box, subs, egress) {
|
|
1177
1537
|
const agent = subs.filter((s) => s.phase === "agent").map(strip);
|
|
1178
1538
|
const all = subs.map(strip);
|
|
1179
1539
|
const perPhase = box.setSubstitutions && all.length > agent.length;
|
|
1180
1540
|
let running = 0;
|
|
1541
|
+
const enter = async (phase, planned) => {
|
|
1542
|
+
if (!box.setEgress) throw new InfraError(`Can't switch the sandbox's egress between the setup and agent phases: its egress gateway isn't running`);
|
|
1543
|
+
debugLog("container", `egress → ${phase}: ${phaseLabel(planned) || "proxy"}`);
|
|
1544
|
+
await box.setEgress({
|
|
1545
|
+
policy: gatewayPolicy(planned),
|
|
1546
|
+
direct: isDirect(planned.policy),
|
|
1547
|
+
phase
|
|
1548
|
+
});
|
|
1549
|
+
};
|
|
1181
1550
|
return {
|
|
1182
1551
|
async start() {
|
|
1183
1552
|
if (all.length > 0) await box.setSubstitutions?.(agent);
|
|
1553
|
+
if (egress?.setup) await enter("setup", egress.setup);
|
|
1554
|
+
},
|
|
1555
|
+
async agent() {
|
|
1556
|
+
if (egress?.setup) await enter("agent", egress);
|
|
1184
1557
|
},
|
|
1185
1558
|
async checks(fn) {
|
|
1186
1559
|
if (!perPhase) return fn();
|
|
@@ -1301,7 +1674,7 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1301
1674
|
},
|
|
1302
1675
|
report: (update) => progress({
|
|
1303
1676
|
at: Date.now(),
|
|
1304
|
-
...update
|
|
1677
|
+
...redactFields(update, ["kind"])
|
|
1305
1678
|
}),
|
|
1306
1679
|
async timed(phase, fn) {
|
|
1307
1680
|
const start = performance.now();
|
|
@@ -1321,7 +1694,8 @@ function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
|
1321
1694
|
artifactBlobs: /* @__PURE__ */ new Map(),
|
|
1322
1695
|
capturePages: /* @__PURE__ */ new Map(),
|
|
1323
1696
|
stepBlobs: /* @__PURE__ */ new Map(),
|
|
1324
|
-
snapshotDir
|
|
1697
|
+
snapshotDir,
|
|
1698
|
+
turnSteps: /* @__PURE__ */ new Map()
|
|
1325
1699
|
};
|
|
1326
1700
|
return a;
|
|
1327
1701
|
}
|
|
@@ -1456,7 +1830,7 @@ async function copyWorkspace(box, workspace, signal) {
|
|
|
1456
1830
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
|
|
1457
1831
|
if (workspace.init) {
|
|
1458
1832
|
const at = "2000-01-01T00:00:00Z";
|
|
1459
|
-
const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
|
|
1833
|
+
const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote$1(`${workspace.label} at ${workspace.commit}`)}`, {
|
|
1460
1834
|
signal,
|
|
1461
1835
|
env: {
|
|
1462
1836
|
GIT_AUTHOR_NAME: "reeval",
|
|
@@ -1471,13 +1845,87 @@ async function copyWorkspace(box, workspace, signal) {
|
|
|
1471
1845
|
}
|
|
1472
1846
|
debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
|
|
1473
1847
|
}
|
|
1848
|
+
/** Bytes per tar: each is built in memory and piped in whole. */
|
|
1849
|
+
const TAR_BATCH_BYTES = 67108864;
|
|
1850
|
+
/**
|
|
1851
|
+
* Writes `files` (workspace-relative, or `~/` for the home directory) with one tar per
|
|
1852
|
+
* destination: on a container or VM each `writeFile` is a process of its own, which a vendored
|
|
1853
|
+
* directory of thousands of files turns into minutes. A lone file is written as it is, and a
|
|
1854
|
+
* sandbox where the tar doesn't unpack gets a write per file.
|
|
1855
|
+
*/
|
|
1856
|
+
async function writeFiles(box, files, signal) {
|
|
1857
|
+
const home = files.flatMap((f) => f.path.startsWith("~/") ? [{
|
|
1858
|
+
...f,
|
|
1859
|
+
path: f.path.slice(2)
|
|
1860
|
+
}] : []);
|
|
1861
|
+
await unpackFiles(box, "", files.filter((f) => !f.path.startsWith("~/")), signal);
|
|
1862
|
+
await unpackFiles(box, "~/", home, signal);
|
|
1863
|
+
}
|
|
1864
|
+
async function unpackFiles(box, prefix, files, signal) {
|
|
1865
|
+
const one = async () => {
|
|
1866
|
+
for (const f of files) await box.writeFile(`${prefix}${f.path}`, f.content);
|
|
1867
|
+
};
|
|
1868
|
+
if (files.length < 2) return one();
|
|
1869
|
+
const entries = files.map((f) => ({
|
|
1870
|
+
type: "file",
|
|
1871
|
+
path: f.path,
|
|
1872
|
+
mode: 420,
|
|
1873
|
+
mtime: 0,
|
|
1874
|
+
data: typeof f.content === "string" ? Buffer.from(f.content, "utf8") : Buffer.from(f.content.buffer, f.content.byteOffset, f.content.byteLength)
|
|
1875
|
+
}));
|
|
1876
|
+
const batches = [[]];
|
|
1877
|
+
let size = 0;
|
|
1878
|
+
for (const e of entries) {
|
|
1879
|
+
if (size + e.data.length > TAR_BATCH_BYTES && batches[batches.length - 1]?.length) {
|
|
1880
|
+
batches.push([]);
|
|
1881
|
+
size = 0;
|
|
1882
|
+
}
|
|
1883
|
+
batches[batches.length - 1]?.push(e);
|
|
1884
|
+
size += e.data.length;
|
|
1885
|
+
}
|
|
1886
|
+
const dir = shellQuote$1(prefix ? box.home : box.workdir);
|
|
1887
|
+
const name = `.reeval-files-${randomUUID()}.tar`;
|
|
1888
|
+
const unpack = [
|
|
1889
|
+
"unset TAR_OPTIONS",
|
|
1890
|
+
`cd ${dir} || exit 1`,
|
|
1891
|
+
"d=$PWD",
|
|
1892
|
+
`t=$(mktemp -d) || { rm -f ${name}; exit 1; }`,
|
|
1893
|
+
"(",
|
|
1894
|
+
` tar -xmf "$d/${name}" -C "$t" && cd "$t" && find . -type f > "$t.all" || exit $?`,
|
|
1895
|
+
" : > \"$t.new\"",
|
|
1896
|
+
" while IFS= read -r f; do",
|
|
1897
|
+
" if [ -e \"$d/$f\" ] || [ -L \"$d/$f\" ]; then",
|
|
1898
|
+
" cat \"$f\" > \"$d/$f\" || exit 1",
|
|
1899
|
+
" continue",
|
|
1900
|
+
" fi",
|
|
1901
|
+
` p=\${f%/*}`,
|
|
1902
|
+
` while [ "$p" != . ] && [ ! -L "$d/$p" ]; do p=\${p%/*}; done`,
|
|
1903
|
+
" if [ \"$p\" = . ]; then printf '%s\\n' \"$f\" >> \"$t.new\"",
|
|
1904
|
+
` else mkdir -p "$d/\${f%/*}" && cat "$f" > "$d/$f" || exit 1`,
|
|
1905
|
+
" fi",
|
|
1906
|
+
" done < \"$t.all\"",
|
|
1907
|
+
" [ ! -s \"$t.new\" ] || { tar -cf \"$t.tar\" -T \"$t.new\" && cd \"$d\" && tar -xf \"$t.tar\"; }",
|
|
1908
|
+
")",
|
|
1909
|
+
"s=$?",
|
|
1910
|
+
`cd "$d"; rm -rf "$t" "$t.all" "$t.new" "$t.tar" ${name}`,
|
|
1911
|
+
"exit $s"
|
|
1912
|
+
].join("\n");
|
|
1913
|
+
for (const batch of batches) {
|
|
1914
|
+
await box.writeFile(`${prefix}${name}`, writeTar(batch));
|
|
1915
|
+
const r = await box.exec(unpack, { signal });
|
|
1916
|
+
if (r.exitCode !== 0) {
|
|
1917
|
+
debugLog("env", `the files' tar didn't unpack (exit ${r.exitCode}: ${tailOf(r.stderr || r.stdout)}): writing ${files.length} files one at a time`);
|
|
1918
|
+
return one();
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1474
1922
|
/**
|
|
1475
1923
|
* A built plugin's directory, copied from where its layer left it in the image, less the parts
|
|
1476
1924
|
* `only` leaves out, with the manifest as reeval read it.
|
|
1477
1925
|
*/
|
|
1478
1926
|
async function copyBuiltPlugin(a, box, plugin, dir) {
|
|
1479
|
-
const dropped = droppedPaths(plugin.parts).map((p) => shellQuote(`${dir}/${p}`));
|
|
1480
|
-
const r = await box.exec(`mkdir -p ${shellQuote(dir)} && cp -a ${shellQuote(plugin.built.dir)}/. ${shellQuote(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
|
|
1927
|
+
const dropped = droppedPaths(plugin.parts).map((p) => shellQuote$1(`${dir}/${p}`));
|
|
1928
|
+
const r = await box.exec(`mkdir -p ${shellQuote$1(dir)} && cp -a ${shellQuote$1(plugin.built.dir)}/. ${shellQuote$1(dir)}/${dropped.length ? ` && rm -rf ${dropped.join(" ")}` : ""}`, { signal: a.signal });
|
|
1481
1929
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't copy the ${plugin.name} plugin's build from ${plugin.built.tag}: ${tailOf(r.stderr || r.stdout)}`);
|
|
1482
1930
|
const manifest = plugin.files.find((f) => f.path === PLUGIN_MANIFEST);
|
|
1483
1931
|
if (manifest) await box.writeFile(`${dir}/${PLUGIN_MANIFEST}`, manifest.content);
|
|
@@ -1500,7 +1948,7 @@ async function copyPlugins(a, box, env) {
|
|
|
1500
1948
|
else for (const f of plugin.files) await box.writeFile(`${at}/${f.path}`, f.content);
|
|
1501
1949
|
const executable = built ? [] : plugin.files.flatMap((f) => f.executable ? [`${dir}/${f.path}`] : []);
|
|
1502
1950
|
if (executable.length) {
|
|
1503
|
-
const r = await box.exec(`chmod +x ${executable.map(shellQuote).join(" ")}`, { signal: a.signal });
|
|
1951
|
+
const r = await box.exec(`chmod +x ${executable.map(shellQuote$1).join(" ")}`, { signal: a.signal });
|
|
1504
1952
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't make the ${plugin.name} plugin's ${executable.length} executable files executable: ${tailOf(r.stderr || r.stdout)}`);
|
|
1505
1953
|
}
|
|
1506
1954
|
const root = (v) => v.replaceAll(PLUGIN_ROOT, dir);
|
|
@@ -1533,7 +1981,7 @@ async function copyPlugins(a, box, env) {
|
|
|
1533
1981
|
* only, so the sandbox has `find`.
|
|
1534
1982
|
*/
|
|
1535
1983
|
async function unpackedFromCopy(a, box, dir) {
|
|
1536
|
-
const r = await box.exec(`cd ${shellQuote(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
|
|
1984
|
+
const r = await box.exec(`cd ${shellQuote$1(dir)} && find rules agents commands hooks -maxdepth 1 -type f 2>/dev/null; true`, { signal: a.signal });
|
|
1537
1985
|
const out = [];
|
|
1538
1986
|
for (const path of r.stdout.split("\n").filter(Boolean)) {
|
|
1539
1987
|
const content = await box.readFile(`${dir}/${path}`);
|
|
@@ -1562,7 +2010,7 @@ async function writeUnpackedPlugins(a, box, env, plugins, unpack) {
|
|
|
1562
2010
|
const files = unpackPlugins(placed, unpack, unpack.hooks ? await box.readFile(unpack.hooks.file) : null);
|
|
1563
2011
|
const fresh = files.filter((f) => f.path !== unpack.hooks?.file);
|
|
1564
2012
|
if (fresh.length) {
|
|
1565
|
-
const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
|
|
2013
|
+
const taken = (await box.exec(`for p in ${fresh.map((f) => shellQuote$1(f.path)).join(" ")}; do { [ -e "$p" ] || [ -L "$p" ]; } && echo "$p"; done; true`, { signal: a.signal })).stdout.trim().split("\n").filter(Boolean);
|
|
1566
2014
|
if (taken.length) throw new InfraError(`The workspace already has ${taken.join(", ")}, which a plugin would replace: rename the plugin's file, or leave the plugin out of this cell`);
|
|
1567
2015
|
}
|
|
1568
2016
|
for (const f of files) {
|
|
@@ -1596,40 +2044,71 @@ async function writePluginMcp(box, env, plugins, servers) {
|
|
|
1596
2044
|
debugLog("env", `plugin ${placed.name}'s MCP servers → ${at}/.mcp.json, through the timer`);
|
|
1597
2045
|
}
|
|
1598
2046
|
}
|
|
2047
|
+
/** How long a start that failed on a transient error waits before it's tried again. */
|
|
2048
|
+
const TRANSIENT_START_DELAY_MS = 2e3;
|
|
1599
2049
|
/**
|
|
1600
|
-
* Starts
|
|
1601
|
-
* is aborted (and closed, should it finish later) and started once more
|
|
1602
|
-
*
|
|
1603
|
-
*
|
|
2050
|
+
* Starts a harness session within `timeoutMs`, at most twice. A start that doesn't become ready in
|
|
2051
|
+
* time is aborted (and closed, should it finish later) and started once more, and so is one that
|
|
2052
|
+
* fails on a transient {@link InfraError} (a 5xx or a network error): nothing has run yet, so the
|
|
2053
|
+
* retry can't change what the attempt measures. Each retry is passed to `onRetry` to record. A
|
|
2054
|
+
* second timeout or transient failure, or any other failure, is the attempt's infra error.
|
|
1604
2055
|
*/
|
|
1605
|
-
async function
|
|
1606
|
-
const {
|
|
1607
|
-
const ms = startTimeoutMs(descriptor);
|
|
1608
|
-
const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
|
|
2056
|
+
async function startWithRetry(start, { kind, timeoutMs, signal, onRetry, transientDelayMs = TRANSIENT_START_DELAY_MS }) {
|
|
2057
|
+
const late = `${kind}: session did not become ready within ${formatDuration(timeoutMs)}`;
|
|
1609
2058
|
for (let tries = 1;; tries++) {
|
|
1610
2059
|
const stop = new AbortController();
|
|
1611
2060
|
let timer;
|
|
1612
2061
|
const timedOut = new Promise((resolve) => {
|
|
1613
|
-
timer = setTimeout(() => resolve(null),
|
|
2062
|
+
timer = setTimeout(() => resolve(null), timeoutMs);
|
|
1614
2063
|
});
|
|
1615
|
-
const
|
|
1616
|
-
const
|
|
2064
|
+
const startedAt = Date.now();
|
|
2065
|
+
const starting = start(AbortSignal.any([signal, stop.signal]));
|
|
2066
|
+
let session;
|
|
2067
|
+
try {
|
|
2068
|
+
session = await Promise.race([starting, timedOut]);
|
|
2069
|
+
} catch (error) {
|
|
2070
|
+
signal.throwIfAborted();
|
|
2071
|
+
if (!(error instanceof InfraError) || !error.transient) throw error;
|
|
2072
|
+
if (tries === 2) throw new InfraError(`${error.message} (tried twice)`);
|
|
2073
|
+
onRetry({
|
|
2074
|
+
error: error.message,
|
|
2075
|
+
waitedMs: Date.now() - startedAt
|
|
2076
|
+
});
|
|
2077
|
+
debugLog("harness", `${error.message}; starting it once more`);
|
|
2078
|
+
await new Promise((resolve) => setTimeout(resolve, transientDelayMs));
|
|
2079
|
+
signal.throwIfAborted();
|
|
2080
|
+
continue;
|
|
2081
|
+
} finally {
|
|
2082
|
+
clearTimeout(timer);
|
|
2083
|
+
}
|
|
1617
2084
|
if (session) return session;
|
|
1618
2085
|
stop.abort(new InfraError(late));
|
|
1619
2086
|
starting.then((s) => s.close().catch(() => {}), () => {});
|
|
1620
|
-
|
|
2087
|
+
signal.throwIfAborted();
|
|
1621
2088
|
if (tries === 2) throw new InfraError(`${late} (tried twice)`);
|
|
1622
|
-
|
|
2089
|
+
onRetry({
|
|
1623
2090
|
error: late,
|
|
1624
|
-
waitedMs:
|
|
1625
|
-
}
|
|
2091
|
+
waitedMs: timeoutMs
|
|
2092
|
+
});
|
|
1626
2093
|
debugLog("harness", `${late}; starting it once more`);
|
|
1627
2094
|
}
|
|
1628
2095
|
}
|
|
2096
|
+
function startHarness(a, harness, box, options) {
|
|
2097
|
+
const { harness: descriptor } = a.spec;
|
|
2098
|
+
return startWithRetry((signal) => harness.start(box, signal, options), {
|
|
2099
|
+
kind: descriptor.kind,
|
|
2100
|
+
timeoutMs: startTimeoutMs(descriptor),
|
|
2101
|
+
signal: a.signal,
|
|
2102
|
+
onRetry: (retry) => {
|
|
2103
|
+
a.record.startRetries = [...a.record.startRetries ?? [], retry];
|
|
2104
|
+
}
|
|
2105
|
+
});
|
|
2106
|
+
}
|
|
1629
2107
|
/**
|
|
1630
2108
|
* Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
|
|
1631
2109
|
* environment's files, workspace skills and plugins, runs its setup steps, snapshots the workspace for
|
|
1632
|
-
* the diff, and starts the harness session with the
|
|
2110
|
+
* the diff, switches to the agent's egress, and starts the harness session with the
|
|
2111
|
+
* harness-placed skills and MCP servers.
|
|
1633
2112
|
*/
|
|
1634
2113
|
async function startSession(a, runId) {
|
|
1635
2114
|
const startedAt = performance.now();
|
|
@@ -1641,7 +2120,10 @@ async function startSession(a, runId) {
|
|
|
1641
2120
|
signal: a.signal,
|
|
1642
2121
|
env: env.vars,
|
|
1643
2122
|
image: a.image,
|
|
2123
|
+
...a.hosts && { hosts: a.hosts },
|
|
2124
|
+
harness: spec.harness,
|
|
1644
2125
|
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
2126
|
+
...agentOf(spec.harness, spec.vertex)?.keychainTrust && { keychainTrust: true },
|
|
1645
2127
|
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1646
2128
|
substitutions: env.substitutions,
|
|
1647
2129
|
egress: spec.egress,
|
|
@@ -1651,8 +2133,9 @@ async function startSession(a, runId) {
|
|
|
1651
2133
|
})
|
|
1652
2134
|
});
|
|
1653
2135
|
a.sandbox = box;
|
|
2136
|
+
a.meter = box.meter?.();
|
|
1654
2137
|
if (!capabilitiesOf(spec.sandbox).isolated) debugLog("container", `${sandboxKind(spec.sandbox).provider(spec.sandbox)} sandbox at ${box.workdir}`);
|
|
1655
|
-
a.phases = brokerPhases(box, env.substitutions);
|
|
2138
|
+
a.phases = brokerPhases(box, env.substitutions, spec.egress);
|
|
1656
2139
|
if (a.signal.aborted) box.destroy();
|
|
1657
2140
|
a.signal.throwIfAborted();
|
|
1658
2141
|
await a.phases.start();
|
|
@@ -1661,23 +2144,29 @@ async function startSession(a, runId) {
|
|
|
1661
2144
|
debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
|
|
1662
2145
|
}
|
|
1663
2146
|
if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
|
|
2147
|
+
const skillDir = workspaceSkillDir(spec.harness);
|
|
2148
|
+
const skillFileList = env.workspaceSkills.map((skill) => ({
|
|
2149
|
+
skill,
|
|
2150
|
+
files: skillFiles(skill).map((f) => ({
|
|
2151
|
+
path: `${skillDir}/${skill.name}/${f.path}`,
|
|
2152
|
+
content: f.content
|
|
2153
|
+
}))
|
|
2154
|
+
}));
|
|
2155
|
+
const toWrite = [...env.files, ...skillFileList.flatMap((s) => s.files)];
|
|
2156
|
+
const filesStart = performance.now();
|
|
2157
|
+
await writeFiles(box, toWrite, a.signal);
|
|
2158
|
+
if (toWrite.length) debugLog("env", `${toWrite.length} files written in ${formatDuration(performance.now() - filesStart)}`);
|
|
1664
2159
|
for (const f of env.files) {
|
|
1665
|
-
await box.writeFile(f.path, f.content);
|
|
1666
2160
|
const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
|
|
1667
2161
|
debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
|
|
1668
2162
|
}
|
|
1669
2163
|
const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
|
|
1670
2164
|
if (executable.length) {
|
|
1671
|
-
const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
|
|
2165
|
+
const paths = executable.map((p) => shellQuote$1(p.startsWith("-") ? `./${p}` : p));
|
|
1672
2166
|
const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
|
|
1673
2167
|
if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
|
|
1674
2168
|
}
|
|
1675
|
-
const
|
|
1676
|
-
for (const skill of env.workspaceSkills) {
|
|
1677
|
-
const files = skillFiles(skill);
|
|
1678
|
-
for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
|
|
1679
|
-
debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
|
|
1680
|
-
}
|
|
2169
|
+
for (const { skill, files } of skillFileList) debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
|
|
1681
2170
|
const plugins = await copyPlugins(a, box, env);
|
|
1682
2171
|
const layout = layoutOf(spec.harness);
|
|
1683
2172
|
if (layout.pluginUnpack && plugins.length) await writeUnpackedPlugins(a, box, env, plugins, layout.pluginUnpack);
|
|
@@ -1691,6 +2180,7 @@ async function startSession(a, runId) {
|
|
|
1691
2180
|
}
|
|
1692
2181
|
await snapshot(box, a.signal);
|
|
1693
2182
|
debugLog("env", "workspace snapshot taken: the diff starts here");
|
|
2183
|
+
await a.phases.agent();
|
|
1694
2184
|
const timed = await timeMcpServers(box, env.mcp, a.signal);
|
|
1695
2185
|
if (timed.log) {
|
|
1696
2186
|
a.mcpStartup = timed.log;
|
|
@@ -1803,8 +2293,8 @@ function cleanArchive(tar) {
|
|
|
1803
2293
|
};
|
|
1804
2294
|
}
|
|
1805
2295
|
const tarCommand = (workdir) => {
|
|
1806
|
-
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
|
|
1807
|
-
return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
2296
|
+
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote$1(d)}`).join(" ");
|
|
2297
|
+
return `cd ${shellQuote$1(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
1808
2298
|
};
|
|
1809
2299
|
const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
|
|
1810
2300
|
const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
|
|
@@ -1833,9 +2323,9 @@ async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
|
|
|
1833
2323
|
},
|
|
1834
2324
|
entries: null
|
|
1835
2325
|
});
|
|
1836
|
-
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
|
|
2326
|
+
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote$1(d)}`).join(" -o ");
|
|
1837
2327
|
const tar = tarCommand(box.workdir);
|
|
1838
|
-
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
2328
|
+
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote$1(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
1839
2329
|
const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
|
|
1840
2330
|
const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
|
|
1841
2331
|
const rawBytes = Number(raw.trim());
|
|
@@ -1950,10 +2440,10 @@ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SN
|
|
|
1950
2440
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1951
2441
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
1952
2442
|
/**
|
|
1953
|
-
* Resolves `grace` ms after `signal` aborts, with the turn as cut off
|
|
1954
|
-
* once the turn settles on its own.
|
|
2443
|
+
* Resolves `grace` ms after `signal` aborts, with the turn as cut off: what the session says it
|
|
2444
|
+
* got to, or an empty turn; `cancel` drops the timer once the turn settles on its own.
|
|
1955
2445
|
*/
|
|
1956
|
-
function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
2446
|
+
function cutOff(signal, startedAt, session, grace = TURN_STOP_GRACE_MS) {
|
|
1957
2447
|
let timer;
|
|
1958
2448
|
let onAbort = () => {};
|
|
1959
2449
|
const cutoff = new Promise((resolve) => {
|
|
@@ -1961,22 +2451,25 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1961
2451
|
timer = setTimeout(() => {
|
|
1962
2452
|
const message = `The agent didn't stop within ${grace / 1e3}s of being canceled, so reeval stopped waiting for its turn`;
|
|
1963
2453
|
const at = Date.now() - Date.parse(startedAt);
|
|
1964
|
-
|
|
2454
|
+
const note = {
|
|
2455
|
+
type: "error",
|
|
2456
|
+
message,
|
|
2457
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2458
|
+
atMs: at
|
|
2459
|
+
};
|
|
2460
|
+
const empty = {
|
|
1965
2461
|
text: "",
|
|
1966
2462
|
status: "aborted",
|
|
1967
|
-
transcript: [
|
|
1968
|
-
type: "error",
|
|
1969
|
-
message,
|
|
1970
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1971
|
-
atMs: at
|
|
1972
|
-
}],
|
|
2463
|
+
transcript: [note],
|
|
1973
2464
|
usage: null,
|
|
1974
2465
|
model: null,
|
|
1975
2466
|
unreported: [],
|
|
1976
2467
|
reasoningMs: null,
|
|
1977
2468
|
startedAt,
|
|
1978
2469
|
durationMs: at
|
|
1979
|
-
}
|
|
2470
|
+
};
|
|
2471
|
+
if (!session.stopped) return resolve(empty);
|
|
2472
|
+
session.stopped(note).then(resolve, () => resolve(empty));
|
|
1980
2473
|
}, grace);
|
|
1981
2474
|
};
|
|
1982
2475
|
if (signal.aborted) onAbort();
|
|
@@ -2064,7 +2557,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2064
2557
|
}, `turn ${turnNumber} prompt`);
|
|
2065
2558
|
const watch = a.mcpStartup && watchMcpStartup(a.mcpStartup);
|
|
2066
2559
|
const signal = watch ? AbortSignal.any([a.signal, watch.signal]) : a.signal;
|
|
2067
|
-
const { cutoff, cancel } = cutOff(signal, entry.startedAt);
|
|
2560
|
+
const { cutoff, cancel } = cutOff(signal, entry.startedAt, agent);
|
|
2068
2561
|
const sendStarted = Date.now();
|
|
2069
2562
|
const sending = agent.send(prompt, signal).catch((error) => {
|
|
2070
2563
|
if (!watch?.signal.aborted || a.signal.aborted) throw error;
|
|
@@ -2081,7 +2574,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2081
2574
|
};
|
|
2082
2575
|
});
|
|
2083
2576
|
const timedBefore = a.record.timings.agentMs;
|
|
2084
|
-
const { raw, rejectedKey, reasoningMs, agentMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2577
|
+
const { raw, rejectedKey, providerStop, reasoningMs, agentMs, steps, cost, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2085
2578
|
entry.reasoningMs = null;
|
|
2086
2579
|
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
2087
2580
|
throw error;
|
|
@@ -2089,6 +2582,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2089
2582
|
cancel();
|
|
2090
2583
|
watch?.close();
|
|
2091
2584
|
});
|
|
2585
|
+
if (steps) a.turnSteps.set(entry, steps);
|
|
2092
2586
|
const sendWallMs = agentMs ?? Date.now() - sendStarted;
|
|
2093
2587
|
if (agentMs !== void 0) a.record.timings.agentMs = timedBefore + agentMs;
|
|
2094
2588
|
const startup = await watch?.end();
|
|
@@ -2104,7 +2598,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2104
2598
|
atMs: Math.max(0, seenAt - Date.parse(sent.startedAt))
|
|
2105
2599
|
}].sort((x, y) => x.atMs - y.atMs)
|
|
2106
2600
|
} : sent;
|
|
2107
|
-
Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, {
|
|
2601
|
+
Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, cost ? { cost } : {}, {
|
|
2108
2602
|
durationMs: Math.max(result.durationMs, sendWallMs),
|
|
2109
2603
|
...startup ? { mcpStartup: mcpStartupOf(startup.entries) } : {}
|
|
2110
2604
|
});
|
|
@@ -2115,7 +2609,8 @@ function turnSender(a, takeContext, checks) {
|
|
|
2115
2609
|
await takeContext(box, a.signal);
|
|
2116
2610
|
const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
|
|
2117
2611
|
const failure = result.transcript.find((e) => e.type === "error");
|
|
2118
|
-
const
|
|
2612
|
+
const stopped = result.status === "aborted" ? void 0 : providerStop;
|
|
2613
|
+
const infra = timedOut || result.status !== "aborted" && rejectedKey || stopped?.message;
|
|
2119
2614
|
let changes;
|
|
2120
2615
|
let diffError;
|
|
2121
2616
|
if (!infra && !a.signal.aborted) try {
|
|
@@ -2138,7 +2633,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2138
2633
|
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${filesChanged === void 0 ? "" : `${plural(filesChanged, "file")} changed · `}${plural(count("file-edit"), "edit tool call")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
2139
2634
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
2140
2635
|
if (infra) {
|
|
2141
|
-
const error = new InfraError(infra);
|
|
2636
|
+
const error = !timedOut && stopped ? new ProviderStopError(stopped) : new InfraError(infra);
|
|
2142
2637
|
a.startFailure ??= { error };
|
|
2143
2638
|
throw error;
|
|
2144
2639
|
}
|
|
@@ -2190,7 +2685,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2190
2685
|
clock?.resume();
|
|
2191
2686
|
}
|
|
2192
2687
|
}, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
|
|
2193
|
-
const destroy = () => void a.sandbox?.destroy();
|
|
2688
|
+
const destroy = () => void Promise.resolve(a.session?.salvage?.()).catch(() => {}).finally(() => a.sandbox?.destroy());
|
|
2194
2689
|
testSignal.addEventListener("abort", destroy, { once: true });
|
|
2195
2690
|
const takeContext = contextTaker(a);
|
|
2196
2691
|
const { exec, readFile, writeFile } = createChecks(a);
|
|
@@ -2234,12 +2729,10 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2234
2729
|
const missed = await unawaited() ?? checkFailure(assertions()) ?? expectError();
|
|
2235
2730
|
if (a.startFailure) throw a.startFailure.error;
|
|
2236
2731
|
if (missed) throw missed;
|
|
2237
|
-
const judgeErrors = a
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
a.judgeThrown.add(message);
|
|
2242
|
-
throw new Error(message);
|
|
2732
|
+
const judgeErrors = judgeErrorsMessage(a);
|
|
2733
|
+
if (judgeErrors) {
|
|
2734
|
+
a.judgeThrown.add(judgeErrors);
|
|
2735
|
+
throw new Error(judgeErrors);
|
|
2243
2736
|
}
|
|
2244
2737
|
} catch (thrown) {
|
|
2245
2738
|
const error = attemptError(a, thrown, assertions());
|
|
@@ -2250,6 +2743,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2250
2743
|
clock?.clear();
|
|
2251
2744
|
await a.endStarts();
|
|
2252
2745
|
for (const bg of a.backgrounds) await bg.kill().catch(() => {});
|
|
2746
|
+
await recordResources(a);
|
|
2253
2747
|
const { sandbox } = a;
|
|
2254
2748
|
if (sandbox && !a.signal.aborted) await takeContext(sandbox, AbortSignal.timeout(3e4));
|
|
2255
2749
|
if (sandbox) a.report({ kind: "cleanup" });
|
|
@@ -2261,6 +2755,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2261
2755
|
oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
|
|
2262
2756
|
if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
|
|
2263
2757
|
await sandbox?.destroy();
|
|
2758
|
+
recordTimeline(a, sandbox);
|
|
2264
2759
|
if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
|
|
2265
2760
|
const { record } = a;
|
|
2266
2761
|
const checked = assertionRecords(assertions());
|
|
@@ -2303,8 +2798,11 @@ function bestOfCell(bestOf) {
|
|
|
2303
2798
|
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2304
2799
|
});
|
|
2305
2800
|
}
|
|
2306
|
-
/**
|
|
2307
|
-
|
|
2801
|
+
/**
|
|
2802
|
+
* Characters a case key can't hold: attempt ids split on `#` and eval ids on `›`, and control
|
|
2803
|
+
* characters. A `/` is allowed: a key is its own path segment wherever it's in one.
|
|
2804
|
+
*/
|
|
2805
|
+
const BAD_KEY = /[#›\u0000-\u001f\u007f]/;
|
|
2308
2806
|
/** The name a case's eval plans under: `adds a health check [nextjs]`. */
|
|
2309
2807
|
const caseEvalName = (name, key) => `${name} [${key}]`;
|
|
2310
2808
|
/** The key a case gets without `caseKey`: a string, a finite number, or an object's `id` or `name`. */
|
|
@@ -2318,7 +2816,7 @@ function defaultKey(item) {
|
|
|
2318
2816
|
}
|
|
2319
2817
|
/**
|
|
2320
2818
|
* Each case's key, in list order: `caseKey(item)` when given, else its default. Trimmed, 1–80
|
|
2321
|
-
* characters, with no `#`,
|
|
2819
|
+
* characters, with no `#`, `›` or control characters, and unique within the eval.
|
|
2322
2820
|
*/
|
|
2323
2821
|
function caseKeys(cases, caseKey, where) {
|
|
2324
2822
|
const seen = /* @__PURE__ */ new Map();
|
|
@@ -2327,7 +2825,7 @@ function caseKeys(cases, caseKey, where) {
|
|
|
2327
2825
|
if (raw === void 0) throw new UsageError(`cases[${i}] in ${where} has no key: a case is its own key when it's a string or a number, or an object's id or name. Give it an id, or pass caseKey: (item) => item.slug.`);
|
|
2328
2826
|
if (typeof raw !== "string") throw new UsageError(`caseKey in ${where} must return a string (got ${redactSecrets(JSON.stringify(raw) ?? String(raw))} for cases[${i}])`);
|
|
2329
2827
|
const key = raw.trim();
|
|
2330
|
-
if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #,
|
|
2828
|
+
if (!key || key.length > 80 || BAD_KEY.test(key)) throw new UsageError(`cases[${i}] in ${where} has the key ${redactSecrets(JSON.stringify(raw))}: a case key is 1–80 characters, with no #, › or control characters.${caseKey ? "" : " Pass caseKey to give it another."}`);
|
|
2331
2829
|
const earlier = seen.get(key);
|
|
2332
2830
|
if (earlier !== void 0) throw new UsageError(`cases[${earlier}] and cases[${i}] in ${where} have the same key "${redactSecrets(key)}". Each case needs its own key${caseKey ? "" : ": give them distinct ids or names, or pass caseKey"}.`);
|
|
2333
2831
|
seen.set(key, i);
|
|
@@ -2604,12 +3102,13 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
|
2604
3102
|
const hid = harnessId(harness);
|
|
2605
3103
|
if (gatewaySet || harness.gateway !== void 0) {
|
|
2606
3104
|
routes.set(hid, route);
|
|
2607
|
-
|
|
3105
|
+
const at = route.name === "vercel" && !vercelBaseUrlProblem() ? vercelBaseUrl() : void 0;
|
|
3106
|
+
gatewayNotes.push(`gateway: ${hid} → ${route.name}${at ? ` at ${at}` : ""}${route.name === "direct" && route.skipped.length ? " (fallback)" : ""}`);
|
|
2608
3107
|
}
|
|
2609
3108
|
if (route.name === "direct") return harness;
|
|
2610
3109
|
if (harness.kind !== "fake") try {
|
|
2611
3110
|
if (route.name === "vercel") {
|
|
2612
|
-
const problem = vercelCredentialProblem();
|
|
3111
|
+
const problem = vercelCredentialProblem() ?? vercelBaseUrlProblem();
|
|
2613
3112
|
if (problem) throw new UsageError(problem);
|
|
2614
3113
|
}
|
|
2615
3114
|
if (route.name === "vertex") {
|
|
@@ -2653,6 +3152,8 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
|
2653
3152
|
if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
|
|
2654
3153
|
const [plain] = plainHeaders(plannedAttempts);
|
|
2655
3154
|
if (plain) throw new UsageError(plainHeaderError(plain, where, ctx.config.file));
|
|
3155
|
+
const [plainVar] = plainEnv(resolved.env, plannedAttempts);
|
|
3156
|
+
if (plainVar) throw new UsageError(plainEnvError(plainVar, where, ctx.config.file));
|
|
2656
3157
|
}
|
|
2657
3158
|
const pluginNotes = [];
|
|
2658
3159
|
const pluginLabels = /* @__PURE__ */ new Map();
|
|
@@ -2660,10 +3161,11 @@ function planTests(declaration, ctx, where, resolvedEval, pick) {
|
|
|
2660
3161
|
const why = unenforceable(planned.egress, planned.sandbox, where);
|
|
2661
3162
|
if (why) throw new UsageError(why);
|
|
2662
3163
|
const on = `${where} on ${harnessId(planned.harness)}`;
|
|
2663
|
-
const unverified = unverifiedHarness(planned.harness, planned.sandbox);
|
|
2664
|
-
if (unverified) throw new UsageError(`${on}: ${unverified}`);
|
|
2665
3164
|
const mcp = refusedMcp(planned.harness, planned.environment.layers);
|
|
2666
3165
|
if (mcp) throw new UsageError(`${on}: ${mcp}`);
|
|
3166
|
+
const slots = sandboxKind(planned.sandbox).slotsPerMachine?.(planned.sandbox) ?? 1;
|
|
3167
|
+
const shared = refusedSharedMachine(planned.harness, slots);
|
|
3168
|
+
if (shared) throw new UsageError(`${on}: ${shared}`);
|
|
2667
3169
|
const plugins = planPlugins(planned, ctx.config.root);
|
|
2668
3170
|
if (plugins.refusal) throw new UsageError(`${on}: ${plugins.refusal}`);
|
|
2669
3171
|
for (const note of plugins.notes) if (!pluginNotes.includes(note)) pluginNotes.push(note);
|
|
@@ -3084,10 +3586,45 @@ function prepare(mode, given, own, fn, source, hint = "", givenFamily, where) {
|
|
|
3084
3586
|
bestOfCells.set(cell.key, of);
|
|
3085
3587
|
hooks = of(n);
|
|
3086
3588
|
}
|
|
3087
|
-
|
|
3589
|
+
const test = {
|
|
3088
3590
|
...options,
|
|
3089
3591
|
...hooks
|
|
3090
|
-
}
|
|
3592
|
+
};
|
|
3593
|
+
if (spec && getRunContext().inputs) {
|
|
3594
|
+
const { root } = getRunContext().config;
|
|
3595
|
+
const found = environmentInputs(spec.environment.layers, spec.sandbox, root);
|
|
3596
|
+
const { attempt } = options;
|
|
3597
|
+
const warnings = [.../* @__PURE__ */ new Set([...attempt.plan.warnings, ...found.warnings])];
|
|
3598
|
+
test.attempt = {
|
|
3599
|
+
...attempt,
|
|
3600
|
+
plan: {
|
|
3601
|
+
...attempt.plan,
|
|
3602
|
+
warnings
|
|
3603
|
+
},
|
|
3604
|
+
inputs: found.inputs
|
|
3605
|
+
};
|
|
3606
|
+
}
|
|
3607
|
+
const readCap = spec && sandboxKind(spec.sandbox).attemptCap?.(spec.sandbox);
|
|
3608
|
+
if (spec && test.gate && readCap) {
|
|
3609
|
+
const gate = {
|
|
3610
|
+
...test.gate,
|
|
3611
|
+
limits: { ...test.gate.limits }
|
|
3612
|
+
};
|
|
3613
|
+
const before = test.ready;
|
|
3614
|
+
test.gate = gate;
|
|
3615
|
+
test.ready = async () => {
|
|
3616
|
+
const { config, runId } = getRunContext();
|
|
3617
|
+
const cap = await readCap({
|
|
3618
|
+
harness: spec.harness,
|
|
3619
|
+
root: config.root,
|
|
3620
|
+
runId
|
|
3621
|
+
});
|
|
3622
|
+
const key = `sandbox:${sandboxProvider(spec.sandbox)}`;
|
|
3623
|
+
if (cap && gate.limits[key]) gate.limits[key] = Math.min(gate.limits[key], cap);
|
|
3624
|
+
return before ? before() : true;
|
|
3625
|
+
};
|
|
3626
|
+
}
|
|
3627
|
+
registerTest(testName, test, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
3091
3628
|
}
|
|
3092
3629
|
};
|
|
3093
3630
|
return {
|
|
@@ -3241,4 +3778,4 @@ function optionsFrom(written, where, base) {
|
|
|
3241
3778
|
}
|
|
3242
3779
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
3243
3780
|
|
|
3244
|
-
//# sourceMappingURL=evaluate-
|
|
3781
|
+
//# sourceMappingURL=evaluate-BHQyAszQ.js.map
|