reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -0,0 +1,807 @@
|
|
|
1
|
+
import { g as parseDuration, t as closest } from "./shared-DtZwIAfy.js";
|
|
2
|
+
import { g as redactSecrets } from "./redact-t-L3VGy8.js";
|
|
3
|
+
import { a as plural, i as formatDuration } from "./text-B8-W3Cvn.js";
|
|
4
|
+
import { r as isThenable, t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
5
|
+
import { a as skillsUsed, c as targetText, l as toolTarget, s as loadedSkill } from "./tool-class-krsmEo44.js";
|
|
6
|
+
import { o as totalTokens } from "./usage-CwIRPPZ9.js";
|
|
7
|
+
import { t as Scheduler } from "./core-CxY40JAs.js";
|
|
8
|
+
import picomatch from "picomatch";
|
|
9
|
+
import { parse } from "shell-quote";
|
|
10
|
+
import { isDeepStrictEqual } from "node:util";
|
|
11
|
+
// src/shell-commands.ts
|
|
12
|
+
const SHELL_TOOLS = /* @__PURE__ */ new Set(["bash", "shell"]);
|
|
13
|
+
const isShellCall = (e) => e.type === "tool" && SHELL_TOOLS.has(e.tool);
|
|
14
|
+
const commandOf = (event) => {
|
|
15
|
+
const command = isObjectLike(event.input) ? event.input.command : void 0;
|
|
16
|
+
return typeof command === "string" ? command : "";
|
|
17
|
+
};
|
|
18
|
+
/** Operators that end a segment. `(` and `)` do too, so a subshell's or `$(…)`'s commands count. */
|
|
19
|
+
const SEPARATORS = /* @__PURE__ */ new Set([
|
|
20
|
+
"&&",
|
|
21
|
+
"||",
|
|
22
|
+
";",
|
|
23
|
+
";;",
|
|
24
|
+
"|",
|
|
25
|
+
"|&",
|
|
26
|
+
"&",
|
|
27
|
+
"(",
|
|
28
|
+
")"
|
|
29
|
+
]);
|
|
30
|
+
/** A line break, as `shellText` hands it to shell-quote: `;;` ends a segment and survives in quotes. */
|
|
31
|
+
const LINE_BREAK = "\n;;";
|
|
32
|
+
const HEREDOC = /(?<!<)<<-?[ \t]*(['"]?)([A-Za-z_][\w.-]*)\1/g;
|
|
33
|
+
/** The line without its heredocs' bodies, which are text, not commands. */
|
|
34
|
+
function withoutHeredocs(line) {
|
|
35
|
+
const lines = line.split("\n");
|
|
36
|
+
const kept = [];
|
|
37
|
+
for (let i = 0; i < lines.length; i++) {
|
|
38
|
+
kept.push(lines[i]);
|
|
39
|
+
for (const [, , end] of lines[i].matchAll(HEREDOC)) do
|
|
40
|
+
i++;
|
|
41
|
+
while (i < lines.length && lines[i].replace(/^\t+/, "") !== end);
|
|
42
|
+
}
|
|
43
|
+
return kept.join("\n");
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The line as shell-quote should read it, where it reads a shell differently: a line break ends
|
|
47
|
+
* a segment, a `\` continuation joins its lines, the `2` of `2>&1` is no word, and a `#` inside
|
|
48
|
+
* a word (`http://host/#top`) starts no comment. Rewritten in quotes too; `words` undoes it there.
|
|
49
|
+
*/
|
|
50
|
+
const shellText = (line) => withoutHeredocs(line).replace(/\\\n/g, "").replace(/(^|[\s;&|()])\d+(?=[<>])/g, "$1").replace(/(?<=[^\s;&|()])#/g, "\\#").replace(/\n/g, LINE_BREAK);
|
|
51
|
+
const word = (token) => token.replaceAll(LINE_BREAK, "\n").replaceAll("\\#", "#");
|
|
52
|
+
/**
|
|
53
|
+
* The command line's words, a list per segment, as shell-quote reads them: variables stay as
|
|
54
|
+
* written, a glob is its pattern, a redirection drops its target, and a comment runs to the end
|
|
55
|
+
* of its line.
|
|
56
|
+
*/
|
|
57
|
+
function words(text) {
|
|
58
|
+
const out = [[]];
|
|
59
|
+
let redirect = false;
|
|
60
|
+
for (const token of parse(text, (name) => `$${name}`)) {
|
|
61
|
+
const current = out.at(-1);
|
|
62
|
+
if (typeof token === "string" || "op" in token && token.op === "glob") {
|
|
63
|
+
if (!redirect) current.push(word(typeof token === "string" ? token : token.pattern));
|
|
64
|
+
redirect = false;
|
|
65
|
+
} else if ("comment" in token) {
|
|
66
|
+
const next = token.comment.indexOf(LINE_BREAK);
|
|
67
|
+
if (next !== -1) out.push(...words(token.comment.slice(next + 3)));
|
|
68
|
+
break;
|
|
69
|
+
} else if (SEPARATORS.has(token.op)) out.push([]);
|
|
70
|
+
else redirect = true;
|
|
71
|
+
}
|
|
72
|
+
return out.filter((w) => w.length > 0);
|
|
73
|
+
}
|
|
74
|
+
const segments = (line) => words(shellText(line));
|
|
75
|
+
const ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/;
|
|
76
|
+
const DURATION = /^\d+(\.\d+)?[smhd]?$/;
|
|
77
|
+
const basename = (word) => word.slice(word.lastIndexOf("/") + 1);
|
|
78
|
+
/** The index after `words[i]`'s flags, skipping each value a flag in `takesValue` takes, and a `--`. */
|
|
79
|
+
function afterFlags(words, i, takesValue = /* @__PURE__ */ new Set()) {
|
|
80
|
+
while (i < words.length && words[i].startsWith("-") && words[i] !== "-") {
|
|
81
|
+
if (words[i] === "--") return i + 1;
|
|
82
|
+
i += takesValue.has(words[i]) ? 2 : 1;
|
|
83
|
+
}
|
|
84
|
+
return i;
|
|
85
|
+
}
|
|
86
|
+
const set = (...flags) => new Set(flags);
|
|
87
|
+
const RUNNER_FLAGS = set("-p", "--package", "-c", "--call", "-w", "--workspace", "--registry");
|
|
88
|
+
const NPM_FLAGS = set("-w", "--workspace", "--prefix", "-C");
|
|
89
|
+
const PNPM_FLAGS = set("-F", "--filter", "-C", "--dir", "--workspace-dir", "--package");
|
|
90
|
+
const YARN_FLAGS = set("--cwd", "-p", "--package");
|
|
91
|
+
/**
|
|
92
|
+
* Where a wrapper's command starts, or undefined when `words[i]` isn't a wrapper: the env
|
|
93
|
+
* assignments, `env`, `timeout`, `time`, `exec`, `sudo`, `nohup`, `nice`, `command` (not
|
|
94
|
+
* `command -v`), and the package runners `npx`, `bunx`, `npm exec` (`npm x`), `pnpm exec`,
|
|
95
|
+
* `pnpm dlx` and `yarn dlx`, each with its flags.
|
|
96
|
+
*/
|
|
97
|
+
function unwrap(words, i) {
|
|
98
|
+
const name = basename(words[i]);
|
|
99
|
+
const flags = (takesValue) => afterFlags(words, i + 1, takesValue);
|
|
100
|
+
const subcommand = (global, subs) => {
|
|
101
|
+
const at = afterFlags(words, i + 1, global);
|
|
102
|
+
return subs.includes(words[at]) ? afterFlags(words, at + 1, global) : void 0;
|
|
103
|
+
};
|
|
104
|
+
switch (name) {
|
|
105
|
+
case "env": return {
|
|
106
|
+
next: flags(set("-u", "--unset", "-C", "--chdir")),
|
|
107
|
+
runner: false
|
|
108
|
+
};
|
|
109
|
+
case "timeout": {
|
|
110
|
+
const at = flags(set("-s", "--signal", "-k", "--kill-after"));
|
|
111
|
+
return {
|
|
112
|
+
next: DURATION.test(words[at] ?? "") ? at + 1 : at,
|
|
113
|
+
runner: false
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
case "time": return {
|
|
117
|
+
next: flags(),
|
|
118
|
+
runner: false
|
|
119
|
+
};
|
|
120
|
+
case "exec": return {
|
|
121
|
+
next: flags(set("-a")),
|
|
122
|
+
runner: false
|
|
123
|
+
};
|
|
124
|
+
case "nohup": return {
|
|
125
|
+
next: flags(),
|
|
126
|
+
runner: false
|
|
127
|
+
};
|
|
128
|
+
case "nice": return {
|
|
129
|
+
next: flags(set("-n", "--adjustment")),
|
|
130
|
+
runner: false
|
|
131
|
+
};
|
|
132
|
+
case "command":
|
|
133
|
+
if (/^-[a-zA-Z]*[vV]/.test(words[i + 1] ?? "")) return void 0;
|
|
134
|
+
return {
|
|
135
|
+
next: flags(),
|
|
136
|
+
runner: false
|
|
137
|
+
};
|
|
138
|
+
case "sudo": return {
|
|
139
|
+
next: flags(set("-u", "--user", "-g", "--group", "-p", "--prompt", "-C", "-D", "--chdir")),
|
|
140
|
+
runner: false
|
|
141
|
+
};
|
|
142
|
+
case "npx":
|
|
143
|
+
case "bunx": return {
|
|
144
|
+
next: flags(RUNNER_FLAGS),
|
|
145
|
+
runner: true
|
|
146
|
+
};
|
|
147
|
+
case "npm": {
|
|
148
|
+
const next = subcommand(/* @__PURE__ */ new Set([...NPM_FLAGS, ...RUNNER_FLAGS]), ["exec", "x"]);
|
|
149
|
+
return next === void 0 ? void 0 : {
|
|
150
|
+
next,
|
|
151
|
+
runner: true
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
case "pnpm": {
|
|
155
|
+
const next = subcommand(PNPM_FLAGS, ["exec", "dlx"]);
|
|
156
|
+
return next === void 0 ? void 0 : {
|
|
157
|
+
next,
|
|
158
|
+
runner: true
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
case "yarn": {
|
|
162
|
+
const next = subcommand(YARN_FLAGS, ["dlx"]);
|
|
163
|
+
return next === void 0 ? void 0 : {
|
|
164
|
+
next,
|
|
165
|
+
runner: true
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/** A package runner's target without its `@version`; any other program without its directory. */
|
|
171
|
+
function programName(word, runner) {
|
|
172
|
+
if (runner && word.startsWith("@")) {
|
|
173
|
+
const at = word.indexOf("@", 1);
|
|
174
|
+
return at === -1 ? word : word.slice(0, at);
|
|
175
|
+
}
|
|
176
|
+
const base = basename(word);
|
|
177
|
+
const at = runner ? base.indexOf("@") : -1;
|
|
178
|
+
return at > 0 ? base.slice(0, at) : base;
|
|
179
|
+
}
|
|
180
|
+
const SHELLS = /* @__PURE__ */ new Set([
|
|
181
|
+
"bash",
|
|
182
|
+
"sh",
|
|
183
|
+
"zsh",
|
|
184
|
+
"dash"
|
|
185
|
+
]);
|
|
186
|
+
const COMMAND_FLAG = /^-[a-z]*c[a-z]*$/;
|
|
187
|
+
/** A shell's `-c` script, after the shell's other options (`bash -e -o pipefail -c '…'`). */
|
|
188
|
+
function scriptOf(args) {
|
|
189
|
+
let command = false;
|
|
190
|
+
let j = 0;
|
|
191
|
+
while (j < args.length && /^[-+]./.test(args[j])) {
|
|
192
|
+
if (args[j] === "--") {
|
|
193
|
+
j++;
|
|
194
|
+
break;
|
|
195
|
+
}
|
|
196
|
+
if (COMMAND_FLAG.test(args[j])) command = true;
|
|
197
|
+
j += /^[-+][oO]$/.test(args[j]) ? 2 : 1;
|
|
198
|
+
}
|
|
199
|
+
return command ? args[j] : void 0;
|
|
200
|
+
}
|
|
201
|
+
/** Reserved words before a command, which run it: `then genkit`, `! genkit`, `{ genkit; }`. */
|
|
202
|
+
const KEYWORDS = set("!", "{", "if", "then", "else", "elif", "while", "until", "do");
|
|
203
|
+
/** Segments that run nothing: a block's end, or a `for`, `case` or `select` header. */
|
|
204
|
+
const NOT_A_COMMAND = set("}", "fi", "done", "esac", "for", "case", "select", "function");
|
|
205
|
+
function parseSegment(words) {
|
|
206
|
+
let i = 0;
|
|
207
|
+
let runner = false;
|
|
208
|
+
let wrapper = -1;
|
|
209
|
+
while (i < words.length && KEYWORDS.has(words[i])) i++;
|
|
210
|
+
if (NOT_A_COMMAND.has(words[i])) return [];
|
|
211
|
+
for (;;) {
|
|
212
|
+
while (i < words.length && ASSIGNMENT.test(words[i])) i++;
|
|
213
|
+
if (i >= words.length) break;
|
|
214
|
+
const unwrapped = unwrap(words, i);
|
|
215
|
+
if (!unwrapped) break;
|
|
216
|
+
wrapper = i;
|
|
217
|
+
i = unwrapped.next;
|
|
218
|
+
runner = unwrapped.runner;
|
|
219
|
+
}
|
|
220
|
+
if (i >= words.length) {
|
|
221
|
+
if (wrapper === -1) return [];
|
|
222
|
+
return [{
|
|
223
|
+
program: basename(words[wrapper]),
|
|
224
|
+
args: words.slice(wrapper + 1)
|
|
225
|
+
}];
|
|
226
|
+
}
|
|
227
|
+
const program = programName(words[i], runner);
|
|
228
|
+
const args = words.slice(i + 1);
|
|
229
|
+
const script = !runner && SHELLS.has(program) ? scriptOf(args) : void 0;
|
|
230
|
+
if (script !== void 0) return parseCommandLine(script);
|
|
231
|
+
return [{
|
|
232
|
+
program,
|
|
233
|
+
args
|
|
234
|
+
}];
|
|
235
|
+
}
|
|
236
|
+
/** The programs a command line runs, one per segment, in order. */
|
|
237
|
+
function parseCommandLine(line) {
|
|
238
|
+
return segments(line).flatMap(parseSegment);
|
|
239
|
+
}
|
|
240
|
+
/** The programs the transcript's shell calls ran, in order. */
|
|
241
|
+
const transcriptCommands = (transcript, turn = 0) => transcript.filter(isShellCall).flatMap((e) => {
|
|
242
|
+
const line = commandOf(e);
|
|
243
|
+
return parseCommandLine(line).map((p) => ({
|
|
244
|
+
...p,
|
|
245
|
+
line,
|
|
246
|
+
ok: e.ok,
|
|
247
|
+
turn
|
|
248
|
+
}));
|
|
249
|
+
});
|
|
250
|
+
/**
|
|
251
|
+
* The programs the agent ran from its shell calls (the tools named `bash` or `shell`), in order:
|
|
252
|
+
* one per segment of each command line, after its env assignments, `env`, `timeout`, `time`,
|
|
253
|
+
* `exec`, `sudo`, `nohup`, `nice`, `command`, `bash -c` and package runners (`npx`, `bunx`, `npm exec`, `pnpm exec`,
|
|
254
|
+
* `pnpm dlx`, `yarn dlx`). Takes a turn from `c.send()`, a list of turns, or `c`.
|
|
255
|
+
*/
|
|
256
|
+
function shellCommands(subject) {
|
|
257
|
+
const given = subject;
|
|
258
|
+
const turns = Array.isArray(given) ? given : isObjectLike(given) && Array.isArray(given.turns) ? given.turns : isObjectLike(given) && Array.isArray(given.transcript) ? [given] : void 0;
|
|
259
|
+
if (!turns) throw new TypeError("shellCommands() expects a turn from c.send(), c.turns, or c");
|
|
260
|
+
return turns.flatMap((t, i) => transcriptCommands(t.transcript, i));
|
|
261
|
+
}
|
|
262
|
+
// src/assert.ts
|
|
263
|
+
/**
|
|
264
|
+
* A matcher handed a subject it can't read, such as `toHaveEdited(1)`: recorded as `error`, an
|
|
265
|
+
* answer it couldn't give, so the eval's own `TypeError`s stay errors that end the attempt.
|
|
266
|
+
*/
|
|
267
|
+
function subjectError(message) {
|
|
268
|
+
return Object.assign(new TypeError(message), { [Symbol.for("reeval.subjectError")]: true });
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* A matcher called wrong, such as `toStayWithin()` with no limit: not a check that ran, so it
|
|
272
|
+
* isn't recorded as one, and the `TypeError` ends the attempt as the eval's own would.
|
|
273
|
+
*/
|
|
274
|
+
function misuseError(message) {
|
|
275
|
+
return Object.assign(new TypeError(message), { [Symbol.for("reeval.misuse")]: true });
|
|
276
|
+
}
|
|
277
|
+
/** A check the turn's harness can't answer either way, e.g. a web fetch it never reports. */
|
|
278
|
+
var HarnessCapabilityError = class extends Error {
|
|
279
|
+
name = "HarnessCapabilityError";
|
|
280
|
+
};
|
|
281
|
+
const isTurn = (value) => isObjectLike(value) && Array.isArray(value.transcript) && Array.isArray(value.unreported);
|
|
282
|
+
const isContext = (value) => isObjectLike(value) && Array.isArray(value.turns);
|
|
283
|
+
/** A turn from `c.send()`, or `c` (every turn so far). */
|
|
284
|
+
function transcriptSubject(actual, matcher) {
|
|
285
|
+
if (isTurn(actual)) return actual;
|
|
286
|
+
if (isContext(actual)) return {
|
|
287
|
+
transcript: actual.turns.flatMap((t) => t.transcript),
|
|
288
|
+
unreported: [...new Set(actual.turns.flatMap((t) => t.unreported))]
|
|
289
|
+
};
|
|
290
|
+
throw subjectError(`${matcher}() expects a turn from c.send(), or c`);
|
|
291
|
+
}
|
|
292
|
+
const matches = (value, pattern) => pattern === void 0 || (typeof pattern === "string" ? value.includes(pattern) : pattern.test(value));
|
|
293
|
+
const listed = (values) => values.length ? JSON.stringify(values) : "none";
|
|
294
|
+
const targetOf = (pattern) => pattern === void 0 ? "" : ` matching ${pattern}`;
|
|
295
|
+
const toolsOf = (transcript) => transcript.flatMap((e) => e.type === "tool" ? [e] : []);
|
|
296
|
+
const editsOf = (transcript) => transcript.flatMap((e) => e.type === "file-edit" ? [e] : []);
|
|
297
|
+
const skillLike = (event) => {
|
|
298
|
+
const loaded = loadedSkill(event.tool, event.input);
|
|
299
|
+
if (loaded) return `${event.tool}:${loaded}`;
|
|
300
|
+
const target = toolTarget(event.input, event.tool);
|
|
301
|
+
if (target.includes("skills/")) return `${event.tool} ${target}`;
|
|
302
|
+
};
|
|
303
|
+
const isSkillCall = (event, name) => skillsUsed(event.tool, targetText(event.input, event.tool), [name]).length > 0;
|
|
304
|
+
const nameMatches = (tool, name) => typeof name === "string" ? tool === name : name.test(tool);
|
|
305
|
+
function inputMatches(input, expected) {
|
|
306
|
+
if (expected === void 0) return true;
|
|
307
|
+
const text = typeof input === "string" ? input : JSON.stringify(input ?? "");
|
|
308
|
+
if (typeof expected === "string") return text.includes(expected);
|
|
309
|
+
if (expected instanceof RegExp) return expected.test(text);
|
|
310
|
+
if (!isObjectLike(input)) return false;
|
|
311
|
+
return Object.entries(expected).every(([key, value]) => {
|
|
312
|
+
const actual = input[key];
|
|
313
|
+
if (value instanceof RegExp) return value.test(String(actual ?? ""));
|
|
314
|
+
if (typeof value === "string") return actual === value || String(actual ?? "").includes(value);
|
|
315
|
+
return JSON.stringify(actual) === JSON.stringify(value);
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
function webEvent(matcher, actual, type, field, pattern) {
|
|
319
|
+
const subject = transcriptSubject(actual, matcher);
|
|
320
|
+
const what = type === "web-search" ? "web searches" : "web fetches";
|
|
321
|
+
if (subject.unreported.includes(type)) throw new HarnessCapabilityError(`${matcher}() can't check this turn: its harness doesn't report ${what} ('${type}' is in turn.unreported), so the agent may have made them with no event. Check the reply or files instead, or use a harness that reports them.`);
|
|
322
|
+
const seen = subject.transcript.filter((e) => e.type === type).map(field);
|
|
323
|
+
const pass = seen.some((v) => matches(v, pattern));
|
|
324
|
+
return {
|
|
325
|
+
pass,
|
|
326
|
+
message: () => pass ? `expected no ${what}${targetOf(pattern)}, got ${listed(seen)}` : `expected ${what}${targetOf(pattern)}, got ${listed(seen)}`
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
function calledSkill(actual, name) {
|
|
330
|
+
const { transcript } = transcriptSubject(actual, "toHaveCalledSkill");
|
|
331
|
+
const calls = toolsOf(transcript);
|
|
332
|
+
const seen = calls.flatMap((e) => {
|
|
333
|
+
const label = skillLike(e);
|
|
334
|
+
return label ? [label] : [];
|
|
335
|
+
});
|
|
336
|
+
const pass = calls.some((e) => isSkillCall(e, name));
|
|
337
|
+
return {
|
|
338
|
+
pass,
|
|
339
|
+
message: () => pass ? `expected no skill-like calls matching ${name}, got ${listed(seen)}` : `expected skill-like calls matching ${name}, got ${listed(seen)}`
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
function calledTool(actual, name, input) {
|
|
343
|
+
const { transcript } = transcriptSubject(actual, "toHaveCalledTool");
|
|
344
|
+
const calls = toolsOf(transcript);
|
|
345
|
+
const inputPart = input === void 0 ? "" : ` with input ${formatInput(input)}`;
|
|
346
|
+
const seen = calls.map((e) => input === void 0 ? e.tool : `${e.tool} ${JSON.stringify(e.input)}`);
|
|
347
|
+
const pass = calls.some((e) => nameMatches(e.tool, name) && inputMatches(e.input, input));
|
|
348
|
+
return {
|
|
349
|
+
pass,
|
|
350
|
+
message: () => pass ? `expected no tool calls matching ${name}${inputPart}, got ${listed(seen)}` : `expected tool calls matching ${name}${inputPart}, got ${listed(seen)}`
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
const MCP_NAME = /^mcp__(.+)__(.+)$/;
|
|
354
|
+
const mcpLabel = (tool) => {
|
|
355
|
+
const m = MCP_NAME.exec(tool);
|
|
356
|
+
return m ? `${m[1]}/${m[2]}` : tool;
|
|
357
|
+
};
|
|
358
|
+
/** `type: 'tool'` named `mcp__<server>__<tool>` after adapters and the normalizer. */
|
|
359
|
+
function calledMcp(actual, server, tool, input) {
|
|
360
|
+
const subject = transcriptSubject(actual, "toHaveCalledMcp");
|
|
361
|
+
const result = calledTool(subject, `mcp__${server}__${tool}`, input);
|
|
362
|
+
const mcpCalls = toolsOf(subject.transcript).filter((e) => e.tool.startsWith("mcp__"));
|
|
363
|
+
const inputPart = input === void 0 ? "" : ` with input ${formatInput(input)}`;
|
|
364
|
+
const seen = mcpCalls.map((e) => input === void 0 ? mcpLabel(e.tool) : `${mcpLabel(e.tool)} ${JSON.stringify(e.input)}`);
|
|
365
|
+
return {
|
|
366
|
+
pass: result.pass,
|
|
367
|
+
message: () => result.pass ? `expected no MCP calls matching ${server}/${tool}${inputPart}, got ${listed(seen)}` : `expected MCP calls matching ${server}/${tool}${inputPart}, got ${listed(seen)}`
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
function formatInput(input) {
|
|
371
|
+
if (typeof input === "string" || input instanceof RegExp) return String(input);
|
|
372
|
+
return JSON.stringify(input);
|
|
373
|
+
}
|
|
374
|
+
const RUN_MATCH_KEYS = ["program", "args"];
|
|
375
|
+
const isPatternArg = (value) => typeof value === "string" || value instanceof RegExp;
|
|
376
|
+
function checkRunMatch(match) {
|
|
377
|
+
const example = "e.g. toHaveRun({ program: 'genkit', args: /^flow:run\\b/ })";
|
|
378
|
+
for (const key of Object.keys(match)) {
|
|
379
|
+
if (RUN_MATCH_KEYS.includes(key)) continue;
|
|
380
|
+
const near = closest(key, RUN_MATCH_KEYS);
|
|
381
|
+
throw misuseError(`toHaveRun() has no "${key}"; it takes program and args.${near ? ` Did you mean "${near}"?` : ""}`);
|
|
382
|
+
}
|
|
383
|
+
if (!isPatternArg(match.program) || match.program === "") throw misuseError(`toHaveRun()'s program is a program name or a regular expression, ${example} (got ${JSON.stringify(match.program)})`);
|
|
384
|
+
if (match.args !== void 0 && !isPatternArg(match.args)) throw misuseError(`toHaveRun()'s args is text or a regular expression, ${example} (got ${JSON.stringify(match.args)})`);
|
|
385
|
+
return match;
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* `toHaveRun({ program, args })`'s default label, `program genkit args /^flow:run\b/`: unlike
|
|
389
|
+
* `toHaveRun('genkit')`'s, so switching an eval to the object form starts a new history key.
|
|
390
|
+
*/
|
|
391
|
+
function runMatchLabel(match) {
|
|
392
|
+
if (!isObjectLike(match) || match instanceof RegExp || !isPatternArg(match.program)) return;
|
|
393
|
+
const { program, args } = match;
|
|
394
|
+
const programPart = `program ${String(program)}`;
|
|
395
|
+
return isPatternArg(args) ? `${programPart} args ${String(args)}` : programPart;
|
|
396
|
+
}
|
|
397
|
+
function ranProgram(transcript, match) {
|
|
398
|
+
const commands = transcriptCommands(transcript);
|
|
399
|
+
const shown = (c) => [c.program, ...c.args].join(" ");
|
|
400
|
+
const hits = commands.filter((c) => (typeof match.program === "string" ? c.program === match.program : match.program.test(c.program)) && matches(c.args.join(" "), match.args));
|
|
401
|
+
const argsPart = match.args === void 0 ? "" : ` with args matching ${match.args}`;
|
|
402
|
+
const target = `${match.program}${argsPart}`;
|
|
403
|
+
return {
|
|
404
|
+
pass: hits.length > 0,
|
|
405
|
+
message: () => hits.length > 0 ? `expected no runs of ${target}, got ${listed(hits.map(shown))}` : `expected runs of ${target}, got ${listed(commands.map(shown))}`
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
function ran(actual, command) {
|
|
409
|
+
const { transcript } = transcriptSubject(actual, "toHaveRun");
|
|
410
|
+
if (isObjectLike(command) && !(command instanceof RegExp)) return ranProgram(transcript, checkRunMatch(command));
|
|
411
|
+
const shells = transcript.filter(isShellCall);
|
|
412
|
+
const seen = shells.map(commandOf);
|
|
413
|
+
const pass = shells.some((e) => matches(commandOf(e), command));
|
|
414
|
+
return {
|
|
415
|
+
pass,
|
|
416
|
+
message: () => pass ? `expected no shell commands${targetOf(command)}, got ${listed(seen)}` : `expected shell commands${targetOf(command)}, got ${listed(seen)}`
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* The workspace files that changed while the agent ran a turn, set on each turn `c.send` returns.
|
|
421
|
+
* Not the turn's `diff`, which also has earlier turns' changes and the eval's own writes.
|
|
422
|
+
*/
|
|
423
|
+
const TURN_EDITED = Symbol("reeval.turnEdited");
|
|
424
|
+
const editedIn = (turn) => {
|
|
425
|
+
if (!isObjectLike(turn)) return [];
|
|
426
|
+
const own = turn[TURN_EDITED];
|
|
427
|
+
if (Array.isArray(own)) return own;
|
|
428
|
+
return hasDiff(turn) ? turn.diff.files.map((f) => f.path) : [];
|
|
429
|
+
};
|
|
430
|
+
/**
|
|
431
|
+
* Edit tools' `file-edit` events and the files the turn changed in the workspace, so a file
|
|
432
|
+
* written from the shell or by a script counts, and one only read doesn't.
|
|
433
|
+
*/
|
|
434
|
+
function edited(actual, path) {
|
|
435
|
+
const { transcript } = transcriptSubject(actual, "toHaveEdited");
|
|
436
|
+
const changed = isContext(actual) ? actual.turns.flatMap(editedIn) : editedIn(actual);
|
|
437
|
+
const seen = [.../* @__PURE__ */ new Set([...editsOf(transcript).map((e) => e.path), ...changed])];
|
|
438
|
+
const pass = seen.some((p) => matches(p, path));
|
|
439
|
+
return {
|
|
440
|
+
pass,
|
|
441
|
+
message: () => pass ? `expected no file edits${targetOf(path)}, got ${listed(seen)}` : `expected file edits${targetOf(path)}, got ${listed(seen)}`
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
const hasDiff = (value) => isObjectLike(value) && isObjectLike(value.diff) && Array.isArray(value.diff.files);
|
|
445
|
+
/** A turn's changed files, or `c`'s as of its latest turn: the diff is cumulative. */
|
|
446
|
+
function changedFiles(actual) {
|
|
447
|
+
if (hasDiff(actual)) return actual.diff.files;
|
|
448
|
+
if (isContext(actual)) return actual.turns.at(-1)?.diff.files ?? [];
|
|
449
|
+
throw subjectError("toHaveChanged() expects a turn from c.send(), or c");
|
|
450
|
+
}
|
|
451
|
+
/** A glob also matches every file under the directory it names, so `test` is `test/**` too. */
|
|
452
|
+
const pathTest = (pattern) => {
|
|
453
|
+
if (pattern instanceof RegExp) return (path) => pattern.test(path);
|
|
454
|
+
const glob = pattern.replace(/^\.?\/+/, "").replace(/\/+$/, "");
|
|
455
|
+
const self = picomatch(glob, { dot: true });
|
|
456
|
+
const inside = picomatch(`${glob}/**`, { dot: true });
|
|
457
|
+
return (path) => self(path) || inside(path);
|
|
458
|
+
};
|
|
459
|
+
/** Files in the workspace diff, so every write counts: a tool's, a shell command's or a subprocess's. */
|
|
460
|
+
function changed(actual, paths) {
|
|
461
|
+
const files = changedFiles(actual);
|
|
462
|
+
const patterns = paths === void 0 ? [] : typeof paths === "string" || paths instanceof RegExp ? [paths] : [...paths];
|
|
463
|
+
const tests = patterns.map(pathTest);
|
|
464
|
+
const hits = files.filter((f) => tests.length === 0 || tests.some((t) => t(f.path)));
|
|
465
|
+
const label = (list) => list.length ? `[${list.map((f) => `${JSON.stringify(f.path)} (${f.status})`).join(", ")}]` : "none";
|
|
466
|
+
const target = patterns.length ? ` to ${patterns.map(String).join(", ")}` : "";
|
|
467
|
+
return {
|
|
468
|
+
pass: hits.length > 0,
|
|
469
|
+
message: () => hits.length > 0 ? `expected no changes${target}, got ${label(hits)}` : `expected changes${target}, got ${label(files)}`
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
function searched(actual, query) {
|
|
473
|
+
return webEvent("toHaveSearched", actual, "web-search", (e) => "query" in e ? e.query : "", query);
|
|
474
|
+
}
|
|
475
|
+
function fetched(actual, url) {
|
|
476
|
+
return webEvent("toHaveFetched", actual, "web-fetch", (e) => "url" in e ? e.url : "", url);
|
|
477
|
+
}
|
|
478
|
+
const LIMIT_KEYS = [
|
|
479
|
+
"tokens",
|
|
480
|
+
"toolCalls",
|
|
481
|
+
"duration"
|
|
482
|
+
];
|
|
483
|
+
const isBudgetTurn = (value) => isObjectLike(value) && Array.isArray(value.transcript) && typeof value.durationMs === "number";
|
|
484
|
+
const count = (n) => n.toLocaleString("en-US");
|
|
485
|
+
const isCount = (n) => typeof n === "number" && Number.isInteger(n) && n >= 0;
|
|
486
|
+
/** The limits as `toStayWithin`'s default label shows them: `tokens 200,000, bash 10`. */
|
|
487
|
+
function limitsLabel(limits) {
|
|
488
|
+
if (!isObjectLike(limits)) return void 0;
|
|
489
|
+
const parts = [];
|
|
490
|
+
if (isCount(limits.tokens)) parts.push(`tokens ${count(limits.tokens)}`);
|
|
491
|
+
const calls = limits.toolCalls;
|
|
492
|
+
if (isCount(calls)) parts.push(`toolCalls ${count(calls)}`);
|
|
493
|
+
else if (isObjectLike(calls)) for (const [tool, n] of Object.entries(calls)) parts.push(`${tool} ${isCount(n) ? count(n) : String(n)}`);
|
|
494
|
+
if (limits.duration !== void 0) parts.push(`duration ${String(limits.duration)}`);
|
|
495
|
+
return parts.length ? parts.join(", ") : void 0;
|
|
496
|
+
}
|
|
497
|
+
function checkLimits(limits) {
|
|
498
|
+
const example = "e.g. toStayWithin({ tokens: 200_000, toolCalls: { bash: 10 } })";
|
|
499
|
+
if (!isObjectLike(limits)) throw misuseError(`toStayWithin() takes limits, ${example} (got ${JSON.stringify(limits)})`);
|
|
500
|
+
for (const key of Object.keys(limits)) {
|
|
501
|
+
if (LIMIT_KEYS.includes(key)) continue;
|
|
502
|
+
const near = closest(key, LIMIT_KEYS);
|
|
503
|
+
throw misuseError(`toStayWithin() has no limit "${key}"; it takes tokens, toolCalls and duration.${near ? ` Did you mean "${near}"?` : ""}`);
|
|
504
|
+
}
|
|
505
|
+
if (!LIMIT_KEYS.some((k) => limits[k] !== void 0)) throw misuseError(`toStayWithin() needs at least one limit, ${example}`);
|
|
506
|
+
const bad = (what, value) => misuseError(`toStayWithin()'s ${what} is a whole number of 0 or more (got ${typeof value === "number" ? String(value) : JSON.stringify(value)})`);
|
|
507
|
+
if (limits.tokens !== void 0 && !isCount(limits.tokens)) throw bad("tokens", limits.tokens);
|
|
508
|
+
const calls = limits.toolCalls;
|
|
509
|
+
if (calls !== void 0 && !isCount(calls)) {
|
|
510
|
+
if (!isObjectLike(calls) || !Object.keys(calls).length) throw misuseError(`toStayWithin()'s toolCalls is a count, or a count per tool such as { bash: 10 } (got ${typeof calls === "number" ? String(calls) : JSON.stringify(calls)})`);
|
|
511
|
+
for (const [tool, n] of Object.entries(calls)) if (!isCount(n)) throw bad(`toolCalls.${tool}`, n);
|
|
512
|
+
}
|
|
513
|
+
if (limits.duration !== void 0) try {
|
|
514
|
+
parseDuration(limits.duration);
|
|
515
|
+
} catch (error) {
|
|
516
|
+
throw misuseError(`toStayWithin()'s duration: ${error instanceof Error ? error.message : String(error)}`);
|
|
517
|
+
}
|
|
518
|
+
return limits;
|
|
519
|
+
}
|
|
520
|
+
/**
|
|
521
|
+
* `toStayWithin`: what the turn, or every turn of `c`, used, against each limit. A null `usage`
|
|
522
|
+
* can't show it stayed within `tokens`, so that's a `HarnessCapabilityError`, not a failure.
|
|
523
|
+
*/
|
|
524
|
+
function withinLimits(actual, limits, not) {
|
|
525
|
+
if (not) throw misuseError("toStayWithin() can't be negated: a budget checks the agent stayed within it. Check the figure itself instead, e.g. expect(turn.usage.output).toBeGreaterThan(…).");
|
|
526
|
+
const turns = isBudgetTurn(actual) ? [actual] : isContext(actual) ? actual.turns : (() => {
|
|
527
|
+
throw subjectError("toStayWithin() expects a turn from c.send(), or c");
|
|
528
|
+
})();
|
|
529
|
+
const checked = checkLimits(limits);
|
|
530
|
+
const over = [];
|
|
531
|
+
if (checked.tokens !== void 0) {
|
|
532
|
+
if (turns.some((t) => t.usage === null)) throw new HarnessCapabilityError("toStayWithin() can't check tokens: the harness reported no token usage for this turn (turn.usage is null). Drop tokens, or use a harness that reports usage.");
|
|
533
|
+
const used = turns.reduce((sum, t) => sum + totalTokens(t.usage), 0);
|
|
534
|
+
if (used > checked.tokens) over.push(`tokens ${count(used)} > ${count(checked.tokens)} (subagents included)`);
|
|
535
|
+
}
|
|
536
|
+
const transcript = turns.flatMap((t) => t.transcript);
|
|
537
|
+
const calls = checked.toolCalls;
|
|
538
|
+
if (typeof calls === "number") {
|
|
539
|
+
const used = transcript.filter((e) => "toolCallId" in e).length;
|
|
540
|
+
if (used > calls) over.push(`toolCalls ${count(used)} > ${count(calls)}`);
|
|
541
|
+
} else if (calls) {
|
|
542
|
+
const tools = toolsOf(transcript);
|
|
543
|
+
for (const [tool, limit] of Object.entries(calls)) {
|
|
544
|
+
const used = tools.filter((e) => e.tool === tool).length;
|
|
545
|
+
if (used > limit) over.push(`${tool} ${count(used)} > ${count(limit)}`);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
if (checked.duration !== void 0) {
|
|
549
|
+
const limit = parseDuration(checked.duration);
|
|
550
|
+
const used = turns.reduce((sum, t) => sum + t.durationMs, 0);
|
|
551
|
+
if (used > limit) over.push(`duration ${formatDuration(used)} > ${String(checked.duration)}`);
|
|
552
|
+
}
|
|
553
|
+
return {
|
|
554
|
+
pass: over.length === 0,
|
|
555
|
+
message: () => over.length ? `went over its budget: ${over.join(" · ")}` : "stayed within its budget"
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
// src/matchers.ts
|
|
559
|
+
const matcher = (run) => run;
|
|
560
|
+
/**
|
|
561
|
+
* The last `lines` lines, and at most `chars` characters of them: one line can be megabytes.
|
|
562
|
+
* Masked first, so a secret the cut would go through is masked whole.
|
|
563
|
+
*/
|
|
564
|
+
const tail = (s, lines = 20, chars = 2e3) => {
|
|
565
|
+
const last = redactSecrets(s).trimEnd().split("\n").slice(-lines).join("\n");
|
|
566
|
+
return last.length <= chars ? last : `[… ${last.length - chars} earlier characters]${last.slice(-chars)}`;
|
|
567
|
+
};
|
|
568
|
+
const hasFiles = (value) => isObjectLike(value) && typeof value.readFile === "function";
|
|
569
|
+
const show = (value) => {
|
|
570
|
+
if (typeof value === "string") return `'${value}'`;
|
|
571
|
+
try {
|
|
572
|
+
return JSON.stringify(value) ?? String(value);
|
|
573
|
+
} catch {
|
|
574
|
+
return String(value);
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
function succeeded(actual) {
|
|
578
|
+
const r = actual;
|
|
579
|
+
if (typeof r?.exitCode !== "number") throw subjectError("toSucceed() expects a c.exec() result");
|
|
580
|
+
return {
|
|
581
|
+
pass: r.exitCode === 0,
|
|
582
|
+
message: () => r.exitCode === 0 ? "expected command to fail" : `expected command to succeed, exit code ${r.exitCode}\n--- stderr ---\n${tail(r.stderr)}\n--- stdout ---\n${tail(r.stdout)}`
|
|
583
|
+
};
|
|
584
|
+
}
|
|
585
|
+
const isCapture = (value) => isObjectLike(value) && typeof value.finalUrl === "string" && Array.isArray(value.errors) && Array.isArray(value.console);
|
|
586
|
+
/** `capture "home" at /login`, or the URL alone for a capture with no name. */
|
|
587
|
+
const captureAt = (c) => typeof c.name === "string" ? `capture "${c.name}" at ${c.finalUrl}` : c.finalUrl;
|
|
588
|
+
/** The first errors, each cut to its message and first frames; the rest counted. */
|
|
589
|
+
const firstErrors = (c, max = 5) => {
|
|
590
|
+
const shown = c.errors.slice(0, max).map((e) => tail(e.split("\n").slice(0, 4).join("\n"), 4, 600));
|
|
591
|
+
const rest = c.errors.length - shown.length;
|
|
592
|
+
const file = typeof c.name === "string" ? ` in ${c.name}.capture.json` : "";
|
|
593
|
+
return [...shown, ...rest > 0 ? [`… ${rest} more${file}`] : []].join("\n");
|
|
594
|
+
};
|
|
595
|
+
/** What `toPass()` and `.not.toPass()` fail with on a judge call that answered nothing. */
|
|
596
|
+
const judgeFailedMessage = (f) => `judge failed for "${f.question}" (${f.model}): ${f.error}`;
|
|
597
|
+
const MATCHERS = {
|
|
598
|
+
toBe: matcher((actual, [expected]) => {
|
|
599
|
+
const pass = Object.is(actual, expected);
|
|
600
|
+
return {
|
|
601
|
+
pass,
|
|
602
|
+
message: () => pass ? `expected ${show(actual)} not to be ${show(expected)} // Object.is equality` : `expected ${show(actual)} to be ${show(expected)} // Object.is equality`
|
|
603
|
+
};
|
|
604
|
+
}),
|
|
605
|
+
toEqual: matcher((actual, [expected]) => {
|
|
606
|
+
const pass = isDeepStrictEqual(actual, expected);
|
|
607
|
+
return {
|
|
608
|
+
pass,
|
|
609
|
+
message: () => pass ? `expected ${show(actual)} not to deeply equal ${show(expected)}` : `expected ${show(actual)} to deeply equal ${show(expected)}`
|
|
610
|
+
};
|
|
611
|
+
}),
|
|
612
|
+
toBeNull: matcher((actual, _) => ({
|
|
613
|
+
pass: actual === null,
|
|
614
|
+
message: () => actual === null ? `expected ${show(actual)} not to be null` : `expected ${show(actual)} to be null`
|
|
615
|
+
})),
|
|
616
|
+
toBeTruthy: matcher((actual, _) => ({
|
|
617
|
+
pass: !!actual,
|
|
618
|
+
message: () => actual ? `expected ${show(actual)} to be falsy` : `expected ${show(actual)} to be truthy`
|
|
619
|
+
})),
|
|
620
|
+
toContain: matcher((actual, [needle]) => {
|
|
621
|
+
const pass = typeof actual === "string" && typeof needle === "string" ? actual.includes(needle) : Array.isArray(actual) && actual.includes(needle);
|
|
622
|
+
return {
|
|
623
|
+
pass,
|
|
624
|
+
message: () => pass ? `expected ${show(actual)} not to contain ${show(needle)}` : `expected ${show(actual)} to contain ${show(needle)}`
|
|
625
|
+
};
|
|
626
|
+
}),
|
|
627
|
+
toHaveFile: matcher(async (actual, [path, pattern]) => {
|
|
628
|
+
if (!hasFiles(actual)) throw subjectError("toHaveFile() expects a turn from c.send(), or c");
|
|
629
|
+
const content = await actual.readFile(path);
|
|
630
|
+
if (content === null) return {
|
|
631
|
+
pass: false,
|
|
632
|
+
message: () => `expected file ${path} to exist`
|
|
633
|
+
};
|
|
634
|
+
if (pattern === void 0) return {
|
|
635
|
+
pass: true,
|
|
636
|
+
message: () => `expected no file at ${path}`
|
|
637
|
+
};
|
|
638
|
+
const pass = typeof pattern === "string" ? content.includes(pattern) : pattern.test(content);
|
|
639
|
+
return {
|
|
640
|
+
pass,
|
|
641
|
+
message: () => `expected ${path} ${pass ? "not " : ""}to match ${pattern}\n--- ${path} ---\n${tail(content)}`
|
|
642
|
+
};
|
|
643
|
+
}),
|
|
644
|
+
toSucceed: matcher((actual, _) => isThenable(actual) ? Promise.resolve(actual).then(succeeded) : succeeded(actual)),
|
|
645
|
+
toHaveSearched: matcher((actual, [query]) => searched(actual, query)),
|
|
646
|
+
toHaveFetched: matcher((actual, [url]) => fetched(actual, url)),
|
|
647
|
+
toHaveCalledSkill: matcher((actual, [name]) => calledSkill(actual, name)),
|
|
648
|
+
toHaveCalledMcp: matcher((actual, [server, tool, input]) => calledMcp(actual, server, tool, input)),
|
|
649
|
+
toHaveCalledTool: matcher((actual, [name, input]) => calledTool(actual, name, input)),
|
|
650
|
+
toHaveRun: matcher((actual, [command]) => ran(actual, command)),
|
|
651
|
+
toHaveEdited: matcher((actual, [path]) => edited(actual, path)),
|
|
652
|
+
toHaveChanged: matcher((actual, [paths]) => changed(actual, paths)),
|
|
653
|
+
toStayWithin: matcher((actual, [limits], { not }) => withinLimits(actual, limits, not)),
|
|
654
|
+
toHaveStatus: matcher((actual, [status]) => {
|
|
655
|
+
if (!isCapture(actual)) throw subjectError("toHaveStatus() expects a c.capture() result");
|
|
656
|
+
const range = typeof status === "string" && /^[1-5]xx$/.test(status) ? Number(status[0]) : 0;
|
|
657
|
+
if (!range && !(typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599)) throw subjectError(`toHaveStatus() expects a status from 100 to 599 or a range from '1xx' to '5xx', got ${JSON.stringify(status)}`);
|
|
658
|
+
const pass = actual.status !== void 0 && (range ? Math.floor(actual.status / 100) === range : actual.status === status);
|
|
659
|
+
return {
|
|
660
|
+
pass,
|
|
661
|
+
message: () => pass ? `expected ${captureAt(actual)} not to answer ${status}` : actual.status === void 0 ? `expected ${captureAt(actual)} to answer ${status}, but no response came` : `expected ${captureAt(actual)} to answer ${status}, got ${actual.status}`
|
|
662
|
+
};
|
|
663
|
+
}),
|
|
664
|
+
toHaveNoPageErrors: matcher((actual, _) => {
|
|
665
|
+
if (!isCapture(actual)) throw subjectError("toHaveNoPageErrors() expects a c.capture() result");
|
|
666
|
+
const pass = actual.errors.length === 0;
|
|
667
|
+
return {
|
|
668
|
+
pass,
|
|
669
|
+
message: () => pass ? `expected ${captureAt(actual)} to have page errors` : `expected no page errors at ${captureAt(actual)}, got ${plural(actual.errors.length, "page error")}: ${firstErrors(actual)}`
|
|
670
|
+
};
|
|
671
|
+
}),
|
|
672
|
+
toPass: matcher((actual, _, { not }) => {
|
|
673
|
+
const failure = actual;
|
|
674
|
+
if (failure?.verdict === "error" && typeof failure.error === "string") return {
|
|
675
|
+
pass: not,
|
|
676
|
+
message: () => judgeFailedMessage(failure)
|
|
677
|
+
};
|
|
678
|
+
const v = actual;
|
|
679
|
+
if (typeof v?.score !== "number") throw subjectError("toPass() expects a c.judge() verdict");
|
|
680
|
+
const score = v.score.toFixed(2);
|
|
681
|
+
return {
|
|
682
|
+
pass: v.verdict === "pass",
|
|
683
|
+
message: () => v.verdict === "pass" ? `expected judge to fail: score ${score} ≥ ${v.threshold} for "${v.question}"` : `judge score ${score} is below ${v.threshold} for "${v.question}" (${v.model})`
|
|
684
|
+
};
|
|
685
|
+
})
|
|
686
|
+
};
|
|
687
|
+
/** Vitest/Jest already ship these; the protocol table still runs them for Python and other SDKs. */
|
|
688
|
+
const GENERIC_MATCHERS = /* @__PURE__ */ new Set([
|
|
689
|
+
"toBe",
|
|
690
|
+
"toEqual",
|
|
691
|
+
"toBeNull",
|
|
692
|
+
"toBeTruthy",
|
|
693
|
+
"toContain"
|
|
694
|
+
]);
|
|
695
|
+
const isMatcherName = (name) => Object.hasOwn(MATCHERS, name);
|
|
696
|
+
/** Runs `name`'s row on arguments its signature hasn't checked, as Chai and the protocol pass them. */
|
|
697
|
+
const runMatcher = (name, actual, args, flags) => MATCHERS[name](actual, args, flags);
|
|
698
|
+
/** The table as `expect.extend` takes it: Chai passes `.not` as `this.isNot`. */
|
|
699
|
+
const matchers = Object.fromEntries(Object.keys(MATCHERS).map((name) => [name, function(actual, ...args) {
|
|
700
|
+
return runMatcher(name, actual, args, { not: !!this?.isNot });
|
|
701
|
+
}]));
|
|
702
|
+
/** Chai `expect.extend` in eval workers: reeval-specific matchers only (not Jest's toEqual, …). */
|
|
703
|
+
const evalMatchers = Object.fromEntries(Object.keys(matchers).filter((name) => !GENERIC_MATCHERS.has(name)).map((name) => [name, matchers[name]]));
|
|
704
|
+
// src/scheduler/client.ts
|
|
705
|
+
var LocalTransport = class {
|
|
706
|
+
#pending = /* @__PURE__ */ new Map();
|
|
707
|
+
#scheduler = new Scheduler((r) => {
|
|
708
|
+
this.#pending.get(r.id)?.resolve();
|
|
709
|
+
this.#pending.delete(r.id);
|
|
710
|
+
});
|
|
711
|
+
request(id, limits, rank, pending) {
|
|
712
|
+
this.#pending.set(id, pending);
|
|
713
|
+
this.#scheduler.request({
|
|
714
|
+
id,
|
|
715
|
+
owner: "local",
|
|
716
|
+
limits,
|
|
717
|
+
...rank ? { rank } : {}
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
release(id) {
|
|
721
|
+
this.#pending.delete(id);
|
|
722
|
+
this.#scheduler.release(id);
|
|
723
|
+
}
|
|
724
|
+
};
|
|
725
|
+
/** Over the worker's IPC channel to reeval's own engine: requests out, grants back. */
|
|
726
|
+
var ChannelTransport = class {
|
|
727
|
+
#pending = /* @__PURE__ */ new Map();
|
|
728
|
+
#send;
|
|
729
|
+
constructor(send) {
|
|
730
|
+
this.#send = send;
|
|
731
|
+
}
|
|
732
|
+
grant(id) {
|
|
733
|
+
const pending = this.#pending.get(id);
|
|
734
|
+
this.#pending.delete(id);
|
|
735
|
+
pending?.resolve();
|
|
736
|
+
}
|
|
737
|
+
request(id, limits, rank, pending) {
|
|
738
|
+
this.#pending.set(id, pending);
|
|
739
|
+
this.#send({
|
|
740
|
+
t: "request",
|
|
741
|
+
id,
|
|
742
|
+
limits,
|
|
743
|
+
...rank ? { rank } : {}
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
release(id) {
|
|
747
|
+
this.#pending.delete(id);
|
|
748
|
+
this.#send({
|
|
749
|
+
t: "release",
|
|
750
|
+
id
|
|
751
|
+
});
|
|
752
|
+
}
|
|
753
|
+
};
|
|
754
|
+
let transport;
|
|
755
|
+
let ids = 0;
|
|
756
|
+
/** Points this process at the run's scheduler over `send`; returns what delivers its grants. */
|
|
757
|
+
function useSchedulerChannel(send) {
|
|
758
|
+
const channel = new ChannelTransport(send);
|
|
759
|
+
transport = channel;
|
|
760
|
+
return (id) => channel.grant(id);
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* Waits until every resource in `limits` has room, then holds a lease on each until released.
|
|
764
|
+
* Rejects with the signal's reason if it aborts first.
|
|
765
|
+
*/
|
|
766
|
+
function acquire(limits, options = {}) {
|
|
767
|
+
const { rank, signal } = options;
|
|
768
|
+
if (signal?.aborted) return Promise.reject(signal.reason);
|
|
769
|
+
transport ??= new LocalTransport();
|
|
770
|
+
const via = transport;
|
|
771
|
+
const id = String(++ids);
|
|
772
|
+
let released = false;
|
|
773
|
+
const lease = { release() {
|
|
774
|
+
if (released) return;
|
|
775
|
+
released = true;
|
|
776
|
+
via.release(id);
|
|
777
|
+
} };
|
|
778
|
+
return new Promise((resolve, reject) => {
|
|
779
|
+
const onAbort = () => {
|
|
780
|
+
lease.release();
|
|
781
|
+
reject(signal?.reason);
|
|
782
|
+
};
|
|
783
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
784
|
+
via.request(id, limits, rank, {
|
|
785
|
+
resolve: () => {
|
|
786
|
+
signal?.removeEventListener("abort", onAbort);
|
|
787
|
+
resolve(lease);
|
|
788
|
+
},
|
|
789
|
+
reject: (error) => {
|
|
790
|
+
signal?.removeEventListener("abort", onAbort);
|
|
791
|
+
reject(error);
|
|
792
|
+
}
|
|
793
|
+
});
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
/** Runs `fn` while holding leases on `limits`. */
|
|
797
|
+
async function withLease(limits, fn, signal) {
|
|
798
|
+
const lease = await acquire(limits, signal ? { signal } : {});
|
|
799
|
+
try {
|
|
800
|
+
return await fn();
|
|
801
|
+
} finally {
|
|
802
|
+
lease.release();
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
export { evalMatchers as a, runMatcher as c, limitsLabel as d, runMatchLabel as f, MATCHERS as i, HarnessCapabilityError as l, useSchedulerChannel as n, isMatcherName as o, shellCommands as p, withLease as r, judgeFailedMessage as s, acquire as t, TURN_EDITED as u };
|
|
806
|
+
|
|
807
|
+
//# sourceMappingURL=client-DMEYR3MA.js.map
|