reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -11,6 +11,10 @@ type EffortSample = {
|
|
|
11
11
|
filesChanged: number;
|
|
12
12
|
/** Total tokens when this attempt reported usage; omitted when the harness dropped it. */
|
|
13
13
|
tokens?: number;
|
|
14
|
+
/** Where its agent time went; omitted for an attempt recorded without it. */
|
|
15
|
+
split?: TimeSplit;
|
|
16
|
+
/** The agent's cost in US dollars (`cost.usd`); omitted when the attempt has none. */
|
|
17
|
+
usd?: number;
|
|
14
18
|
};
|
|
15
19
|
type Spread = {
|
|
16
20
|
median: number;
|
|
@@ -28,6 +32,13 @@ type CellEffort = {
|
|
|
28
32
|
filesChanged: number;
|
|
29
33
|
/** Over the counted attempts that reported usage; omitted when none did. */
|
|
30
34
|
tokens?: Spread;
|
|
35
|
+
/**
|
|
36
|
+
* Median model, tool, waiting and other time over the counted attempts that have a split;
|
|
37
|
+
* omitted when none does. `modelMs` is null when any of them saw no model calls.
|
|
38
|
+
*/
|
|
39
|
+
split?: TimeSplit;
|
|
40
|
+
/** The agent's cost in US dollars over the counted attempts that have one; omitted when none does. */
|
|
41
|
+
usd?: Spread;
|
|
31
42
|
};
|
|
32
43
|
/**
|
|
33
44
|
* `fewer` / `more`: the exact rank test's p is below `EFFORT_ALPHA`. `noise`: it isn't.
|
|
@@ -67,11 +78,13 @@ type EffortContrast = {
|
|
|
67
78
|
/** Set when some counted attempts on either side reported no usage. Not a verdict. */
|
|
68
79
|
tokensNote?: typeof TOKENS_UNREPORTED;
|
|
69
80
|
};
|
|
70
|
-
declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage' | 'changedFiles'>) => EffortSample;
|
|
81
|
+
declare const effortSample: (a: Pick<AttemptRecord, 'activity' | 'timings' | 'usage' | 'changedFiles' | 'timeSplit' | 'cost'>) => EffortSample;
|
|
71
82
|
/** The files the workspace diff changed, a collapsed directory counting each of its files. */
|
|
72
83
|
declare const filesChanged: (files: AttemptRecord['changedFiles']) => number;
|
|
73
84
|
/** Token totals from the attempts that reported usage; empty when none did. */
|
|
74
85
|
declare const tokenValues: (samples: EffortSample[]) => number[];
|
|
86
|
+
/** The samples' median split, `modelMs` null when any sample's is; undefined when none has one. */
|
|
87
|
+
declare function splitOf(samples: EffortSample[]): TimeSplit | undefined;
|
|
75
88
|
declare function effortOf(samples: EffortSample[]): CellEffort | null;
|
|
76
89
|
/**
|
|
77
90
|
* The verdict of an exact rank test of `xs` (variant) against `ys` (control). Its direction is
|
|
@@ -109,6 +122,11 @@ declare const formatRatio: (r: number | null) => string;
|
|
|
109
122
|
declare const bothUnderATenth: (a: number, b: number) => boolean;
|
|
110
123
|
/** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
|
|
111
124
|
declare function formatSeconds(ms: number): string;
|
|
125
|
+
/**
|
|
126
|
+
* `model 1m 05s · tools 12s · waiting 1m 21s · other 3m 22s`: model left out when unknown,
|
|
127
|
+
* waiting when there was none.
|
|
128
|
+
*/
|
|
129
|
+
declare const splitLabel: (s: TimeSplit) => string;
|
|
112
130
|
declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
|
|
113
131
|
/**
|
|
114
132
|
* `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
|
|
@@ -118,10 +136,16 @@ declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
|
|
|
118
136
|
* times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
|
|
119
137
|
*/
|
|
120
138
|
declare function effortLabel(c: EffortContrast): string;
|
|
121
|
-
/**
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
139
|
+
/**
|
|
140
|
+
* One cell's effort on its own: `tools 43 · agent 46s`, plus tokens and cost when any attempt
|
|
141
|
+
* reported them.
|
|
142
|
+
*/
|
|
143
|
+
declare const effortSummary: (e: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'usd'>) => string;
|
|
144
|
+
/**
|
|
145
|
+
* The same cell between two runs: `tools 44 → 10 (×0.23) · agent 45s → 21s (×0.47)`, then
|
|
146
|
+
* `waiting 0s → 1m 21s` when either side waited on a background job.
|
|
147
|
+
*/
|
|
148
|
+
declare function effortChangeLabel(base: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'split' | 'usd'>, head: Pick<CellEffort, 'tools' | 'agentMs' | 'tokens' | 'split' | 'usd'>): string;
|
|
125
149
|
/** Stays 1 while reeval is pre-release: the record changes in place. */
|
|
126
150
|
declare const SCHEMA_VERSION = 1;
|
|
127
151
|
type ArtifactLevel = 'all' | 'failures' | 'none';
|
|
@@ -132,6 +156,16 @@ type FileRef = {
|
|
|
132
156
|
sha256: string;
|
|
133
157
|
bytes: number;
|
|
134
158
|
};
|
|
159
|
+
type RunDebug = {
|
|
160
|
+
/** `requests`: each attempt's `requests.ndjson`; `all`: its `debug.log` and `trace.ndjson` too. */
|
|
161
|
+
level: 'requests' | 'all';
|
|
162
|
+
files: FileRef[];
|
|
163
|
+
/** Files left out because the secret scan found something in them, with what it found. */
|
|
164
|
+
dropped?: Array<{
|
|
165
|
+
path: string;
|
|
166
|
+
note: string;
|
|
167
|
+
}>;
|
|
168
|
+
};
|
|
135
169
|
/** A generic attempt file from `c.capture()`. */
|
|
136
170
|
type ArtifactRef = FileRef & {
|
|
137
171
|
name: string;
|
|
@@ -222,6 +256,10 @@ type JudgmentRecord = {
|
|
|
222
256
|
upstream?: string;
|
|
223
257
|
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
224
258
|
costUsd?: number;
|
|
259
|
+
/** The call's tokens; unset when the provider reported none, and for the fake judge. */
|
|
260
|
+
usage?: TokenUsage;
|
|
261
|
+
/** `costUsd` as reported, else the call's tokens at the judge model's list price; unset when neither is known. */
|
|
262
|
+
cost?: CostUsd;
|
|
225
263
|
/**
|
|
226
264
|
* `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
|
|
227
265
|
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
@@ -292,6 +330,13 @@ type AttemptRecord = {
|
|
|
292
330
|
* other sandboxes, and in attempts that didn't reach their sandbox.
|
|
293
331
|
*/
|
|
294
332
|
vmImage?: VmImage;
|
|
333
|
+
/**
|
|
334
|
+
* The container or VM the attempt ran on and the CPU, memory and disk it used there, measured
|
|
335
|
+
* out of the agent's reach: a container's from a sibling container, a VM's as root
|
|
336
|
+
* (`AttemptResources`). Absent on the fake sandbox, which runs on the host, and in attempts that
|
|
337
|
+
* didn't reach their sandbox.
|
|
338
|
+
*/
|
|
339
|
+
resources?: AttemptResources;
|
|
295
340
|
/** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
|
|
296
341
|
env: {
|
|
297
342
|
vars: string[];
|
|
@@ -348,8 +393,7 @@ type AttemptRecord = {
|
|
|
348
393
|
* `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
|
|
349
394
|
* file.
|
|
350
395
|
* `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
|
|
351
|
-
* `reasoningMs` is how long the turn's
|
|
352
|
-
* time it.
|
|
396
|
+
* `reasoningMs` is how long the turn's reasoning took, null when the harness can't time it.
|
|
353
397
|
*/
|
|
354
398
|
turns: Array<{
|
|
355
399
|
prompt: string;
|
|
@@ -361,6 +405,11 @@ type AttemptRecord = {
|
|
|
361
405
|
usage: TokenUsage | null;
|
|
362
406
|
/** The model the harness says served the turn; null when it reports none. */
|
|
363
407
|
model: string | null;
|
|
408
|
+
/**
|
|
409
|
+
* What the harness says the turn cost (Claude Code, pi), else its tokens at the list price of
|
|
410
|
+
* its model (or the cell's); unset when neither is known.
|
|
411
|
+
*/
|
|
412
|
+
cost?: CostUsd;
|
|
364
413
|
/** Event types the harness can't report; see `Turn.unreported`. */
|
|
365
414
|
unreported: WebEventType[];
|
|
366
415
|
/** The turn's first error, such as a provider's billing or auth error, cut at 1 KB; else null. */
|
|
@@ -377,9 +426,25 @@ type AttemptRecord = {
|
|
|
377
426
|
* script's.
|
|
378
427
|
*/
|
|
379
428
|
reasoningMs?: number | null;
|
|
429
|
+
/**
|
|
430
|
+
* Where the turn's time went: its model calls, tool calls, the background jobs it waited on,
|
|
431
|
+
* and the rest, summing to `durationMs` (`TimeSplit`). Absent in records from before reeval
|
|
432
|
+
* split it.
|
|
433
|
+
*/
|
|
434
|
+
timeSplit?: TimeSplit;
|
|
380
435
|
}>;
|
|
436
|
+
/**
|
|
437
|
+
* The turns' `timeSplit` added up, its `modelMs` null when any turn's is. Absent when no turn
|
|
438
|
+
* has one.
|
|
439
|
+
*/
|
|
440
|
+
timeSplit?: TimeSplit;
|
|
381
441
|
/** Tokens across the attempt's turns; null when the harness reported none. */
|
|
382
442
|
usage: TokenUsage | null;
|
|
443
|
+
/**
|
|
444
|
+
* The gateway's billed `reportedCostUsd`, else the turns' costs added up; unset when a turn with
|
|
445
|
+
* tokens has none.
|
|
446
|
+
*/
|
|
447
|
+
cost?: CostUsd;
|
|
383
448
|
/**
|
|
384
449
|
* Tokens the capture vision loop used. Infrastructure only: never included
|
|
385
450
|
* in `usage`, effort, compare, or lift.
|
|
@@ -418,7 +483,10 @@ type AttemptRecord = {
|
|
|
418
483
|
maxToolMs: number | null;
|
|
419
484
|
};
|
|
420
485
|
judgements: JudgmentRecord[];
|
|
421
|
-
/**
|
|
486
|
+
/**
|
|
487
|
+
* Harness session starts that didn't become ready within `startTimeout`, or failed on a transient
|
|
488
|
+
* error (a 5xx or a network error), and were started again (at most one).
|
|
489
|
+
*/
|
|
422
490
|
startRetries?: StartRetry[];
|
|
423
491
|
/**
|
|
424
492
|
* The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
|
|
@@ -810,6 +878,11 @@ type CellRecord = Cell & {
|
|
|
810
878
|
* gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
|
|
811
879
|
*/
|
|
812
880
|
bestOf?: number;
|
|
881
|
+
/** The cell's attempts' agent and judge costs added up; unset where none has one. */
|
|
882
|
+
cost?: {
|
|
883
|
+
agent?: CostTotal;
|
|
884
|
+
judge?: CostTotal;
|
|
885
|
+
};
|
|
813
886
|
runs: number;
|
|
814
887
|
pass: number;
|
|
815
888
|
fail: number;
|
|
@@ -867,7 +940,7 @@ type RunSummary = {
|
|
|
867
940
|
infraError: number;
|
|
868
941
|
skipped: number;
|
|
869
942
|
};
|
|
870
|
-
/** The tokens the run used, as the harnesses count them
|
|
943
|
+
/** The tokens the run used, as the harnesses count them, and what they cost. */
|
|
871
944
|
type Cost = {
|
|
872
945
|
/** Prompt tokens, cache reads and writes included. */
|
|
873
946
|
inputTokens: number | null;
|
|
@@ -875,6 +948,10 @@ type Cost = {
|
|
|
875
948
|
/** The same totals split as each turn reports them. */
|
|
876
949
|
usage: TokenUsage | null;
|
|
877
950
|
source: 'reported' | 'unreported';
|
|
951
|
+
/** The attempts' costs added up; unset when none has one. */
|
|
952
|
+
agentUsd?: CostTotal;
|
|
953
|
+
/** The judge calls' costs added up; unset when none has one. */
|
|
954
|
+
judgeUsd?: CostTotal;
|
|
878
955
|
};
|
|
879
956
|
/**
|
|
880
957
|
* The `reeval run` settings that shaped a run's plan, as it was first started. A resume keeps
|
|
@@ -893,6 +970,11 @@ type RunInvocation = {
|
|
|
893
970
|
tags: string[];
|
|
894
971
|
/** `--trait`, as given (`key=value`); unset without any. */
|
|
895
972
|
traits?: string[];
|
|
973
|
+
/**
|
|
974
|
+
* The file `--events` wrote the run's event stream to, from the project root in posix form
|
|
975
|
+
* (`../…` outside it), so a resume appends to it; unset without `--events`.
|
|
976
|
+
*/
|
|
977
|
+
events?: string;
|
|
896
978
|
};
|
|
897
979
|
type RunRecord = {
|
|
898
980
|
kind: 'reeval-run';
|
|
@@ -988,6 +1070,11 @@ type RunRecord = {
|
|
|
988
1070
|
cost: Cost;
|
|
989
1071
|
/** Which artifacts sit next to this record. Refs to files that weren't kept are removed. */
|
|
990
1072
|
artifacts: ArtifactLevel;
|
|
1073
|
+
/**
|
|
1074
|
+
* The attempts' debug data uploaded with the run to its store (`history.debug`), each file by
|
|
1075
|
+
* its path in `.reeval/debug/<run>/` (`<eval>/<cell>/<n>/requests.ndjson`); absent when none was.
|
|
1076
|
+
*/
|
|
1077
|
+
debug?: RunDebug;
|
|
991
1078
|
redaction: {
|
|
992
1079
|
version: 1;
|
|
993
1080
|
};
|
|
@@ -1205,6 +1292,11 @@ type TranscriptEvent = EventTime & ({
|
|
|
1205
1292
|
durationMs: number | null;
|
|
1206
1293
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1207
1294
|
stalled?: boolean;
|
|
1295
|
+
/**
|
|
1296
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1297
|
+
* event's `atMs` (the turn's end for the last).
|
|
1298
|
+
*/
|
|
1299
|
+
durationInferred?: true;
|
|
1208
1300
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1209
1301
|
subagent?: string;
|
|
1210
1302
|
}) | {
|
|
@@ -1221,6 +1313,11 @@ type TranscriptEvent = EventTime & ({
|
|
|
1221
1313
|
durationMs: number | null;
|
|
1222
1314
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1223
1315
|
stalled?: boolean;
|
|
1316
|
+
/**
|
|
1317
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1318
|
+
* event's `atMs` (the turn's end for the last).
|
|
1319
|
+
*/
|
|
1320
|
+
durationInferred?: true;
|
|
1224
1321
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1225
1322
|
subagent?: string;
|
|
1226
1323
|
} | {
|
|
@@ -1232,6 +1329,11 @@ type TranscriptEvent = EventTime & ({
|
|
|
1232
1329
|
durationMs: number | null;
|
|
1233
1330
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1234
1331
|
stalled?: boolean;
|
|
1332
|
+
/**
|
|
1333
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1334
|
+
* event's `atMs` (the turn's end for the last).
|
|
1335
|
+
*/
|
|
1336
|
+
durationInferred?: true;
|
|
1235
1337
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1236
1338
|
subagent?: string;
|
|
1237
1339
|
} | {
|
|
@@ -1240,11 +1342,31 @@ type TranscriptEvent = EventTime & ({
|
|
|
1240
1342
|
tool: string;
|
|
1241
1343
|
input: unknown;
|
|
1242
1344
|
output: unknown;
|
|
1345
|
+
/**
|
|
1346
|
+
* What `output` holds: `model`, the result as the harness gave it to the model; `full`,
|
|
1347
|
+
* the tool's whole output, which can be more than the model saw (the harness cut it for
|
|
1348
|
+
* the model, or saved it to a file and gave the model a pointer, which reeval read).
|
|
1349
|
+
*/
|
|
1350
|
+
outputView?: 'model' | 'full';
|
|
1351
|
+
/**
|
|
1352
|
+
* Set when `output` is only part of the output, with a line where the rest was cut:
|
|
1353
|
+
* `transcript.jsonl` keeps the first 96K and last 16K characters of one past 128K (of its
|
|
1354
|
+
* JSON, for structured output), and the harness adapter keeps the first and last 512 KiB
|
|
1355
|
+
* of a saved output past 1 MiB. `bytes` is the whole output's size in UTF-8.
|
|
1356
|
+
*/
|
|
1357
|
+
outputTruncated?: {
|
|
1358
|
+
bytes: number;
|
|
1359
|
+
};
|
|
1243
1360
|
/** Whether the call succeeded; null when the harness never reported its result. */
|
|
1244
1361
|
ok: boolean | null;
|
|
1245
1362
|
durationMs: number | null;
|
|
1246
1363
|
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
1247
1364
|
stalled?: boolean;
|
|
1365
|
+
/**
|
|
1366
|
+
* Set when the harness never reported the call's end, so `durationMs` runs to the next
|
|
1367
|
+
* event's `atMs` (the turn's end for the last).
|
|
1368
|
+
*/
|
|
1369
|
+
durationInferred?: true;
|
|
1248
1370
|
/** The `toolCallId` of the subagent call whose agent made this call; absent for the agent's own. */
|
|
1249
1371
|
subagent?: string;
|
|
1250
1372
|
} | {
|
|
@@ -1255,7 +1377,40 @@ type TranscriptEvent = EventTime & ({
|
|
|
1255
1377
|
{
|
|
1256
1378
|
type: 'retry';
|
|
1257
1379
|
message: string;
|
|
1258
|
-
});
|
|
1380
|
+
} | ModelCallEvent);
|
|
1381
|
+
/**
|
|
1382
|
+
* One of the agent's calls to its model API, as the egress gateway saw it (in a sandbox behind
|
|
1383
|
+
* it, such as Docker's): when it started, when its response's first byte came (on a streamed call,
|
|
1384
|
+
* about its first token) and when it ended, with the tokens and the model its response named.
|
|
1385
|
+
* Cursor sends a whole turn over one long stream to `api2.cursor.sh` (`AgentService/RunSSE`),
|
|
1386
|
+
* so a Cursor turn has one call, with `stream: true`, its tool calls inside it and its tokens
|
|
1387
|
+
* only in the turn's `usage`.
|
|
1388
|
+
*/
|
|
1389
|
+
type ModelCallEvent = {
|
|
1390
|
+
type: 'model-call';
|
|
1391
|
+
/** `POST api.anthropic.com/v1/messages`: the method, host and path. */
|
|
1392
|
+
request: string;
|
|
1393
|
+
/** The response's HTTP status; null when none came. */
|
|
1394
|
+
status: number | null;
|
|
1395
|
+
durationMs: number;
|
|
1396
|
+
/** Milliseconds from the call's start to its response body's first byte; null when none came. */
|
|
1397
|
+
firstByteMs: number | null;
|
|
1398
|
+
/** The tokens its response body named; null when it named none (Cursor's are protobuf). */
|
|
1399
|
+
usage: TokenUsage | null;
|
|
1400
|
+
/** The model that served it, from its request or response; null when neither named one. */
|
|
1401
|
+
model: string | null;
|
|
1402
|
+
/** Set on a call that carries the whole turn (Cursor's), so it isn't counted as model time. */
|
|
1403
|
+
stream?: true;
|
|
1404
|
+
/**
|
|
1405
|
+
* `main` for one of the agent's own steps, `side` for a call its CLI made around them: a
|
|
1406
|
+
* session title, a web search's own model call, Antigravity's Flash Lite calls. A call is
|
|
1407
|
+
* `main` when its tokens are one of the turn's steps' as the harness reported them. Otherwise
|
|
1408
|
+
* it's `side` when its request declared at most one tool while another call in the attempt
|
|
1409
|
+
* declared more, or when another model served it than served the turn's other calls, and `main`
|
|
1410
|
+
* when neither holds. Absent in records from before roles.
|
|
1411
|
+
*/
|
|
1412
|
+
role?: 'main' | 'side';
|
|
1413
|
+
};
|
|
1259
1414
|
/** The transcript events a harness may be unable to report. */
|
|
1260
1415
|
type WebEventType = 'web-search' | 'web-fetch';
|
|
1261
1416
|
type ExecResult = {
|
|
@@ -1270,6 +1425,13 @@ type ExecRetry = {
|
|
|
1270
1425
|
/** Milliseconds to wait before each extra try. Default 1000. */
|
|
1271
1426
|
interval?: number;
|
|
1272
1427
|
};
|
|
1428
|
+
/**
|
|
1429
|
+
* Variables `c.exec` sets for one command, over the environment's own. A value whose name looks
|
|
1430
|
+
* like a secret's (`KEY`, `TOKEN`, `SECRET`, `PASSWORD`, `CREDENTIAL`) is masked in everything
|
|
1431
|
+
* reeval writes, as a host variable's is. The host value of a secret the environment brokers
|
|
1432
|
+
* goes in as its placeholder, which the gateway swaps back on that secret's hosts.
|
|
1433
|
+
*/
|
|
1434
|
+
type ExecEnv = Record<string, string>;
|
|
1273
1435
|
/** One run of a `c.exec` command that had `retries`. */
|
|
1274
1436
|
type ExecTry = {
|
|
1275
1437
|
exitCode: number;
|
|
@@ -1487,6 +1649,45 @@ type TokenUsage = {
|
|
|
1487
1649
|
cacheWrite: number;
|
|
1488
1650
|
reasoning: number | null;
|
|
1489
1651
|
};
|
|
1652
|
+
/**
|
|
1653
|
+
* Where a turn's time went, in ms, summing to its `durationMs`. Each moment counts once, as the
|
|
1654
|
+
* first of: `waitingMs`, a background job the agent waited on (from the call that started it
|
|
1655
|
+
* through the polls that checked on it), `toolMs`, a tool call, `modelMs`, a model call, and
|
|
1656
|
+
* `otherMs`, the rest (the harness's own work, and time between calls). `modelMs` is null when
|
|
1657
|
+
* the turn's model calls weren't seen one by one: on a sandbox without the egress gateway, and
|
|
1658
|
+
* on Cursor, whose turn is one stream; that time is in `otherMs`. Which calls wait is a heuristic
|
|
1659
|
+
* over the transcript: a shell call run in the background (`run_in_background`, a trailing `&`,
|
|
1660
|
+
* `nohup`, or the command a Cursor terminal file names), then `sleep` and `wait` commands, output
|
|
1661
|
+
* tools (`BashOutput`, `TaskOutput`) and reads of a Cursor terminal file.
|
|
1662
|
+
*/
|
|
1663
|
+
type TimeSplit = {
|
|
1664
|
+
modelMs: number | null;
|
|
1665
|
+
toolMs: number;
|
|
1666
|
+
waitingMs: number;
|
|
1667
|
+
otherMs: number;
|
|
1668
|
+
};
|
|
1669
|
+
/** A dollar figure and where it came from. */
|
|
1670
|
+
type CostUsd = {
|
|
1671
|
+
/** US dollars. */
|
|
1672
|
+
usd: number;
|
|
1673
|
+
/**
|
|
1674
|
+
* `reported`: what the harness (Claude Code, pi) or the gateway (OpenRouter) said it cost;
|
|
1675
|
+
* `estimated`: the tokens at list prices, which leave out discounts and long-context rates.
|
|
1676
|
+
*/
|
|
1677
|
+
basis: 'reported' | 'estimated';
|
|
1678
|
+
/**
|
|
1679
|
+
* Who reported it (`claude-code`, `pi`, `openrouter`, `fake`), or the prices it was estimated
|
|
1680
|
+
* at (`models.dev 2026-11-05`, a `REEVAL_PRICES` file); several, joined with `, `, in a sum.
|
|
1681
|
+
*/
|
|
1682
|
+
source: string;
|
|
1683
|
+
};
|
|
1684
|
+
/**
|
|
1685
|
+
* Costs added up: `basis` is `reported` only when every part was. `unpriced` counts the parts
|
|
1686
|
+
* (attempts, or judge calls) that used tokens but have no cost, which `usd` leaves out.
|
|
1687
|
+
*/
|
|
1688
|
+
type CostTotal = CostUsd & {
|
|
1689
|
+
unpriced?: number;
|
|
1690
|
+
};
|
|
1490
1691
|
/** One agent turn. Immutable once `c.send()` resolves. */
|
|
1491
1692
|
/** A turn as data, as a record or another language's runner carries it: `Turn` without its methods. */
|
|
1492
1693
|
type TurnData = {
|
|
@@ -1517,7 +1718,9 @@ type TurnData = {
|
|
|
1517
1718
|
};
|
|
1518
1719
|
type Turn = TurnData & {
|
|
1519
1720
|
/** Runs in the eval's sandbox. Same as `c.exec`. */
|
|
1520
|
-
exec(command: string, options?: ExecRetry
|
|
1721
|
+
exec(command: string, options?: ExecRetry & {
|
|
1722
|
+
env?: ExecEnv;
|
|
1723
|
+
}): Promise<ExecResult>;
|
|
1521
1724
|
readFile(path: string): Promise<string | null>;
|
|
1522
1725
|
};
|
|
1523
1726
|
/** Any JSON value. */
|
|
@@ -1695,15 +1898,19 @@ type AttemptContext = {
|
|
|
1695
1898
|
judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1696
1899
|
/**
|
|
1697
1900
|
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1698
|
-
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
1901
|
+
* many extra times, `interval` ms apart; it resolves with the last try. `env` adds variables
|
|
1902
|
+
* for this command only.
|
|
1699
1903
|
*/
|
|
1700
1904
|
exec(command: string, options?: ExecRetry & {
|
|
1905
|
+
env?: ExecEnv;
|
|
1701
1906
|
background?: false;
|
|
1702
1907
|
}): Promise<ExecResult>;
|
|
1703
1908
|
exec(command: string, options: {
|
|
1704
1909
|
background: true;
|
|
1910
|
+
env?: ExecEnv;
|
|
1705
1911
|
}): Promise<BackgroundHandle>;
|
|
1706
1912
|
exec(command: string, options?: ExecRetry & {
|
|
1913
|
+
env?: ExecEnv;
|
|
1707
1914
|
background?: boolean;
|
|
1708
1915
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1709
1916
|
/**
|
|
@@ -1815,6 +2022,12 @@ type FakeHarnessDescriptor = {
|
|
|
1815
2022
|
failWith?: string;
|
|
1816
2023
|
/** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
|
|
1817
2024
|
startHangs?: number | true;
|
|
2025
|
+
/**
|
|
2026
|
+
* A shell command the fake agent runs as its session starts, once per sandbox, as a real
|
|
2027
|
+
* adapter's bootstrap installs its CLI: `tart()` bakes it into the image it clones, as it does a
|
|
2028
|
+
* real harness's, so an attempt there finds it done.
|
|
2029
|
+
*/
|
|
2030
|
+
bootstrap?: string;
|
|
1818
2031
|
/** A reasoning block at the start of every turn. */
|
|
1819
2032
|
reasoning?: string;
|
|
1820
2033
|
/**
|
|
@@ -1829,13 +2042,15 @@ type FakeHarnessDescriptor = {
|
|
|
1829
2042
|
effort?: string;
|
|
1830
2043
|
/**
|
|
1831
2044
|
* Tool calls the fake agent reports on every turn, before writing files. `ok` defaults to true;
|
|
1832
|
-
* null reports a call whose result never came.
|
|
2045
|
+
* null reports a call whose result never came. `durationMs` defaults to 0; null reports a call
|
|
2046
|
+
* whose end the harness never said.
|
|
1833
2047
|
*/
|
|
1834
2048
|
tools?: Array<{
|
|
1835
2049
|
tool: string;
|
|
1836
2050
|
input: unknown;
|
|
1837
2051
|
output: unknown;
|
|
1838
2052
|
ok?: boolean | null;
|
|
2053
|
+
durationMs?: number | null;
|
|
1839
2054
|
}>;
|
|
1840
2055
|
/** Files and message used instead when the environment gives the agent a skill. */
|
|
1841
2056
|
withSkill?: {
|
|
@@ -1844,6 +2059,8 @@ type FakeHarnessDescriptor = {
|
|
|
1844
2059
|
};
|
|
1845
2060
|
/** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
|
|
1846
2061
|
flakeRate?: number;
|
|
2062
|
+
/** US dollars the fake agent says every turn cost, as a harness that reports its cost does; 0 reports none. */
|
|
2063
|
+
costUsd?: number;
|
|
1847
2064
|
/** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
|
|
1848
2065
|
usage?: {
|
|
1849
2066
|
input: number;
|
|
@@ -1875,9 +2092,13 @@ type FakeHarnessDescriptor = {
|
|
|
1875
2092
|
layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
|
|
1876
2093
|
/**
|
|
1877
2094
|
* In Docker, broker the key of the agent `layout` names (`OPENAI_API_KEY` for `codex`, Cursor's
|
|
1878
|
-
* when unset) on that agent's endpoints, as its adapter does
|
|
2095
|
+
* when unset) on that agent's endpoints, as its adapter does; OpenCode, Goose and pi take their
|
|
2096
|
+
* model's provider's key, so with their layouts it's `FAKE_AGENT_API_KEY`, on no endpoint. The key stays on the host, and
|
|
1879
2097
|
* `exec` steps get a placeholder in its variable, which the egress gateway swaps in on those
|
|
1880
2098
|
* endpoints when sent as `Authorization: Bearer <placeholder>`, and refuses on any other request.
|
|
2099
|
+
* For an agent whose bridge is a server in the sandbox (Claude Code, OpenCode, Goose, fx, Grok
|
|
2100
|
+
* Build), it also serves on the session's first exposed port and reaches it from the host, as
|
|
2101
|
+
* that adapter's start does.
|
|
1881
2102
|
*/
|
|
1882
2103
|
brokers?: true;
|
|
1883
2104
|
};
|
|
@@ -1886,7 +2107,8 @@ type FakeHarnessDescriptor = {
|
|
|
1886
2107
|
* the command's stdout, `write` as file edits, and `reply` is the turn's final message. `mcp`
|
|
1887
2108
|
* (`'<server>/<tool>'`) starts the environment's stdio MCP server in the sandbox, as an agent
|
|
1888
2109
|
* would, and calls the tool with no arguments; it's reported as `mcp__<server>__<tool>`.
|
|
1889
|
-
* `error` ends the turn there with status `error` and that message, as a provider error would
|
|
2110
|
+
* `error` ends the turn there with status `error` and that message, as a provider error would; one
|
|
2111
|
+
* worded as a provider's credit, quota or rate limit makes the attempt an infra error.
|
|
1890
2112
|
* `refuseKey` ends it as the model API rejecting the key in that variable would, which makes the
|
|
1891
2113
|
* attempt an infra error.
|
|
1892
2114
|
*/
|
|
@@ -2021,8 +2243,9 @@ type OpenCodeHarnessDescriptor = {
|
|
|
2021
2243
|
startTimeout?: Duration;
|
|
2022
2244
|
};
|
|
2023
2245
|
/**
|
|
2024
|
-
* Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs
|
|
2025
|
-
*
|
|
2246
|
+
* Goose through `@reeval/harness-goose` (`goose acp`), made by `goose()`. It runs a model on its
|
|
2247
|
+
* provider's key, and reads each turn's usage from Goose's `usage_ledger` (its ACP server counts
|
|
2248
|
+
* one call).
|
|
2026
2249
|
*/
|
|
2027
2250
|
type GooseHarnessDescriptor = {
|
|
2028
2251
|
kind: 'goose';
|
|
@@ -2031,7 +2254,10 @@ type GooseHarnessDescriptor = {
|
|
|
2031
2254
|
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2032
2255
|
*/
|
|
2033
2256
|
gateway?: GatewayName;
|
|
2034
|
-
/**
|
|
2257
|
+
/**
|
|
2258
|
+
* `<provider>/<model>`: Goose's `google`, `anthropic`, `openai` or `xai` provider on its own key,
|
|
2259
|
+
* or the gateway's id on `vercel()` and `openrouter()`, e.g. `google/gemini-3.8-flash`.
|
|
2260
|
+
*/
|
|
2035
2261
|
model: string;
|
|
2036
2262
|
/**
|
|
2037
2263
|
* Reasoning effort as `GOOSE_THINKING_EFFORT`, e.g. `off`, `low`, `medium`, `high` or `max`.
|
|
@@ -2048,8 +2274,8 @@ type GooseHarnessDescriptor = {
|
|
|
2048
2274
|
startTimeout?: Duration;
|
|
2049
2275
|
};
|
|
2050
2276
|
/**
|
|
2051
|
-
* pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs
|
|
2052
|
-
*
|
|
2277
|
+
* pi through `@reeval/harness-pi` (`pi --mode json` in the sandbox), made by `pi()`. It runs a
|
|
2278
|
+
* model on its provider's key and reports each model call's tokens. It takes stdio and
|
|
2053
2279
|
* HTTP MCP servers, not SSE ones.
|
|
2054
2280
|
*/
|
|
2055
2281
|
type PiHarnessDescriptor = {
|
|
@@ -2059,7 +2285,10 @@ type PiHarnessDescriptor = {
|
|
|
2059
2285
|
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
2060
2286
|
*/
|
|
2061
2287
|
gateway?: GatewayName;
|
|
2062
|
-
/**
|
|
2288
|
+
/**
|
|
2289
|
+
* `<provider>/<model>`: one of pi's providers whose key it sends in a header on a fixed host, or
|
|
2290
|
+
* the gateway's id on `vercel()` and `openrouter()`, e.g. `google/gemini-3.8-flash`.
|
|
2291
|
+
*/
|
|
2063
2292
|
model: string;
|
|
2064
2293
|
/**
|
|
2065
2294
|
* Reasoning effort, passed as pi's `--thinking` (the CLI's name). Values are whatever that
|
|
@@ -2309,6 +2538,27 @@ type TartSandboxDescriptor = {
|
|
|
2309
2538
|
user?: string;
|
|
2310
2539
|
key: string;
|
|
2311
2540
|
};
|
|
2541
|
+
/**
|
|
2542
|
+
* Macs that run the VMs instead of this machine, each one plain `ssh` reaches without a prompt
|
|
2543
|
+
* (an `~/.ssh/config` alias, the agent or a key file; no passwords): `'mac-studio'`,
|
|
2544
|
+
* `'elliot@mac-mini.local'`, `'ssh://ci@10.0.0.5:2222'` or `{ host, user?, port?, identityFile?, vms? }`.
|
|
2545
|
+
* reeval runs `tart` on each over SSH and keeps the egress gateway, and every key, here. Each
|
|
2546
|
+
* host needs the image; preflight bakes the harness onto it there. This machine when unset.
|
|
2547
|
+
*/
|
|
2548
|
+
hosts?: TartHost[];
|
|
2549
|
+
};
|
|
2550
|
+
/** A Mac that runs `tart()`'s VMs over SSH: an SSH destination, or its parts. */
|
|
2551
|
+
type TartHost = string | {
|
|
2552
|
+
/** The host name, address or `~/.ssh/config` alias. */
|
|
2553
|
+
host: string;
|
|
2554
|
+
/** The login user; `ssh`'s own (the config's, or this machine's user) when unset. */
|
|
2555
|
+
user?: string;
|
|
2556
|
+
/** The SSH port; `ssh`'s own (the config's, or 22) when unset. */
|
|
2557
|
+
port?: number;
|
|
2558
|
+
/** The private key `ssh -i` takes: absolute, or `~/…`; the agent's or the config's when unset. */
|
|
2559
|
+
identityFile?: string;
|
|
2560
|
+
/** The most of reeval's macOS VMs it runs at once: 1 or 2, and 2 when unset. */
|
|
2561
|
+
vms?: number;
|
|
2312
2562
|
};
|
|
2313
2563
|
/** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
|
|
2314
2564
|
type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
|
|
@@ -2331,8 +2581,21 @@ type BoatSandboxDescriptor = {
|
|
|
2331
2581
|
prefix: string;
|
|
2332
2582
|
/** The machine size. Default `small`. */
|
|
2333
2583
|
type?: BoatMachineType;
|
|
2334
|
-
/** The most Boat
|
|
2584
|
+
/** The most attempts on Boat at once, across the run: machines times `attemptsPerMachine`. */
|
|
2335
2585
|
maxConcurrent?: number;
|
|
2586
|
+
/**
|
|
2587
|
+
* A machine takes another of the run's attempts once one ends, instead of being deleted: its
|
|
2588
|
+
* user's processes, cron jobs and files are removed and its home put back as it was after the
|
|
2589
|
+
* harness installed, which isn't installed again. Default false, one attempt per slot.
|
|
2590
|
+
*/
|
|
2591
|
+
reuse?: boolean;
|
|
2592
|
+
/**
|
|
2593
|
+
* The attempts one machine runs at once, each as a user of its own with its own home (which
|
|
2594
|
+
* each sees at `/home/reeval`), workspace and egress gateway. The harness installs once per
|
|
2595
|
+
* machine, and every other slot's home gets its own copy of the install. Default 1. A harness whose bridge takes a fixed port in the
|
|
2596
|
+
* machine (Claude Code, OpenCode, Goose, fx, Grok Build) runs one per machine.
|
|
2597
|
+
*/
|
|
2598
|
+
attemptsPerMachine?: number;
|
|
2336
2599
|
};
|
|
2337
2600
|
type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor | TartSandboxDescriptor | BoatSandboxDescriptor;
|
|
2338
2601
|
/**
|
|
@@ -2492,6 +2755,17 @@ type SetupStep = string | {
|
|
|
2492
2755
|
type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
|
|
2493
2756
|
allow: readonly string[];
|
|
2494
2757
|
};
|
|
2758
|
+
/**
|
|
2759
|
+
* An egress policy per phase of the attempt: `setup` covers its files and setup steps, `agent`
|
|
2760
|
+
* the harness's start, every turn and every check after them. A phase it leaves unset keeps the
|
|
2761
|
+
* policy it inherits.
|
|
2762
|
+
*/
|
|
2763
|
+
type PhasedEgress = {
|
|
2764
|
+
setup?: EgressPolicy;
|
|
2765
|
+
agent?: EgressPolicy;
|
|
2766
|
+
};
|
|
2767
|
+
/** One policy for the whole attempt, or one per phase. */
|
|
2768
|
+
type Egress = EgressPolicy | PhasedEgress;
|
|
2495
2769
|
/**
|
|
2496
2770
|
* A git repository (or one directory of it) the workspace starts as, from the `git` helper. `url`
|
|
2497
2771
|
* is what reeval fetches: the clone URL, or an absolute path for a local repository.
|
|
@@ -2528,8 +2802,11 @@ type EnvironmentSpec = {
|
|
|
2528
2802
|
workspace?: GitWorkspace;
|
|
2529
2803
|
/** Variables and secrets for the sandbox, merged by name over the parent's. */
|
|
2530
2804
|
env?: EnvVars;
|
|
2531
|
-
/**
|
|
2532
|
-
|
|
2805
|
+
/**
|
|
2806
|
+
* Replaces the inherited policy, per phase: a plain policy sets both, `{ setup, agent }` the
|
|
2807
|
+
* phases it names. An `extend` child's replaces its parent's the same way.
|
|
2808
|
+
*/
|
|
2809
|
+
egress?: Egress;
|
|
2533
2810
|
/** Workspace-relative, or `~/` for home. */
|
|
2534
2811
|
files?: Record<string, string | HostFile>;
|
|
2535
2812
|
/** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
|
|
@@ -2998,10 +3275,10 @@ type EvalOptionFields<N extends number = number> = {
|
|
|
2998
3275
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
2999
3276
|
env?: EnvVars;
|
|
3000
3277
|
/**
|
|
3001
|
-
* Where the sandbox may connect. Replaces the inherited policy: config, profile
|
|
3002
|
-
* environment, then groups and the eval.
|
|
3278
|
+
* Where the sandbox may connect. Replaces the inherited policy, per phase: config, profile
|
|
3279
|
+
* defaults, the environment, then groups and the eval.
|
|
3003
3280
|
*/
|
|
3004
|
-
egress?:
|
|
3281
|
+
egress?: Egress;
|
|
3005
3282
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3006
3283
|
judge?: JudgeOptions;
|
|
3007
3284
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
@@ -3089,11 +3366,78 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
|
3089
3366
|
/** `'provider/model'`, or a fake judge. */
|
|
3090
3367
|
model?: string | FakeJudgeModel;
|
|
3091
3368
|
};
|
|
3092
|
-
|
|
3093
|
-
|
|
3369
|
+
/** A history folder committed with the evals: `local()` from `reeval/store/local`. */
|
|
3370
|
+
type LocalStoreDescriptor = {
|
|
3371
|
+
kind: 'local';
|
|
3372
|
+
/**
|
|
3373
|
+
* The folder, relative to the config's directory (default: `.history` in the directory the
|
|
3374
|
+
* evals live in).
|
|
3375
|
+
*/
|
|
3094
3376
|
dir?: string;
|
|
3095
|
-
|
|
3377
|
+
};
|
|
3378
|
+
/**
|
|
3379
|
+
* The host variables an `s3()` store reads. Each names a variable, never a value: credentials
|
|
3380
|
+
* come from the environment only.
|
|
3381
|
+
*/
|
|
3382
|
+
type S3StoreEnv = {
|
|
3383
|
+
/** The access key id (default `AWS_ACCESS_KEY_ID`). */
|
|
3384
|
+
accessKeyId?: string;
|
|
3385
|
+
/** The secret access key (default `AWS_SECRET_ACCESS_KEY`). */
|
|
3386
|
+
secretAccessKey?: string;
|
|
3387
|
+
/** A session token for temporary credentials (default `AWS_SESSION_TOKEN`), when it's set. */
|
|
3388
|
+
sessionToken?: string;
|
|
3389
|
+
/** The endpoint, when `endpoint` isn't set (default `AWS_ENDPOINT_URL_S3`, then `AWS_ENDPOINT_URL`). */
|
|
3390
|
+
endpoint?: string;
|
|
3391
|
+
/** The bucket, when `bucket` isn't set. */
|
|
3392
|
+
bucket?: string;
|
|
3393
|
+
/** The region, when `region` isn't set (default `AWS_REGION`, then `AWS_DEFAULT_REGION`). */
|
|
3394
|
+
region?: string;
|
|
3395
|
+
};
|
|
3396
|
+
/**
|
|
3397
|
+
* An S3-compatible bucket, such as AWS S3, Cloudflare R2 or MinIO: `s3()` from `reeval/store/s3`.
|
|
3398
|
+
* reeval keeps a copy of it in `.reeval/store/` that every history command reads.
|
|
3399
|
+
*/
|
|
3400
|
+
type S3StoreDescriptor = {
|
|
3401
|
+
kind: 's3';
|
|
3402
|
+
/** The bucket; `env.bucket` names a variable with it instead. */
|
|
3403
|
+
bucket?: string;
|
|
3404
|
+
/**
|
|
3405
|
+
* The endpoint's origin, such as `https://<account>.r2.cloudflarestorage.com` or
|
|
3406
|
+
* `http://127.0.0.1:9000`; without it, `env.endpoint`'s variable, else AWS S3 in the region.
|
|
3407
|
+
*/
|
|
3408
|
+
endpoint?: string;
|
|
3409
|
+
/** The signing region (default: `env.region`'s variable, else `auto`). */
|
|
3410
|
+
region?: string;
|
|
3411
|
+
/** A key prefix every object goes under, so several projects can share a bucket (default none). */
|
|
3412
|
+
prefix?: string;
|
|
3413
|
+
/** Which host variables hold the credentials, and the bucket, endpoint or region when they aren't set here. */
|
|
3414
|
+
env?: S3StoreEnv;
|
|
3415
|
+
};
|
|
3416
|
+
/** A history store as data, which `reeval.yaml` can name too. */
|
|
3417
|
+
type HistoryStoreDescriptor = LocalStoreDescriptor | S3StoreDescriptor;
|
|
3418
|
+
type HistoryConfig = {
|
|
3419
|
+
/**
|
|
3420
|
+
* Where recorded runs are kept: `local()` (the default, a folder committed with the evals) or
|
|
3421
|
+
* `s3()`.
|
|
3422
|
+
*/
|
|
3423
|
+
store?: HistoryStoreDescriptor;
|
|
3424
|
+
/**
|
|
3425
|
+
* Artifacts recorded next to each run: every attempt's, failed attempts' only (the default), or
|
|
3426
|
+
* none. The setup log and each `c.exec` log are an attempt's artifacts too.
|
|
3427
|
+
*/
|
|
3096
3428
|
artifacts?: ArtifactLevel;
|
|
3429
|
+
/**
|
|
3430
|
+
* Each attempt's debug data uploaded with the run to a remote store: none (the default), its
|
|
3431
|
+
* gateway requests (`requests.ndjson`, which holds prompts and responses), or all of it
|
|
3432
|
+
* (`debug.log` and `trace.ndjson` too). Rescanned for secrets first; `local()` takes `'none'` only.
|
|
3433
|
+
*/
|
|
3434
|
+
debug?: DebugUpload;
|
|
3435
|
+
};
|
|
3436
|
+
type DebugUpload = 'none' | 'requests' | 'all';
|
|
3437
|
+
/** `HistoryConfig` as data: the store is `local` or `s3`. */
|
|
3438
|
+
type HistoryConfigData = Omit<HistoryConfig, 'store'> & {
|
|
3439
|
+
/** Where recorded runs are kept: `{ kind: local }` (the default) or `{ kind: s3, … }`. */
|
|
3440
|
+
store?: HistoryStoreDescriptor;
|
|
3097
3441
|
};
|
|
3098
3442
|
/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
|
|
3099
3443
|
type Budget = {
|
|
@@ -3116,8 +3460,8 @@ type ProfileDefaults = {
|
|
|
3116
3460
|
gate?: GatePolicy;
|
|
3117
3461
|
/** Variables and secrets for the sandbox, merged by name over the config's. */
|
|
3118
3462
|
env?: EnvVars;
|
|
3119
|
-
/** Where the sandbox may connect; replaces the config's policy. */
|
|
3120
|
-
egress?:
|
|
3463
|
+
/** Where the sandbox may connect; replaces the config's policy, per phase. */
|
|
3464
|
+
egress?: Egress;
|
|
3121
3465
|
/** Defaults for `c.judge()`, merged field by field over the config's. */
|
|
3122
3466
|
judge?: ConfigJudgeOptions;
|
|
3123
3467
|
/** Per attempt. */
|
|
@@ -3158,6 +3502,11 @@ type Profile = {
|
|
|
3158
3502
|
budget?: Budget;
|
|
3159
3503
|
/** When this profile's runs are copied into the history folder. */
|
|
3160
3504
|
record?: RecordMode;
|
|
3505
|
+
/**
|
|
3506
|
+
* The history this profile records to and every history command reads with `--profile`, merged
|
|
3507
|
+
* over the config's `history` key by key.
|
|
3508
|
+
*/
|
|
3509
|
+
history?: HistoryConfig;
|
|
3161
3510
|
/** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
|
|
3162
3511
|
tags?: string[];
|
|
3163
3512
|
/** Replaces the config's `gateway` list for this profile's runs. */
|
|
@@ -3292,8 +3641,8 @@ type ReevalConfig = {
|
|
|
3292
3641
|
gate?: GatePolicy;
|
|
3293
3642
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
3294
3643
|
env?: EnvVars;
|
|
3295
|
-
/** Where the sandbox may connect; default `'proxy'
|
|
3296
|
-
egress?:
|
|
3644
|
+
/** Where the sandbox may connect; default `'proxy'` in both phases. */
|
|
3645
|
+
egress?: Egress;
|
|
3297
3646
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3298
3647
|
judge?: ConfigJudgeOptions;
|
|
3299
3648
|
/** Per attempt; default `'15m'`. */
|
|
@@ -3314,7 +3663,7 @@ type ReevalConfig = {
|
|
|
3314
3663
|
concurrency?: number;
|
|
3315
3664
|
/** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
|
|
3316
3665
|
record?: RecordMode;
|
|
3317
|
-
/**
|
|
3666
|
+
/** Where recorded runs are kept, and which attempts keep their artifacts there. */
|
|
3318
3667
|
history?: HistoryConfig;
|
|
3319
3668
|
/** Caps on a run; a profile's `budget` is merged over it. */
|
|
3320
3669
|
budget?: Budget;
|
|
@@ -3325,7 +3674,7 @@ type ReevalConfig = {
|
|
|
3325
3674
|
ci?: string;
|
|
3326
3675
|
local?: string;
|
|
3327
3676
|
};
|
|
3328
|
-
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
|
|
3677
|
+
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers and `env` values written as plain strings, instead of warning (default `'warn'`). */
|
|
3329
3678
|
secrets?: {
|
|
3330
3679
|
unbrokered?: 'warn' | 'error';
|
|
3331
3680
|
};
|
|
@@ -3382,15 +3731,26 @@ type ResourceFailureRecord = {
|
|
|
3382
3731
|
cause: 'oom';
|
|
3383
3732
|
kills: number;
|
|
3384
3733
|
};
|
|
3734
|
+
/**
|
|
3735
|
+
* The model provider ended a turn, not the agent: out of credit, over its quota, rate limited or
|
|
3736
|
+
* overloaded, with the HTTP status of the last model call the egress gateway brokered when that
|
|
3737
|
+
* call failed (`null` when it didn't, as in a stream that failed inside an HTTP 200, or none was).
|
|
3738
|
+
* The attempt is then an infra error, so `--retry-errors` runs it again.
|
|
3739
|
+
*/
|
|
3740
|
+
type ProviderStopRecord = {
|
|
3741
|
+
cause: 'credit' | 'quota' | 'rate_limit' | 'overloaded';
|
|
3742
|
+
status: number | null;
|
|
3743
|
+
};
|
|
3385
3744
|
/**
|
|
3386
3745
|
* Why an attempt failed; `prepare` when its sandbox couldn't be prepared, `resource` when it ran
|
|
3387
|
-
* out of one.
|
|
3746
|
+
* out of one, `provider` when the model provider stopped a turn.
|
|
3388
3747
|
*/
|
|
3389
3748
|
type AttemptError = {
|
|
3390
3749
|
kind: ErrorKind;
|
|
3391
3750
|
message: string;
|
|
3392
3751
|
prepare?: PrepareFailureRecord;
|
|
3393
3752
|
resource?: ResourceFailureRecord;
|
|
3753
|
+
provider?: ProviderStopRecord;
|
|
3394
3754
|
};
|
|
3395
3755
|
/**
|
|
3396
3756
|
* One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
|
|
@@ -3527,8 +3887,16 @@ type EnvironmentRecord = {
|
|
|
3527
3887
|
brokered: string[];
|
|
3528
3888
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3529
3889
|
unset: string[];
|
|
3530
|
-
/**
|
|
3890
|
+
/**
|
|
3891
|
+
* The policy the attempt's agent phase ran under (the harness's start, its turns and the checks),
|
|
3892
|
+
* from the options and the environment; in the digest unless `proxy`.
|
|
3893
|
+
*/
|
|
3531
3894
|
egress: EgressPolicy;
|
|
3895
|
+
/**
|
|
3896
|
+
* The policy the setup phase (files and setup steps) ran under, when it wasn't the agent's; in
|
|
3897
|
+
* the digest when set.
|
|
3898
|
+
*/
|
|
3899
|
+
setupEgress?: EgressPolicy;
|
|
3532
3900
|
/**
|
|
3533
3901
|
* This records the Git repository the workspace started as: the repo as written (a local one
|
|
3534
3902
|
* relative to the project root), the ref and path asked for (null when unset), the commit the
|
|
@@ -3553,11 +3921,21 @@ type EnvironmentRecord = {
|
|
|
3553
3921
|
* contains only unexpected refusals. A request that carries the harness's brokered key off the
|
|
3554
3922
|
* endpoints reeval brokers it for is refused on a host the policy let through: it's in `blocked`
|
|
3555
3923
|
* once per method and path, its `reason` names them, and `connections` counts its requests. Past
|
|
3556
|
-
* the first 20 pairs, the rest share one entry per host.
|
|
3924
|
+
* the first 20 pairs, the rest share one entry per host. With a setup policy of its own, every
|
|
3925
|
+
* entry names the `phase` it happened in.
|
|
3557
3926
|
*/
|
|
3558
3927
|
type EgressRecord = {
|
|
3928
|
+
/** The agent phase's policy: the harness's start, its turns and the checks. */
|
|
3559
3929
|
policy: EgressPolicy;
|
|
3560
3930
|
allowed: string[] | null;
|
|
3931
|
+
/**
|
|
3932
|
+
* The setup phase's policy (files and setup steps) and the hosts it allowed, when it wasn't the
|
|
3933
|
+
* agent's.
|
|
3934
|
+
*/
|
|
3935
|
+
setup?: {
|
|
3936
|
+
policy: EgressPolicy;
|
|
3937
|
+
allowed: string[] | null;
|
|
3938
|
+
};
|
|
3561
3939
|
recorded: boolean;
|
|
3562
3940
|
/**
|
|
3563
3941
|
* True when traffic could leave around the gateway (`allow-all` in a container that kept its
|
|
@@ -3571,23 +3949,29 @@ type EgressRecord = {
|
|
|
3571
3949
|
connections: number;
|
|
3572
3950
|
sent: number;
|
|
3573
3951
|
received: number;
|
|
3952
|
+
/** With a setup policy of its own: the phase the connections were made in. */
|
|
3953
|
+
phase?: 'setup' | 'agent';
|
|
3574
3954
|
}>;
|
|
3575
3955
|
/**
|
|
3576
3956
|
* Hosts the policy refused, and requests that carry the harness's brokered key off the endpoints
|
|
3577
3957
|
* reeval brokers it for (one entry per method and path), each with why. A name that didn't
|
|
3578
|
-
* resolve is in `unresolved` instead.
|
|
3958
|
+
* resolve is in `unresolved` instead. A tunnel setup opened to a host the agent's policy refuses
|
|
3959
|
+
* is closed as the agent phase starts, and is here with `closed: true`.
|
|
3579
3960
|
*/
|
|
3580
3961
|
blocked: Array<{
|
|
3581
3962
|
host: string;
|
|
3582
3963
|
port: number;
|
|
3583
3964
|
connections: number;
|
|
3584
3965
|
reason: string;
|
|
3966
|
+
phase?: 'setup' | 'agent';
|
|
3967
|
+
closed?: true;
|
|
3585
3968
|
}>;
|
|
3586
3969
|
expectedBlocked: Array<{
|
|
3587
3970
|
host: string;
|
|
3588
3971
|
port: number;
|
|
3589
3972
|
connections: number;
|
|
3590
3973
|
reason: string;
|
|
3974
|
+
phase?: 'setup' | 'agent';
|
|
3591
3975
|
}>;
|
|
3592
3976
|
/**
|
|
3593
3977
|
* Hosts the policy let through whose names didn't resolve (`metadata.google.internal` outside
|
|
@@ -3600,6 +3984,7 @@ type EgressRecord = {
|
|
|
3600
3984
|
port: number;
|
|
3601
3985
|
connections: number;
|
|
3602
3986
|
reason: string;
|
|
3987
|
+
phase?: 'setup' | 'agent';
|
|
3603
3988
|
}>;
|
|
3604
3989
|
};
|
|
3605
3990
|
/** The image an attempt's container started from, and how reeval made it. */
|
|
@@ -3633,11 +4018,98 @@ type VmImage = {
|
|
|
3633
4018
|
/** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
|
|
3634
4019
|
digest: string | null;
|
|
3635
4020
|
};
|
|
3636
|
-
/**
|
|
4021
|
+
/**
|
|
4022
|
+
* What an attempt's sandbox gave it and what it used, measured from when the sandbox started
|
|
4023
|
+
* until teardown. A value the sandbox can't measure is left out, never estimated.
|
|
4024
|
+
*/
|
|
4025
|
+
type AttemptResources = {
|
|
4026
|
+
/**
|
|
4027
|
+
* `container`: the container's own cgroup, so only the attempt's processes count (Docker,
|
|
4028
|
+
* Podman). `machine`: the whole VM, its system processes included (`boat()`, `tart()`).
|
|
4029
|
+
*/
|
|
4030
|
+
scope: 'container' | 'machine';
|
|
4031
|
+
/**
|
|
4032
|
+
* The VM ran other attempts at the same time (`boat({ attemptsPerMachine })` above 1), so its
|
|
4033
|
+
* whole-VM usage isn't this attempt's: `usage` is left out.
|
|
4034
|
+
*/
|
|
4035
|
+
shared?: true;
|
|
4036
|
+
/** What the attempt ran on. */
|
|
4037
|
+
machine: {
|
|
4038
|
+
/** The sandbox's provider: `docker`, `podman`, `boat` or `tart`. */
|
|
4039
|
+
sandbox: string;
|
|
4040
|
+
/** The provider's machine size, where it has sizes: `boat({ type })`. */
|
|
4041
|
+
size?: string;
|
|
4042
|
+
/**
|
|
4043
|
+
* CPUs the attempt could use: the container's CPU quota, else the CPUs the VM has (a macOS
|
|
4044
|
+
* VM's `sysctl hw.ncpu`).
|
|
4045
|
+
*/
|
|
4046
|
+
vcpus?: number;
|
|
4047
|
+
/**
|
|
4048
|
+
* Memory the attempt could use, in bytes: the container's limit, else the VM's total (a
|
|
4049
|
+
* macOS VM's `sysctl hw.memsize`).
|
|
4050
|
+
*/
|
|
4051
|
+
memoryBytes?: number;
|
|
4052
|
+
};
|
|
4053
|
+
/** What it used; absent when the sandbox gave no readings (a macOS VM has no `/proc`). */
|
|
4054
|
+
usage?: {
|
|
4055
|
+
/** Milliseconds between the first reading and the last. */
|
|
4056
|
+
durationMs: number;
|
|
4057
|
+
/** Milliseconds between readings while the attempt ran. */
|
|
4058
|
+
sampleMs: number;
|
|
4059
|
+
/** CPU time over `durationMs`, in seconds: user and system, every CPU added up. */
|
|
4060
|
+
cpuSeconds: number;
|
|
4061
|
+
/** `cpuSeconds` over `durationMs`: how many CPUs it kept busy on average. */
|
|
4062
|
+
avgCores: number;
|
|
4063
|
+
/** The most CPUs it kept busy between two readings; absent with fewer than two. */
|
|
4064
|
+
peakCores?: number;
|
|
4065
|
+
/**
|
|
4066
|
+
* The 95th percentile of the CPUs it kept busy between two readings, a second apart: what it
|
|
4067
|
+
* needed but for its busiest twentieth of the time. Absent with fewer than two readings.
|
|
4068
|
+
*/
|
|
4069
|
+
p95Cores?: number;
|
|
4070
|
+
/**
|
|
4071
|
+
* The highest memory reading, in bytes, page cache the kernel can reclaim left out: the
|
|
4072
|
+
* cgroup's `memory.current` less its `inactive_file`, or the VM's `MemTotal` less
|
|
4073
|
+
* `MemAvailable`. Sampled, so a spike between readings is missed.
|
|
4074
|
+
*/
|
|
4075
|
+
peakMemoryBytes?: number;
|
|
4076
|
+
/**
|
|
4077
|
+
* cgroup v2's `memory.peak`, in bytes: the highest `memory.current` the kernel saw since the
|
|
4078
|
+
* container started, page cache included, so no spike is missed but it reads higher than
|
|
4079
|
+
* `peakMemoryBytes`. `container` scope on Linux 5.19 or later only.
|
|
4080
|
+
*/
|
|
4081
|
+
cgroupPeakMemoryBytes?: number;
|
|
4082
|
+
/**
|
|
4083
|
+
* How often the container's CPU quota held it back over `durationMs`, from cgroup v2's
|
|
4084
|
+
* `cpu.stat`: `periods` is `nr_periods` (quota periods it ran in), `throttledPeriods`
|
|
4085
|
+
* `nr_throttled` (those it used up its quota in and waited out) and `throttledSeconds`
|
|
4086
|
+
* `throttled_usec`. `container` scope only.
|
|
4087
|
+
*/
|
|
4088
|
+
throttling?: {
|
|
4089
|
+
periods: number;
|
|
4090
|
+
throttledPeriods: number;
|
|
4091
|
+
throttledSeconds: number;
|
|
4092
|
+
};
|
|
4093
|
+
/** Bytes the VM's disks read over `durationMs`; `machine` scope only. */
|
|
4094
|
+
diskReadBytes?: number;
|
|
4095
|
+
/** Bytes the VM's disks wrote over `durationMs`; `machine` scope only. */
|
|
4096
|
+
diskWriteBytes?: number;
|
|
4097
|
+
/**
|
|
4098
|
+
* The sandbox went before the last reading (a timeout or cancel): the values stop at the last
|
|
4099
|
+
* sample.
|
|
4100
|
+
*/
|
|
4101
|
+
partial?: true;
|
|
4102
|
+
};
|
|
4103
|
+
};
|
|
4104
|
+
/** A harness session start that timed out, or failed on a transient error, and was started again. */
|
|
3637
4105
|
type StartRetry = {
|
|
3638
|
-
/**
|
|
4106
|
+
/**
|
|
4107
|
+
* `<harness kind>: session did not become ready within <startTimeout>`, or the transient error
|
|
4108
|
+
* the start failed on, such as `Cursor harness failed to start: Cursor's model catalog request
|
|
4109
|
+
* failed: HTTP 502 (tried 3 times)`.
|
|
4110
|
+
*/
|
|
3639
4111
|
error: string;
|
|
3640
|
-
/** How long it waited before giving up on that start. */
|
|
4112
|
+
/** How long it waited before giving up on that start, or until that start failed. */
|
|
3641
4113
|
waitedMs: number;
|
|
3642
4114
|
};
|
|
3643
4115
|
/**
|
|
@@ -3800,5 +4272,5 @@ type ResolvedGateway = {
|
|
|
3800
4272
|
why: string;
|
|
3801
4273
|
}>;
|
|
3802
4274
|
};
|
|
3803
|
-
export {
|
|
3804
|
-
//# sourceMappingURL=types-
|
|
4275
|
+
export { EnvironmentInput as $, EffortSample as $n, Question as $t, CellRole as A, CheckTally as An, JudgeArtifactRef as At, CostUsd as B, JudgmentRecord as Bn, Observation as Bt, CaptureUseAction as C, VertexPublisher as Cn, HarnessEdit as Ct, CaseFn as D, ArtifactRef as Dn, HistoryStoreDescriptor as Dt, CaptureWindow as E, WebEventType as En, HistoryConfigData as Et, CompareCut as F, FamilyRecord as Fn, JudgeOptions as Ft, DirectoryOptions as G, StackSummary as Gn, PairVote as Gt, DebugUpload as H, RunDebug as Hn, OpenRouterGatewayDescriptor as Ht, CompareOptions as I, FamilyVerdict as In, JudgeState as It, EgressPolicy as J, DIRECTIONAL_MIN as Jn, PluginPart as Jt, DockerSandboxDescriptor as K, UpstreamCount as Kn, PiHarnessDescriptor as Kt, ConfigHelpers as L, FileRef as Ln, JudgeSubject as Lt, ClaudeCodeHarnessDescriptor as M, FamilyLift as Mn, JudgeFailure as Mt, CodexHarnessDescriptor as N, FamilyLiftSummary as Nn, JudgeInputs as Nt, CaseOptions as O, AttemptRecord as On, HostFile as Ot, CompareAnswer as P, FamilyRate as Pn, JudgeModel as Pt, EnvironmentHelpers as Q, EffortMetric as Qn, ProfileLimit as Qt, ConfigJudgeOptions as R, HistoryCell as Rn, LocalStoreDescriptor as Rt, CaptureUse as S, VertexGatewayDescriptor as Sn, tokenValues as Sr, HarnessDescriptor as St, CaptureWait as T, Wait as Tn, HistoryConfig as Tt, Diff as U, RunInvocation as Un, Outcome as Ut, CursorHarnessDescriptor as V, Lift as Vn, OpenCodeHarnessDescriptor as Vt, DirectGatewayDescriptor as W, RunRecord as Wn, PairFactor as Wt, EnvVars as X, EFFORT_MIN_RUNS as Xn, Profile as Xt, EgressRecord as Y, EFFORT_ALPHA as Yn, PluginSource as Yt, Environment as Z, EffortContrast as Zn, ProfileDefaults as Zt, CaptureInstruction as _, Turn as _n, formatCount as _r, GatewayName as _t, AttemptContext as a, S3StoreEnv as an, cellContrast as ar, ExecEnv as at, CapturePage as b, VercelGatewayDescriptor as bn, splitLabel as br, GroupOptions as bt, BlindPair as c, Secret as cn, contrastEffort as cr, FakeAnalysisModel as ct, BoatSandboxDescriptor as d, SetupStep as dn, effortLabel as dr, FakeJudgeModel as dt, QuestionInputs as en, EffortVerdict as er, EnvironmentSpec as et, Budget as f, SkillSource as fn, effortOf as fr, FakeSandboxDescriptor as ft, CaptureFailedRequest as g, TranscriptEvent as gn, filesChanged as gr, GatewayDescriptor as gt, CaptureConsoleMessage as h, TokenUsage as hn, effortVerdict as hr, FxHarnessDescriptor as ht, AttemptAnswer as i, S3StoreDescriptor as in, bothUnderATenth as ir, EvalOptionsData as it, ClaudeCodeEffort as j, EvalEntry as jn, JudgeCallOptions as jt, Cell as k, CellRecord as kn, JSONValue as kt, BoatApiKey as l, SecretTemplate as ln, directional as lr, FakeAnswer as lt, CaptureCall as m, TartSandboxDescriptor as mn, effortSummary as mr, FakeVote as mt, AnalysisConfig as n, ReevalConfig as nn, TOKENS_UNREPORTED as nr, EvalFn as nt, BackgroundHandle as o, SandboxDescriptor as on, cellEffort as or, ExecResult as ot, Capture as p, StatusRange as pn, effortSample as pr, FakeScriptStep as pt, Duration as q, CellEffort as qn, PinnableKey as qt, AntigravityHarnessDescriptor as r, ResolvedGateway as rn, VERDICT_TEXT as rr, EvalOptions as rt, BlindAttempt as s, SandboxWarning as sn, cellSamples as sr, ExecRetry as st, AgentHarnessDescriptor as t, QuestionOptions as tn, Spread as tr, ErrorKind as tt, BoatMachineType as u, ServeHandle as un, effortChangeLabel as ur, FakeHarnessDescriptor as ut, CaptureKey as v, TurnData as vn, formatRatio as vr, GooseHarnessDescriptor as vt, CaptureUseScript as w, Viewport as wn, HarnessOption as wt, CapturePageFile as x, Verdict as xn, splitOf as xr, HarnessData as xt, CaptureOptions as y, TurnStatus as yn, formatSeconds as yr, GrokBuildHarnessDescriptor as yt, CostTotal as z, HistoryEntry as zn, McpServer as zt };
|
|
4276
|
+
//# sourceMappingURL=types-DKK8jj--.d.ts.map
|