reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
|
@@ -1,226 +1,22 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-ttMTnIYX.js";
|
|
2
|
-
import {
|
|
3
|
-
import { a as totalTokens } from "./usage-DJiDnh2D.js";
|
|
2
|
+
import { d as ownRequire, t as BUILD_ID } from "./root-BzeoyfGd.js";
|
|
4
3
|
import { t as RUN_ID } from "./paths-Dwmi7L4X.js";
|
|
5
|
-
import { c as
|
|
6
|
-
import {
|
|
4
|
+
import { c as relativeFile, l as sha256 } from "./ids-B-JKTCP8.js";
|
|
5
|
+
import { a as SKILL_DIRS } from "./agents-KBTYhSSh.js";
|
|
6
|
+
import { c as targetText, i as skillUses, l as toolTarget, n as classifyCall, o as skillsWritten, r as skillDirSkill, t as TOOL_CLASSES } from "./tool-class-krsmEo44.js";
|
|
7
|
+
import { o as totalTokens } from "./usage-CwIRPPZ9.js";
|
|
7
8
|
import { a as pairTallies } from "./pairs-BGb134Vu.js";
|
|
8
9
|
import { a as liftInterval, i as familySmallSample, l as median, m as round3, r as containsZero, v as smallSample, y as wilson } from "./stats-C18W4xPX.js";
|
|
9
10
|
import { n as gateLabel } from "./gate-CgYdaprG.js";
|
|
10
|
-
import { n as isRuntimeState } from "./context-
|
|
11
|
-
import {
|
|
12
|
-
import { f as cellMeasures, p as familyMeasures } from "./suite-
|
|
13
|
-
import {
|
|
11
|
+
import { n as isRuntimeState } from "./context-DrEAtUyg.js";
|
|
12
|
+
import { I as filesChanged, M as effortOf, O as contrastEffort, _ as contrastPairs, g as checkVerdict, r as caseName, t as caseEntries } from "./cases-CM-MQRQp.js";
|
|
13
|
+
import { f as cellMeasures, p as familyMeasures } from "./suite-DjGxmDSY.js";
|
|
14
|
+
import { c as readRecord, d as recordNames, g as warnUnreadable, h as scratchDir } from "./files-qjeqXnfE.js";
|
|
14
15
|
import { dirname, join } from "node:path";
|
|
15
16
|
import { mkdir, readFile, stat } from "node:fs/promises";
|
|
16
17
|
// src/history/rows.ts
|
|
17
18
|
/** A statement's rows as `Row`, the columns its SQL selects: SQLite checks none of them. */
|
|
18
19
|
const rowsOf = (statement, ...params) => statement.all(...params);
|
|
19
|
-
// src/history/tool-class.ts
|
|
20
|
-
const TOOL_CLASSES = [
|
|
21
|
-
"context",
|
|
22
|
-
"dependency",
|
|
23
|
-
"web",
|
|
24
|
-
"verify",
|
|
25
|
-
"workspace"
|
|
26
|
-
];
|
|
27
|
-
const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
|
|
28
|
-
const WEB_TOOL = /web|fetch|browse|url/i;
|
|
29
|
-
const DEPENDENCY = /(^|\/)(node_modules|site-packages|dist-packages|vendor|\.venv|\.cargo\/registry|go\/pkg\/mod)(\/|$)/;
|
|
30
|
-
const VERIFY = /\b(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|build|lint|typecheck|check)|npx\s+(?:tsc|vitest|jest|eslint|biome)|tsc|vitest|jest|pytest|eslint|cargo\s+(?:test|build|check)|go\s+(?:test|build|vet))\b/;
|
|
31
|
-
/** Words of a shell command, quotes removed; `;`, `|` and `&&` split words. */
|
|
32
|
-
function shellWords(command) {
|
|
33
|
-
const words = [];
|
|
34
|
-
let word = "";
|
|
35
|
-
let quote = null;
|
|
36
|
-
let started = false;
|
|
37
|
-
const flush = () => {
|
|
38
|
-
if (started) words.push(word);
|
|
39
|
-
word = "";
|
|
40
|
-
started = false;
|
|
41
|
-
};
|
|
42
|
-
for (const ch of command) {
|
|
43
|
-
if (quote) {
|
|
44
|
-
if (ch === quote) quote = null;
|
|
45
|
-
else word += ch;
|
|
46
|
-
continue;
|
|
47
|
-
}
|
|
48
|
-
if (ch === "\"" || ch === "'") {
|
|
49
|
-
quote = ch;
|
|
50
|
-
started = true;
|
|
51
|
-
} else if (/\s/.test(ch)) flush();
|
|
52
|
-
else if (ch === ";" || ch === "|" || ch === "&") {
|
|
53
|
-
flush();
|
|
54
|
-
words.push(ch);
|
|
55
|
-
} else {
|
|
56
|
-
word += ch;
|
|
57
|
-
started = true;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
flush();
|
|
61
|
-
return words;
|
|
62
|
-
}
|
|
63
|
-
/** A command's words, with a `bash -c '…'` (or `-lc`) script split into its own words. */
|
|
64
|
-
function commandWords(command) {
|
|
65
|
-
const words = shellWords(command);
|
|
66
|
-
return words.flatMap((w, i) => i > 0 && /^-[a-z]*c$/.test(words[i - 1]) && /\s/.test(w) ? commandWords(w) : [w]);
|
|
67
|
-
}
|
|
68
|
-
const EXCLUDING_FLAG = /^--(?:exclude|exclude-dir|ignore|ignore-dir)(?:=|$)/;
|
|
69
|
-
/** The words of a command that name something it reads, without the ones it excludes. */
|
|
70
|
-
function readWords(words) {
|
|
71
|
-
const out = [];
|
|
72
|
-
let test = false;
|
|
73
|
-
for (let i = 0; i < words.length; i++) {
|
|
74
|
-
const w = words[i];
|
|
75
|
-
const prev = words[i - 1];
|
|
76
|
-
if (w === "[" || w === "[[" || w === "test") test = true;
|
|
77
|
-
else if (w === "]" || w === "]]" || w === ";" || w === "|" || w === "&") test = false;
|
|
78
|
-
if (test && /^-[defsrwx]$/.test(w)) {
|
|
79
|
-
i++;
|
|
80
|
-
continue;
|
|
81
|
-
}
|
|
82
|
-
if (EXCLUDING_FLAG.test(w)) {
|
|
83
|
-
if (!w.includes("=")) i++;
|
|
84
|
-
continue;
|
|
85
|
-
}
|
|
86
|
-
if ((w === "-path" || w === "-name" || w === "-ipath" || w === "-wholename") && i + 1 < words.length) {
|
|
87
|
-
const pruned = words[i + 2] === "-prune";
|
|
88
|
-
if (prev === "-not" || prev === "!" || pruned) {
|
|
89
|
-
i++;
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
if ((w === "-g" || w === "--glob" || w === "--iglob") && words[i + 1]?.startsWith("!")) {
|
|
94
|
-
i++;
|
|
95
|
-
continue;
|
|
96
|
-
}
|
|
97
|
-
if (/^--i?glob=!/.test(w)) continue;
|
|
98
|
-
if (w === "-v" && i > 0) {
|
|
99
|
-
i++;
|
|
100
|
-
continue;
|
|
101
|
-
}
|
|
102
|
-
out.push(w);
|
|
103
|
-
}
|
|
104
|
-
return out;
|
|
105
|
-
}
|
|
106
|
-
/** `/home/<user>/x` and `/root/x` as `~/x`; `./x` as `x`. */
|
|
107
|
-
const normalize$1 = (p) => p.replace(/^\/home\/[^/]+\//, "~/").replace(/^\/root\//, "~/").replace(/^\$HOME\//, "~/").replace(/^\.\//, "");
|
|
108
|
-
const GLOB = /[*?[{]/;
|
|
109
|
-
/** `*`, `**` and `?` as wildcards; every other character, `[` included, literally. */
|
|
110
|
-
function wildcards(glob) {
|
|
111
|
-
let re = "";
|
|
112
|
-
for (let i = 0; i < glob.length; i++) {
|
|
113
|
-
const ch = glob[i];
|
|
114
|
-
if (ch === "*") {
|
|
115
|
-
if (glob[i + 1] === "*") {
|
|
116
|
-
re += ".*";
|
|
117
|
-
i++;
|
|
118
|
-
} else re += "[^/]*";
|
|
119
|
-
} else if (ch === "?") re += "[^/]";
|
|
120
|
-
else re += ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
121
|
-
}
|
|
122
|
-
return re;
|
|
123
|
-
}
|
|
124
|
-
/** A glob as a regex, with `{a,b}` alternatives. Words come from transcripts, so it never throws. */
|
|
125
|
-
function globRegex(glob) {
|
|
126
|
-
let re = "";
|
|
127
|
-
let from = 0;
|
|
128
|
-
for (let i = 0; i < glob.length; i++) {
|
|
129
|
-
if (glob[i] !== "{") continue;
|
|
130
|
-
const end = glob.indexOf("}", i);
|
|
131
|
-
if (end < 0) break;
|
|
132
|
-
re += wildcards(glob.slice(from, i));
|
|
133
|
-
re += `(?:${glob.slice(i + 1, end).split(",").map(wildcards).join("|")})`;
|
|
134
|
-
from = end + 1;
|
|
135
|
-
i = end;
|
|
136
|
-
}
|
|
137
|
-
return new RegExp(`^${re}${wildcards(glob.slice(from))}$`);
|
|
138
|
-
}
|
|
139
|
-
/** The declared files one word names: the file itself, one of a glob's matches, or a directory's. */
|
|
140
|
-
function filesOf(word, files) {
|
|
141
|
-
const w = normalize$1(word).replace(/\/+$/, "");
|
|
142
|
-
if (!w) return {
|
|
143
|
-
files: [],
|
|
144
|
-
dir: false
|
|
145
|
-
};
|
|
146
|
-
const forms = (f) => f.startsWith("~/") ? [f, f.slice(2)] : [f];
|
|
147
|
-
const matches = (f, test) => forms(f).some((form) => test(form) || !w.startsWith("~/") && test(`~/${form}`));
|
|
148
|
-
if (GLOB.test(w)) {
|
|
149
|
-
const re = globRegex(w);
|
|
150
|
-
const fromWorkspace = /^[*?[{]/.test(w);
|
|
151
|
-
return {
|
|
152
|
-
files: files.filter((f) => fromWorkspace ? !f.startsWith("~/") && re.test(f) : matches(f, (form) => re.test(form))),
|
|
153
|
-
dir: false
|
|
154
|
-
};
|
|
155
|
-
}
|
|
156
|
-
const exact = files.filter((f) => matches(f, (form) => form === w || form.endsWith(`/${w}`)));
|
|
157
|
-
if (exact.length) return {
|
|
158
|
-
files: exact,
|
|
159
|
-
dir: false
|
|
160
|
-
};
|
|
161
|
-
return {
|
|
162
|
-
files: [],
|
|
163
|
-
dir: files.filter((f) => matches(f, (form) => form.startsWith(`${w}/`))).length > 0
|
|
164
|
-
};
|
|
165
|
-
}
|
|
166
|
-
/** A tool call's class, and the declared files it read (for `skill_files`). */
|
|
167
|
-
function classifyCall(tool, target, ctx) {
|
|
168
|
-
if (SKILL_LOAD_TOOLS.has(tool)) {
|
|
169
|
-
if (!ctx.skills.includes(target)) return {
|
|
170
|
-
class: "workspace",
|
|
171
|
-
files: []
|
|
172
|
-
};
|
|
173
|
-
return {
|
|
174
|
-
class: "context",
|
|
175
|
-
files: ctx.files.filter((f) => f.endsWith(`skills/${target}/SKILL.md`))
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
if (ctx.mcp.some((s) => tool.startsWith(`mcp__${s}__`) || tool.startsWith(`${s}__`))) return {
|
|
179
|
-
class: "context",
|
|
180
|
-
files: []
|
|
181
|
-
};
|
|
182
|
-
const shell = SHELL_TOOL.test(tool);
|
|
183
|
-
const words = shell ? readWords(commandWords(target)) : [target];
|
|
184
|
-
const files = /* @__PURE__ */ new Set();
|
|
185
|
-
let context = false;
|
|
186
|
-
for (const w of words) {
|
|
187
|
-
const hit = filesOf(w, ctx.files);
|
|
188
|
-
for (const f of hit.files) files.add(f);
|
|
189
|
-
if (hit.files.length || hit.dir) context = true;
|
|
190
|
-
if (ctx.skills.some((s) => normalize$1(w).includes(`skills/${s}/`) || normalize$1(w).endsWith(`skills/${s}`))) context = true;
|
|
191
|
-
}
|
|
192
|
-
if (context) return {
|
|
193
|
-
class: "context",
|
|
194
|
-
files: [...files]
|
|
195
|
-
};
|
|
196
|
-
if (words.some((w) => DEPENDENCY.test(normalize$1(w))) || shell && words.includes("--help")) return {
|
|
197
|
-
class: "dependency",
|
|
198
|
-
files: []
|
|
199
|
-
};
|
|
200
|
-
if (!shell && WEB_TOOL.test(tool)) return {
|
|
201
|
-
class: "web",
|
|
202
|
-
files: []
|
|
203
|
-
};
|
|
204
|
-
if (shell && words.some((w) => w === "curl" || w === "wget") && /https?:\/\//.test(target)) return {
|
|
205
|
-
class: "web",
|
|
206
|
-
files: []
|
|
207
|
-
};
|
|
208
|
-
if (shell) {
|
|
209
|
-
const command = target.trim();
|
|
210
|
-
if (ctx.verify.some((v) => v.trim().length >= 4 && command.includes(v.trim()))) return {
|
|
211
|
-
class: "verify",
|
|
212
|
-
files: []
|
|
213
|
-
};
|
|
214
|
-
if (VERIFY.test(command)) return {
|
|
215
|
-
class: "verify",
|
|
216
|
-
files: []
|
|
217
|
-
};
|
|
218
|
-
}
|
|
219
|
-
return {
|
|
220
|
-
class: "workspace",
|
|
221
|
-
files: []
|
|
222
|
-
};
|
|
223
|
-
}
|
|
224
20
|
/** A failing verdict this close to its threshold points at the judge, not the agent. */
|
|
225
21
|
const JUDGE_MARGIN = .1;
|
|
226
22
|
const DERIVED_TABLES = `
|
|
@@ -229,15 +25,16 @@ CREATE TABLE effort_m (
|
|
|
229
25
|
environment TEXT, n INTEGER, tools_median REAL, tools_q1 REAL, tools_q3 REAL,
|
|
230
26
|
agent_s_median REAL, agent_s_q1 REAL, agent_s_q3 REAL, file_edits_median REAL,
|
|
231
27
|
files_changed_median REAL,
|
|
232
|
-
tokens_median REAL, tokens_q1 REAL, tokens_q3 REAL,
|
|
28
|
+
tokens_median REAL, tokens_q1 REAL, tokens_q3 REAL, cost_usd_median REAL,
|
|
233
29
|
control_key TEXT, control_n INTEGER, control_tools_median REAL, control_agent_s_median REAL,
|
|
234
|
-
control_tokens_median REAL,
|
|
30
|
+
control_tokens_median REAL, control_cost_usd_median REAL,
|
|
235
31
|
tools_ratio REAL, agent_s_ratio REAL, agent_s_delta REAL, tokens_ratio REAL,
|
|
236
32
|
tools_p REAL, agent_s_p REAL, tokens_p REAL,
|
|
237
33
|
separated INTEGER, time_separated INTEGER, tokens_separated INTEGER,
|
|
238
34
|
verdict TEXT, time_verdict TEXT, tokens_verdict TEXT, tokens_note TEXT,
|
|
239
35
|
context_calls_median REAL, dependency_calls_median REAL, web_calls_median REAL,
|
|
240
|
-
verify_calls_median REAL, workspace_calls_median REAL
|
|
36
|
+
verify_calls_median REAL, workspace_calls_median REAL,
|
|
37
|
+
model_s_median REAL, tool_s_median REAL, waiting_s_median REAL, other_s_median REAL
|
|
241
38
|
);
|
|
242
39
|
CREATE TABLE contrasts_m (
|
|
243
40
|
run_id TEXT, eval_key TEXT, harness TEXT, sandbox TEXT, a_key TEXT, b_key TEXT,
|
|
@@ -431,7 +228,8 @@ function deriveFacts(db) {
|
|
|
431
228
|
for (const table of DERIVED) db.exec(`DELETE FROM ${table}`);
|
|
432
229
|
const cells = rowsOf(db.prepare("SELECT c.*, r.started_at FROM cells c JOIN runs r USING (run_id) ORDER BY c.run_id, c.rowid"));
|
|
433
230
|
const attempts = rowsOf(db.prepare(`SELECT run_id, eval_key, cell_key, key, outcome, error_kind, case_key, tools, agent_ms, file_edits, files_changed,
|
|
434
|
-
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens
|
|
231
|
+
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens,
|
|
232
|
+
model_ms, tool_ms, waiting_ms, other_ms, cost_usd
|
|
435
233
|
FROM attempts ORDER BY run_id, rowid`));
|
|
436
234
|
const nearThreshold = new Set(rowsOf(db.prepare(`SELECT DISTINCT run_id, attempt_key FROM judgements WHERE verdict = 'fail' AND ABS(score - threshold) <= ${JUDGE_MARGIN}`)).map((r) => `${r.run_id}\0${r.attempt_key}`));
|
|
437
235
|
const diffs = new Map(rowsOf(db.prepare(`SELECT run_id, attempt_key, sha256 FROM artifacts WHERE kind = 'diff' AND bytes > 0`)).flatMap((r) => r.sha256 ? [[`${r.run_id}\0${r.attempt_key}`, r.sha256]] : []));
|
|
@@ -453,7 +251,14 @@ function deriveFacts(db) {
|
|
|
453
251
|
agentMs: a.agent_ms,
|
|
454
252
|
fileEdits: a.file_edits,
|
|
455
253
|
filesChanged: a.files_changed,
|
|
456
|
-
...tokens !== void 0 ? { tokens } : {}
|
|
254
|
+
...tokens !== void 0 ? { tokens } : {},
|
|
255
|
+
...a.tool_ms !== null ? { split: {
|
|
256
|
+
modelMs: a.model_ms,
|
|
257
|
+
toolMs: a.tool_ms,
|
|
258
|
+
waitingMs: a.waiting_ms ?? 0,
|
|
259
|
+
otherMs: a.other_ms ?? 0
|
|
260
|
+
} } : {},
|
|
261
|
+
...a.cost_usd !== null ? { usd: a.cost_usd } : {}
|
|
457
262
|
}]);
|
|
458
263
|
}
|
|
459
264
|
const side = (c) => ({
|
|
@@ -500,11 +305,13 @@ function deriveFacts(db) {
|
|
|
500
305
|
tokens_median: own?.tokens?.median ?? null,
|
|
501
306
|
tokens_q1: own?.tokens?.q1 ?? null,
|
|
502
307
|
tokens_q3: own?.tokens?.q3 ?? null,
|
|
308
|
+
cost_usd_median: own?.usd?.median ?? null,
|
|
503
309
|
control_key: control?.key ?? null,
|
|
504
310
|
control_n: vs?.control.n ?? null,
|
|
505
311
|
control_tools_median: vs?.control.tools.median ?? null,
|
|
506
312
|
control_agent_s_median: vs ? s1(vs.control.agentMs.median) : null,
|
|
507
313
|
control_tokens_median: vs?.control.tokens?.median ?? null,
|
|
314
|
+
control_cost_usd_median: vs?.control.usd?.median ?? null,
|
|
508
315
|
tools_ratio: r4(vs?.toolsRatio ?? null),
|
|
509
316
|
agent_s_ratio: r4(vs?.agentMsRatio ?? null),
|
|
510
317
|
agent_s_delta: vs ? s1(vs.agentMsDelta) : null,
|
|
@@ -519,7 +326,11 @@ function deriveFacts(db) {
|
|
|
519
326
|
time_verdict: vs?.timeVerdict ?? null,
|
|
520
327
|
tokens_verdict: vs?.tokensVerdict ?? null,
|
|
521
328
|
tokens_note: vs?.tokensNote ?? null,
|
|
522
|
-
...classMedians(c)
|
|
329
|
+
...classMedians(c),
|
|
330
|
+
model_s_median: own?.split?.modelMs == null ? null : s1(own.split.modelMs),
|
|
331
|
+
tool_s_median: own?.split ? s1(own.split.toolMs) : null,
|
|
332
|
+
waiting_s_median: own?.split ? s1(own.split.waitingMs) : null,
|
|
333
|
+
other_s_median: own?.split ? s1(own.split.otherMs) : null
|
|
523
334
|
};
|
|
524
335
|
}));
|
|
525
336
|
const checkRows = rowsOf(db.prepare("SELECT run_id, eval_key, cell_key, label, pass, fail, error, not_run FROM cell_checks"));
|
|
@@ -954,7 +765,7 @@ function checkStreaks(db) {
|
|
|
954
765
|
}
|
|
955
766
|
// src/history/db.ts
|
|
956
767
|
/** Bump when the tables or views change: a database from another version is rebuilt. */
|
|
957
|
-
const DB_VERSION =
|
|
768
|
+
const DB_VERSION = 25;
|
|
958
769
|
/** This build of reeval, like the header cache: another reeval re-derives every row. */
|
|
959
770
|
const BUILD = BUILD_ID;
|
|
960
771
|
const PART_KEYS = Object.keys(PART_COLUMNS);
|
|
@@ -1078,12 +889,18 @@ CREATE TABLE attempts (
|
|
|
1078
889
|
last_text TEXT, input_tokens INTEGER, output_tokens INTEGER, cache_read_tokens INTEGER,
|
|
1079
890
|
cache_write_tokens INTEGER, reasoning_tokens INTEGER, served_model TEXT, oom_kills INTEGER,
|
|
1080
891
|
reasoning_effort TEXT, served_reasoning_effort TEXT, files_changed INTEGER, reasoning_ms INTEGER,
|
|
1081
|
-
case_key TEXT,
|
|
892
|
+
case_key TEXT, model_ms INTEGER, tool_ms INTEGER, waiting_ms INTEGER, other_ms INTEGER,
|
|
893
|
+
cost_usd REAL, cost_basis TEXT, resource_scope TEXT, machine_size TEXT, machine_vcpus REAL,
|
|
894
|
+
machine_memory_bytes INTEGER, cpu_seconds REAL, avg_cores REAL, peak_cores REAL,
|
|
895
|
+
peak_memory_bytes INTEGER, disk_read_bytes INTEGER, disk_write_bytes INTEGER, p95_cores REAL,
|
|
896
|
+
cgroup_peak_memory_bytes INTEGER, cpu_periods INTEGER, throttled_periods INTEGER,
|
|
897
|
+
throttled_seconds REAL,
|
|
1082
898
|
PRIMARY KEY (run_id, key)
|
|
1083
899
|
);
|
|
1084
900
|
CREATE TABLE judgements (
|
|
1085
901
|
run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, key TEXT NOT NULL, question TEXT,
|
|
1086
902
|
verdict TEXT, score REAL, threshold REAL, model TEXT, resolved_model TEXT, input_path TEXT,
|
|
903
|
+
cost_usd REAL, cost_basis TEXT,
|
|
1087
904
|
PRIMARY KEY (run_id, attempt_key, key)
|
|
1088
905
|
);
|
|
1089
906
|
CREATE TABLE environments (
|
|
@@ -1116,6 +933,22 @@ CREATE TABLE tool_calls (
|
|
|
1116
933
|
CREATE TABLE context_reads (
|
|
1117
934
|
run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, line INTEGER NOT NULL, path TEXT NOT NULL
|
|
1118
935
|
);
|
|
936
|
+
-- Each skill a tool call used, by the call's line: a load by name (Claude Code's Skill, OpenCode's
|
|
937
|
+
-- and fx's skill, Goose's load_skill) or a target under skills/<name>. source: placed (by the
|
|
938
|
+
-- environment's skills or a plugin, by toHaveCalledSkill's rule), skill-dir (another skill in a
|
|
939
|
+
-- skill directory some agent reads: a setup step, the workspace or a package manager put it there)
|
|
940
|
+
-- or elsewhere (a skills/<name>/SKILL.md read anywhere else, such as a package cache).
|
|
941
|
+
CREATE TABLE skill_uses (
|
|
942
|
+
run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, line INTEGER NOT NULL, skill TEXT NOT NULL,
|
|
943
|
+
source TEXT NOT NULL
|
|
944
|
+
);
|
|
945
|
+
-- Each skill an attempt had or used that its environment didn't place: in an agent's skill
|
|
946
|
+
-- directory when the context was taken (source skill-dir, with its SKILL.md's path), or only seen
|
|
947
|
+
-- in a tool call (that call's source, and a null path).
|
|
948
|
+
CREATE TABLE found_skills (
|
|
949
|
+
run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, skill TEXT NOT NULL, source TEXT NOT NULL,
|
|
950
|
+
path TEXT
|
|
951
|
+
);
|
|
1119
952
|
-- Each stdio MCP server start in each attempt's turns (turn from 1): ms from spawning it to its
|
|
1120
953
|
-- answer to initialize, null when it never answered.
|
|
1121
954
|
CREATE TABLE mcp_startup (
|
|
@@ -1139,6 +972,8 @@ CREATE INDEX cells_by_eval ON cells (eval_key, key);
|
|
|
1139
972
|
CREATE INDEX tool_calls_by_attempt ON tool_calls (run_id, attempt_key);
|
|
1140
973
|
CREATE INDEX context_by_attempt ON attempt_context (run_id, attempt_key);
|
|
1141
974
|
CREATE INDEX context_reads_by_attempt ON context_reads (run_id, attempt_key);
|
|
975
|
+
CREATE INDEX skill_uses_by_attempt ON skill_uses (run_id, attempt_key);
|
|
976
|
+
CREATE INDEX found_skills_by_attempt ON found_skills (run_id, attempt_key);
|
|
1142
977
|
CREATE INDEX attempts_by_cell ON attempts (run_id, eval_key, cell_key);
|
|
1143
978
|
`;
|
|
1144
979
|
/** Canned questions about the history, as views to filter. */
|
|
@@ -1179,7 +1014,8 @@ CREATE VIEW drift AS SELECT * FROM drift_m;
|
|
|
1179
1014
|
-- the run that ended the streak (null: it never failed).
|
|
1180
1015
|
CREATE VIEW check_streaks AS SELECT * FROM check_streaks_m;
|
|
1181
1016
|
|
|
1182
|
-
-- Tool calls, agent time and
|
|
1017
|
+
-- Tool calls, agent time and where it went (model, tool, waiting, other), and tokens per cell, and
|
|
1018
|
+
-- a variant's against its control in the same run.
|
|
1183
1019
|
CREATE VIEW effort AS SELECT * FROM effort_m;
|
|
1184
1020
|
|
|
1185
1021
|
-- Each pair of arms (cells a and b) on one harness and sandbox, when an eval has two variants or
|
|
@@ -1209,25 +1045,37 @@ CREATE VIEW question_pairs AS SELECT * FROM question_pairs_m;
|
|
|
1209
1045
|
-- pooled (comparable runs, and across evals), observed (over time), not_measured (one level).
|
|
1210
1046
|
CREATE VIEW factor_effects AS SELECT * FROM factor_effects_m;
|
|
1211
1047
|
|
|
1212
|
-
-- Whether each attempt read each skill
|
|
1213
|
-
--
|
|
1214
|
-
--
|
|
1048
|
+
-- Whether each attempt read each skill it had, and at which transcript line: the tool calls
|
|
1049
|
+
-- skill_uses has for it. A cell's skills are the ones its environment placed (source placed, with
|
|
1050
|
+
-- their placement) and the ones any of its attempts found (skill-dir or elsewhere, as found_skills
|
|
1051
|
+
-- has them), so a found skill an attempt never opened reads 0. An attempt whose record has tool
|
|
1052
|
+
-- calls but whose transcript wasn't kept (or couldn't be read) is left out: unknown, not unread.
|
|
1215
1053
|
CREATE VIEW skill_reads AS
|
|
1054
|
+
WITH skills AS (
|
|
1055
|
+
SELECT c.run_id, c.eval_key, c.key AS cell_key, s.name AS skill, s.placement,
|
|
1056
|
+
'placed' AS source
|
|
1057
|
+
FROM cells c JOIN env_skills s ON s.run_id = c.run_id AND s.environment = c.environment
|
|
1058
|
+
AND s.digest = c.environment_digest
|
|
1059
|
+
UNION ALL
|
|
1060
|
+
SELECT a.run_id, a.eval_key, a.cell_key, f.skill, NULL AS placement,
|
|
1061
|
+
CASE WHEN MAX(f.source = 'skill-dir') THEN 'skill-dir' ELSE 'elsewhere' END AS source
|
|
1062
|
+
FROM found_skills f JOIN attempts a ON a.run_id = f.run_id AND a.key = f.attempt_key
|
|
1063
|
+
GROUP BY a.run_id, a.eval_key, a.cell_key, f.skill
|
|
1064
|
+
)
|
|
1216
1065
|
SELECT a.run_id, a.eval_key, a.cell_key, a.key AS attempt_key, a.id AS attempt_id, c.variant,
|
|
1217
|
-
s.
|
|
1218
|
-
(SELECT MIN(
|
|
1219
|
-
AND
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
AND (t.target LIKE '%skills/' || s.name || '/%'
|
|
1223
|
-
OR (t.tool IN ('skill', 'Skill', 'load_skill') AND t.target = s.name))) AS reads
|
|
1066
|
+
s.skill, s.placement, s.source,
|
|
1067
|
+
(SELECT MIN(u.line) FROM skill_uses u WHERE u.run_id = a.run_id AND u.attempt_key = a.key
|
|
1068
|
+
AND u.skill = s.skill) AS read_line,
|
|
1069
|
+
(SELECT COUNT(*) FROM skill_uses u WHERE u.run_id = a.run_id AND u.attempt_key = a.key
|
|
1070
|
+
AND u.skill = s.skill) AS reads
|
|
1224
1071
|
FROM attempts a
|
|
1225
1072
|
JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
|
|
1226
|
-
JOIN
|
|
1227
|
-
|
|
1073
|
+
JOIN skills s ON s.run_id = a.run_id AND s.eval_key = a.eval_key AND s.cell_key = a.cell_key
|
|
1074
|
+
WHERE a.tools = 0
|
|
1075
|
+
OR EXISTS (SELECT 1 FROM tool_calls t WHERE t.run_id = a.run_id AND t.attempt_key = a.key);
|
|
1228
1076
|
|
|
1229
|
-
-- Tool calls per attempt by class: context (a placed
|
|
1230
|
-
--
|
|
1077
|
+
-- Tool calls per attempt by class: context (a skill, placed or found, a rules file or other
|
|
1078
|
+
-- placed file, an MCP tool of the environment), dependency (node_modules, site-packages, vendor,
|
|
1231
1079
|
-- --help), web, verify (a command the eval's checks run, or test/build/lint/tsc) and workspace.
|
|
1232
1080
|
-- A heuristic: shell commands are split into words and exclusions are not counted.
|
|
1233
1081
|
CREATE VIEW context_use AS
|
|
@@ -1242,6 +1090,7 @@ CREATE VIEW context_use AS
|
|
|
1242
1090
|
|
|
1243
1091
|
-- Each file the environment declared (its skills' files, rules), per attempt: how many tool calls
|
|
1244
1092
|
-- named it, directly or through a glob, and the first one's transcript line (null: never read).
|
|
1093
|
+
-- Left out, like skill_reads, for an attempt whose tool calls weren't read from a transcript.
|
|
1245
1094
|
CREATE VIEW skill_files AS
|
|
1246
1095
|
SELECT a.run_id, a.eval_key, a.cell_key, c.environment, a.key AS attempt_key,
|
|
1247
1096
|
a.id AS attempt_id, x.path,
|
|
@@ -1252,7 +1101,8 @@ CREATE VIEW skill_files AS
|
|
|
1252
1101
|
FROM attempt_context x
|
|
1253
1102
|
JOIN attempts a ON a.run_id = x.run_id AND a.key = x.attempt_key
|
|
1254
1103
|
JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
|
|
1255
|
-
WHERE x.declared = 1
|
|
1104
|
+
WHERE x.declared = 1 AND (a.tools = 0
|
|
1105
|
+
OR EXISTS (SELECT 1 FROM tool_calls t WHERE t.run_id = a.run_id AND t.attempt_key = a.key));
|
|
1256
1106
|
|
|
1257
1107
|
-- Failed attempts and infra errors (an OOM kill among them) with the cell's harness, model, each
|
|
1258
1108
|
-- of its environment's skills and the label of the first check that failed.
|
|
@@ -1268,6 +1118,27 @@ CREATE VIEW failing_attempts AS
|
|
|
1268
1118
|
AND s.digest = c.environment_digest
|
|
1269
1119
|
WHERE a.outcome IN ('fail', 'infra_error');
|
|
1270
1120
|
|
|
1121
|
+
-- Each attempt's machine (its sandbox's provider and size, and the vCPUs and memory it gave) and
|
|
1122
|
+
-- what the attempt used there: CPU seconds, average, 95th percentile and peak cores, peak memory,
|
|
1123
|
+
-- a container's cgroup memory.peak and CPU throttling, and a VM's disk bytes; cpu_share and
|
|
1124
|
+
-- memory_share are average cores and peak memory over what it gave, throttled_share the share of
|
|
1125
|
+
-- a container's quota periods the quota held it back in. scope
|
|
1126
|
+
-- container counts the container's own processes, machine the whole VM. The fake sandbox's
|
|
1127
|
+
-- attempts are left out; one whose sandbox had nothing to read (cgroup v1, a macOS VM) has its
|
|
1128
|
+
-- machine and null usage.
|
|
1129
|
+
CREATE VIEW attempt_resources AS
|
|
1130
|
+
SELECT a.run_id, r.started_at, a.eval_key, a.cell_key, a.key AS attempt_key, c.harness,
|
|
1131
|
+
c.model, c.environment, c.sandbox, a.resource_scope AS scope, a.machine_size,
|
|
1132
|
+
a.machine_vcpus, a.machine_memory_bytes, a.outcome, a.duration_ms, a.cpu_seconds,
|
|
1133
|
+
a.avg_cores, a.p95_cores, a.peak_cores, a.peak_memory_bytes, a.cgroup_peak_memory_bytes,
|
|
1134
|
+
a.cpu_periods, a.throttled_periods, a.throttled_seconds, a.disk_read_bytes,
|
|
1135
|
+
a.disk_write_bytes, a.avg_cores / a.machine_vcpus AS cpu_share,
|
|
1136
|
+
CAST(a.peak_memory_bytes AS REAL) / a.machine_memory_bytes AS memory_share,
|
|
1137
|
+
CAST(a.throttled_periods AS REAL) / NULLIF(a.cpu_periods, 0) AS throttled_share
|
|
1138
|
+
FROM attempts a JOIN runs r USING (run_id)
|
|
1139
|
+
JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
|
|
1140
|
+
WHERE a.resource_scope IS NOT NULL;
|
|
1141
|
+
|
|
1271
1142
|
-- Each check of a variant cell beside its control's: whether it tells them apart.
|
|
1272
1143
|
CREATE VIEW check_discrimination AS
|
|
1273
1144
|
SELECT k.run_id, k.eval_key, c.harness, c.variant, c.key AS cell_key, k.label,
|
|
@@ -1324,6 +1195,7 @@ const VIEW_NAMES = [
|
|
|
1324
1195
|
"flaky_cells",
|
|
1325
1196
|
"drift",
|
|
1326
1197
|
"failing_attempts",
|
|
1198
|
+
"attempt_resources",
|
|
1327
1199
|
"eval_trend",
|
|
1328
1200
|
"verdict_flips",
|
|
1329
1201
|
"check_discrimination",
|
|
@@ -1360,20 +1232,44 @@ function toolCalls(transcript) {
|
|
|
1360
1232
|
turn: typeof event.turn === "number" ? event.turn : null,
|
|
1361
1233
|
tool: String(event.tool ?? ""),
|
|
1362
1234
|
target: toolTarget(event.input, String(event.tool ?? "")),
|
|
1235
|
+
text: targetText(event.input, String(event.tool ?? "")),
|
|
1363
1236
|
ok: event.ok === null ? null : event.ok !== false,
|
|
1364
|
-
durationMs: typeof event.durationMs === "number" ? event.durationMs : null
|
|
1237
|
+
durationMs: typeof event.durationMs === "number" && !event.durationInferred ? event.durationMs : null
|
|
1365
1238
|
});
|
|
1366
1239
|
});
|
|
1367
1240
|
return calls;
|
|
1368
1241
|
}
|
|
1242
|
+
/**
|
|
1243
|
+
* The skills whose `SKILL.md` an attempt wrote: by a file-editing tool that created or rewrote it,
|
|
1244
|
+
* or by a shell command (a redirect, `tee`, `cp` and the like).
|
|
1245
|
+
*/
|
|
1246
|
+
function skillsAuthored(transcript, calls) {
|
|
1247
|
+
const authored = new Set(calls.flatMap((t) => skillsWritten(t.tool, t.text)));
|
|
1248
|
+
for (const line of transcript.split("\n")) {
|
|
1249
|
+
if (!line.includes("\"file-edit\"")) continue;
|
|
1250
|
+
let event;
|
|
1251
|
+
try {
|
|
1252
|
+
event = JSON.parse(line);
|
|
1253
|
+
} catch {
|
|
1254
|
+
continue;
|
|
1255
|
+
}
|
|
1256
|
+
if (event?.type !== "file-edit" || event.kind !== "create" && event.kind !== "write") continue;
|
|
1257
|
+
for (const name of skillsWritten("write", String(event.path ?? ""))) authored.add(name);
|
|
1258
|
+
}
|
|
1259
|
+
return authored;
|
|
1260
|
+
}
|
|
1369
1261
|
/** `classifyCall`, but a call its heuristic can't read counts as workspace: one transcript line can't stop the sync. */
|
|
1370
1262
|
function classifyOrWorkspace(tool, target, ctx) {
|
|
1371
1263
|
try {
|
|
1372
|
-
return
|
|
1264
|
+
return {
|
|
1265
|
+
...classifyCall(tool, target, ctx),
|
|
1266
|
+
skills: skillUses(tool, target, ctx.skills, ctx.skillDirs, ctx.found)
|
|
1267
|
+
};
|
|
1373
1268
|
} catch {
|
|
1374
1269
|
return {
|
|
1375
1270
|
class: "workspace",
|
|
1376
|
-
files: []
|
|
1271
|
+
files: [],
|
|
1272
|
+
skills: []
|
|
1377
1273
|
};
|
|
1378
1274
|
}
|
|
1379
1275
|
}
|
|
@@ -1433,27 +1329,49 @@ function insertRun(db, record, recorded, transcripts) {
|
|
|
1433
1329
|
for (const t of pairTallies(answer.votes)) run(`INSERT INTO question_pairs_m VALUES (${Array(14).fill("?").join(", ")})`, record.runId, e.key, answer.question, t.cut, t.held, t.a, t.b, t.pairs, t.aWins, t.bWins, t.ties, t.splits, t.splitsFirst, caseKey);
|
|
1434
1330
|
}
|
|
1435
1331
|
for (const a of e.attempts) {
|
|
1436
|
-
run(`INSERT INTO attempts VALUES (${Array(
|
|
1332
|
+
run(`INSERT INTO attempts VALUES (${Array(54).fill("?").join(", ")})`, record.runId, e.key, a.cell, a.key, a.id, a.n, a.seq, a.startedAt, a.outcome, a.error?.kind ?? null, a.error?.message ?? null, a.durationMs, a.imageId, a.turns.length, a.activity.tools, a.activity.fileEdits, a.timings.setupMs, a.timings.agentMs, a.timings.checksMs, a.timings.judgeMs, a.turns.at(-1)?.text.slice(0, 2048) ?? null, a.usage?.input ?? null, a.usage?.output ?? null, a.usage?.cacheRead ?? null, a.usage?.cacheWrite ?? null, a.usage?.reasoning ?? null, a.turns.findLast((t) => t.model)?.model ?? null, a.error?.resource?.cause === "oom" ? a.error.resource.kills : 0, a.effort ?? null, a.servedEffort ?? null, filesChanged(a.changedFiles), a.activity.reasoningMs, a.case ?? null, a.timeSplit?.modelMs ?? null, a.timeSplit?.toolMs ?? null, a.timeSplit?.waitingMs ?? null, a.timeSplit?.otherMs ?? null, a.cost?.usd ?? null, a.cost?.basis ?? null, a.resources?.scope ?? null, a.resources?.machine.size ?? null, a.resources?.machine.vcpus ?? null, a.resources?.machine.memoryBytes ?? null, a.resources?.usage?.cpuSeconds ?? null, a.resources?.usage?.avgCores ?? null, a.resources?.usage?.peakCores ?? null, a.resources?.usage?.peakMemoryBytes ?? null, a.resources?.usage?.diskReadBytes ?? null, a.resources?.usage?.diskWriteBytes ?? null, a.resources?.usage?.p95Cores ?? null, a.resources?.usage?.cgroupPeakMemoryBytes ?? null, a.resources?.usage?.throttling?.periods ?? null, a.resources?.usage?.throttling?.throttledPeriods ?? null, a.resources?.usage?.throttling?.throttledSeconds ?? null);
|
|
1437
1333
|
const env = envOf.get(cellEnv.get(a.cell) ?? "");
|
|
1438
1334
|
const skills = env?.skills.map((x) => x.name) ?? [];
|
|
1439
|
-
|
|
1335
|
+
const transcript = transcripts.get(a.key);
|
|
1336
|
+
const calls = transcript ? toolCalls(transcript) : [];
|
|
1337
|
+
const authored = transcript ? skillsAuthored(transcript, calls) : /* @__PURE__ */ new Set();
|
|
1338
|
+
const found = /* @__PURE__ */ new Map();
|
|
1339
|
+
for (const c of a.context) {
|
|
1340
|
+
const skill = isRuntimeState(c.path) ? void 0 : skillDirSkill(c.path, SKILL_DIRS);
|
|
1341
|
+
if (skill && !skills.includes(skill) && !authored.has(skill) && !found.has(skill)) found.set(skill, {
|
|
1342
|
+
source: "skill-dir",
|
|
1343
|
+
path: c.path
|
|
1344
|
+
});
|
|
1345
|
+
run("INSERT INTO attempt_context VALUES (?, ?, ?, ?, ?, ?)", record.runId, a.key, c.path, c.sha256, isRuntimeState(c.path) ? 1 : 0, declaredContext(c.path, skills) ? 1 : 0);
|
|
1346
|
+
}
|
|
1440
1347
|
for (const [i, t] of a.turns.entries()) for (const m of t.mcpStartup ?? []) run("INSERT INTO mcp_startup VALUES (?, ?, ?, ?, ?)", record.runId, a.key, i + 1, m.server, m.ms);
|
|
1441
1348
|
for (const [i, x] of (a.assertions ?? []).entries()) run(`INSERT INTO assertions VALUES (${Array(12).fill("?").join(", ")})`, record.runId, a.key, e.key, a.cell, i + 1, x.label, x.labelled ? 1 : 0, x.matcher, x.status, x.soft ? 1 : 0, x.message ?? null, x.durationMs ?? null);
|
|
1442
|
-
const transcript = transcripts.get(a.key);
|
|
1443
1349
|
const ctx = {
|
|
1444
1350
|
skills,
|
|
1351
|
+
skillDirs: SKILL_DIRS,
|
|
1352
|
+
found: [...found.keys()],
|
|
1445
1353
|
files: a.context.map((c) => c.path).filter((p) => declaredContext(p, skills)),
|
|
1446
1354
|
mcp: env?.mcp ?? [],
|
|
1447
1355
|
verify: a.checks.map((c) => c.command)
|
|
1448
1356
|
};
|
|
1449
|
-
for (const t of
|
|
1450
|
-
const call = classifyOrWorkspace(t.tool, t.
|
|
1357
|
+
for (const t of calls) {
|
|
1358
|
+
const call = classifyOrWorkspace(t.tool, t.text, ctx);
|
|
1451
1359
|
run("INSERT INTO tool_calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, t.line, t.turn, t.tool, t.target, t.ok === null ? null : t.ok ? 1 : 0, t.durationMs, call.class);
|
|
1452
1360
|
for (const path of call.files) run("INSERT INTO context_reads VALUES (?, ?, ?, ?)", record.runId, a.key, t.line, path);
|
|
1361
|
+
for (const { skill, source } of call.skills) {
|
|
1362
|
+
if (source !== "placed" && (t.ok === false || authored.has(skill))) continue;
|
|
1363
|
+
run("INSERT INTO skill_uses VALUES (?, ?, ?, ?, ?)", record.runId, a.key, t.line, skill, source);
|
|
1364
|
+
const had = found.get(skill);
|
|
1365
|
+
if (source !== "placed" && had?.source !== "skill-dir") found.set(skill, {
|
|
1366
|
+
source,
|
|
1367
|
+
path: had?.path ?? null
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1453
1370
|
}
|
|
1371
|
+
for (const [skill, f] of found) run("INSERT INTO found_skills VALUES (?, ?, ?, ?, ?)", record.runId, a.key, skill, f.source, f.path);
|
|
1454
1372
|
for (const q of a.answers ?? []) run(`INSERT INTO attempt_answers_m VALUES (${Array(14).fill("?").join(", ")})`, record.runId, e.key, a.cell, a.key, a.id, a.outcome, q.question, q.choice ?? null, q.text, q.error ?? null, q.model, q.resolvedModel ?? null, q.input?.path ?? null, a.case ?? null);
|
|
1455
1373
|
for (const o of a.observations ?? []) run(`INSERT INTO attempt_values_m VALUES (${Array(13).fill("?").join(", ")})`, record.runId, e.key, a.cell, a.key, a.id, a.outcome, o.name, JSON.stringify(o.value), valueKind(o.value), typeof o.value === "number" ? o.value : null, typeof o.value === "string" ? o.value : null, o.turn, a.case ?? null);
|
|
1456
|
-
for (const j of a.judgements) run("INSERT OR IGNORE INTO judgements VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, j.key, j.question, j.verdict, j.score, j.threshold, j.model, j.resolvedModel ?? null, a.files.judgeInput?.path ?? null);
|
|
1374
|
+
for (const j of a.judgements) run("INSERT OR IGNORE INTO judgements VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, j.key, j.question, j.verdict, j.score, j.threshold, j.model, j.resolvedModel ?? null, a.files.judgeInput?.path ?? null, j.cost?.usd ?? null, j.cost?.basis ?? null);
|
|
1457
1375
|
for (const [kind, ref] of Object.entries(a.files)) {
|
|
1458
1376
|
if (ref == null || Array.isArray(ref) || typeof ref.path !== "string") continue;
|
|
1459
1377
|
run("INSERT INTO artifacts VALUES (?, ?, ?, ?, ?, ?)", record.runId, a.key, kind, ref.path, ref.sha256, ref.bytes);
|
|
@@ -1495,6 +1413,8 @@ const DATA_TABLES = [
|
|
|
1495
1413
|
"attempt_context",
|
|
1496
1414
|
"tool_calls",
|
|
1497
1415
|
"context_reads",
|
|
1416
|
+
"skill_uses",
|
|
1417
|
+
"found_skills",
|
|
1498
1418
|
"mcp_startup",
|
|
1499
1419
|
"assertions",
|
|
1500
1420
|
"cell_checks",
|
|
@@ -1630,6 +1550,6 @@ async function openHistoryDb(root, historyDir, options = {}) {
|
|
|
1630
1550
|
close: () => db.close()
|
|
1631
1551
|
};
|
|
1632
1552
|
}
|
|
1633
|
-
export { historyDbPath as a, toolCalls as c, comparableFingerprint as i,
|
|
1553
|
+
export { historyDbPath as a, toolCalls as c, comparableFingerprint as i, SQLITE_NODE as n, nodeAtLeast as o, VIEW_NAMES as r, openHistoryDb as s, LAST_TEXT_LIMIT as t };
|
|
1634
1554
|
|
|
1635
|
-
//# sourceMappingURL=db-
|
|
1555
|
+
//# sourceMappingURL=db-ErnJ-gMD.js.map
|