reeval 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +841 -0
- package/README.md +15 -15
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C60z3wLq.js → adapter-Di9H2etg.js} +8 -49
- package/dist/adapter-Di9H2etg.js.map +1 -0
- package/dist/{agent-BmsMXZXx.js → agent-B-6GFDqA.js} +185 -38
- package/dist/agent-B-6GFDqA.js.map +1 -0
- package/dist/{agents-DXuFxtGC.js → agents-KBTYhSSh.js} +385 -116
- package/dist/agents-KBTYhSSh.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-Cmh9FGr5.js} +3 -3
- package/dist/app-files-Cmh9FGr5.js.map +1 -0
- package/dist/{assemble-CE4pHKIa.js → assemble-CBMRIhso.js} +1 -1
- package/dist/{assemble-DM2PNU4R.js → assemble-DNVBTBp_.js} +172 -18
- package/dist/assemble-DNVBTBp_.js.map +1 -0
- package/dist/attach-EbDGKGTO.js +152 -0
- package/dist/attach-EbDGKGTO.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-_dUsblVx.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-_dUsblVx.js.map} +1 -1
- package/dist/{attempt-host-7VviAJrl.js → attempt-host-DQvynDHk.js} +17 -8
- package/dist/attempt-host-DQvynDHk.js.map +1 -0
- package/dist/boat-Bzz8L-KQ.js +701 -0
- package/dist/boat-Bzz8L-KQ.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-Dek8ftIt.js} +20 -8
- package/dist/boat-Dek8ftIt.js.map +1 -0
- package/dist/bridge/index.mjs +6 -1
- package/dist/bundle-DZstHDeW.js +718 -0
- package/dist/bundle-DZstHDeW.js.map +1 -0
- package/dist/{capture-Ccf0Hmyx.js → capture-BkbPsfej.js} +61 -32
- package/dist/capture-BkbPsfej.js.map +1 -0
- package/dist/capture-CUAvk9KM.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-DTuLSlmW.js → cli-BgBBuxxJ.js} +412 -1044
- package/dist/cli-BgBBuxxJ.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-B5L9n-DV.js} +199 -35
- package/dist/cli-spec-B5L9n-DV.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-DMEYR3MA.js +807 -0
- package/dist/client-DMEYR3MA.js.map +1 -0
- package/dist/{command-C2C7P5qo.js → command-BKb8eN_X.js} +10 -10
- package/dist/{command-C2C7P5qo.js.map → command-BKb8eN_X.js.map} +1 -1
- package/dist/{command-DSL7I-SI.js → command-D_w_kDpI.js} +25 -21
- package/dist/command-D_w_kDpI.js.map +1 -0
- package/dist/{command-Ff0T-1Hi.js → command-Dwlh0Vhu.js} +17 -16
- package/dist/command-Dwlh0Vhu.js.map +1 -0
- package/dist/{command-D8oMB2fa.js → command-loXH2jHr.js} +16 -16
- package/dist/{command-D8oMB2fa.js.map → command-loXH2jHr.js.map} +1 -1
- package/dist/{commands-xAOi_VEk.js → commands-Cuf57OYb.js} +174 -40
- package/dist/commands-Cuf57OYb.js.map +1 -0
- package/dist/{commands-Bk9NG1s_.js → commands-Dqli45sR.js} +1 -1
- package/dist/{config-SvK8pjEK.js → config-C0RKt00M.js} +288 -106
- package/dist/config-C0RKt00M.js.map +1 -0
- package/dist/{context-BdgCao9Q.js → context-DrEAtUyg.js} +2 -2
- package/dist/{context-BdgCao9Q.js.map → context-DrEAtUyg.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-DEzan4IT.js → cursor-catalog-CRsFxorF.js} +4 -4
- package/dist/{cursor-catalog-DEzan4IT.js.map → cursor-catalog-CRsFxorF.js.map} +1 -1
- package/dist/db-3o8s2EyD.js +2 -0
- package/dist/{db-BzOJ4Js4.js → db-ErnJ-gMD.js} +168 -248
- package/dist/db-ErnJ-gMD.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BDKIxJyC.js → default-effort-D70cTG00.js} +4 -4
- package/dist/{default-effort-BDKIxJyC.js.map → default-effort-D70cTG00.js.map} +1 -1
- package/dist/{default-effort-BBDPbJ4D.js → default-effort-DHzj84dD.js} +1 -1
- package/dist/{dist-Db7ifchA.js → dist-DeqeCUZe.js} +5 -5
- package/dist/{dist-Db7ifchA.js.map → dist-DeqeCUZe.js.map} +1 -1
- package/dist/{docker-DFrjoNs-.js → docker-CFWzZOEk.js} +4 -4
- package/dist/{docker-DFrjoNs-.js.map → docker-CFWzZOEk.js.map} +1 -1
- package/dist/docker-DrMnpL84.js +1 -0
- package/dist/{doctor-DWK4ttex.js → doctor-B7utCSdh.js} +121 -100
- package/dist/doctor-B7utCSdh.js.map +1 -0
- package/dist/{environment-cPMCOk-h.js → environment-C0Gg8kyN.js} +49 -11
- package/dist/environment-C0Gg8kyN.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-C9b1h8L6.js → eval-source-CsW5VIXS.js} +24 -12
- package/dist/eval-source-CsW5VIXS.js.map +1 -0
- package/dist/{evaluate-B12Tosvq.js → evaluate-BHQyAszQ.js} +679 -142
- package/dist/evaluate-BHQyAszQ.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DzHq4EsU.d.ts} +2 -2
- package/dist/fake-bootstrap-VpisF1WE.js +15 -0
- package/dist/fake-bootstrap-VpisF1WE.js.map +1 -0
- package/dist/{files-tiNWe8Ih.js → files-BCraBodb.js} +3 -3
- package/dist/{files-tiNWe8Ih.js.map → files-BCraBodb.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/{files-BOBdC0uI.js → files-BuKWZCzY.js} +6 -6
- package/dist/{files-BOBdC0uI.js.map → files-BuKWZCzY.js.map} +1 -1
- package/dist/files-CAzVG3Rr.js +1 -0
- package/dist/{files-F5EG0WPT.js → files-Cv856jLN.js} +5 -5
- package/dist/{files-F5EG0WPT.js.map → files-Cv856jLN.js.map} +1 -1
- package/dist/files-D-Lir4aE.js +1 -0
- package/dist/files-DRlJ0Xr2.js +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/{files-s28a6czq.js → files-qjeqXnfE.js} +143 -20
- package/dist/files-qjeqXnfE.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-BpoqEgDV.js → git-DOrOtAiD.js} +3 -3
- package/dist/{git-BpoqEgDV.js.map → git-DOrOtAiD.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/harness-install-Dxho1yaU.js +53 -0
- package/dist/harness-install-Dxho1yaU.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-D2jFze2v.js → images-Ch1y1t6_.js} +122 -73
- package/dist/images-Ch1y1t6_.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/{init-Bi1Q3lRi.js → init-C2HqCwnt.js} +33 -14
- package/dist/init-C2HqCwnt.js.map +1 -0
- package/dist/init-CM9xwyre.js +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-BbKXElcQ.js} +54 -15
- package/dist/ink-list-BbKXElcQ.js.map +1 -0
- package/dist/inputs-5UIDpjz-.js +190 -0
- package/dist/inputs-5UIDpjz-.js.map +1 -0
- package/dist/inputs-load-C82HOKnb.js +1 -0
- package/dist/inputs-load-CGPyigMC.js +95 -0
- package/dist/inputs-load-CGPyigMC.js.map +1 -0
- package/dist/{invocations-CxGNOlh0.js → invocations-BrZAMuTb.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BrZAMuTb.js.map} +1 -1
- package/dist/jiti-radm1Soi.js +52 -0
- package/dist/jiti-radm1Soi.js.map +1 -0
- package/dist/{judge-COMXT1Hx.js → judge-DwzqucLS.js} +17 -12
- package/dist/{judge-COMXT1Hx.js.map → judge-DwzqucLS.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-Czh5wOpz.js} +14 -11
- package/dist/layout-Czh5wOpz.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-B_C0k5ex.js} +33 -19
- package/dist/{list-DuThsE28.js.map → list-B_C0k5ex.js.map} +1 -1
- package/dist/live-C9Et_BVT.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-DGgIrVw0.js} +7 -5
- package/dist/live-DGgIrVw0.js.map +1 -0
- package/dist/{live-DL0se4Zd.js → live-DgFm0H90.js} +9 -9
- package/dist/{live-DL0se4Zd.js.map → live-DgFm0H90.js.map} +1 -1
- package/dist/machine-service-2lIv5UWF.js +1 -0
- package/dist/machine-service-Ct_1XMMi.js +520 -0
- package/dist/machine-service-Ct_1XMMi.js.map +1 -0
- package/dist/machines-BnquHcBZ.js +120 -0
- package/dist/machines-BnquHcBZ.js.map +1 -0
- package/dist/{model-CkNVAxob.js → model-B1tdv9-d.js} +1 -1
- package/dist/model-Bcy4Qjmm.js +1 -0
- package/dist/{model-V3mMXp6x.js → model-DF77SgZv.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DF77SgZv.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-DRa7yv6K.js} +9 -6
- package/dist/model-DRa7yv6K.js.map +1 -0
- package/dist/{model-cgt5uvu8.js → model-E5X8YBJM.js} +7 -6
- package/dist/model-E5X8YBJM.js.map +1 -0
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{node-D7pAU--0.js → node-BjuhqQwa.js} +21 -10
- package/dist/node-BjuhqQwa.js.map +1 -0
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-B1sIOTD2.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-B1sIOTD2.js.map} +1 -1
- package/dist/options-C989kuVE.js +1 -0
- package/dist/own-reads-CYmBlrHK.js +15 -0
- package/dist/own-reads-CYmBlrHK.js.map +1 -0
- package/dist/{patch-DAi1nCuH.js → patch-BR8rdl7r.js} +2 -2
- package/dist/{patch-DAi1nCuH.js.map → patch-BR8rdl7r.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-CsPilzpk.js → plan-BcT4G5jj.js} +104 -15
- package/dist/plan-BcT4G5jj.js.map +1 -0
- package/dist/{plugins-BfnhazOv.js → plugins-cC0sqoDA.js} +608 -12
- package/dist/plugins-cC0sqoDA.js.map +1 -0
- package/dist/preflight-BMVDBcux.js +1 -0
- package/dist/{preflight-BixaE4BA.js → preflight-C9OtTb7k.js} +4 -4
- package/dist/{preflight-BixaE4BA.js.map → preflight-C9OtTb7k.js.map} +1 -1
- package/dist/{preflight-GkpgrBxP.js → preflight-DySHXeFJ.js} +9 -10
- package/dist/preflight-DySHXeFJ.js.map +1 -0
- package/dist/{preflight-D1C-iH0U.js → preflight-zsAn3heN.js} +1 -1
- package/dist/{prepare-C--rM_wO.js → prepare-MaRWRhNT.js} +2 -2
- package/dist/prepare-nbX1CXXg.js +409 -0
- package/dist/prepare-nbX1CXXg.js.map +1 -0
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-BXyYwUC8.js → provenance-CkykifD9.js} +5 -5
- package/dist/{provenance-BXyYwUC8.js.map → provenance-CkykifD9.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/providers-BkLGzaSY.js +1 -0
- package/dist/{providers-1p4HKYIp.js → providers-bNgfKa6m.js} +224 -49
- package/dist/providers-bNgfKa6m.js.map +1 -0
- package/dist/query-B2SQddoV.js +1 -0
- package/dist/{query-DhhU9ADI.js → query-DhV3a8lM.js} +2 -2
- package/dist/{query-DhhU9ADI.js.map → query-DhV3a8lM.js.map} +1 -1
- package/dist/{questions-CMAq4pr8.js → questions-Csv64WCb.js} +7 -7
- package/dist/{questions-CMAq4pr8.js.map → questions-Csv64WCb.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record--vHreTLk.js} +11 -5
- package/dist/record--vHreTLk.js.map +1 -0
- package/dist/{redact-BhjtnjrU.js → redact-t-L3VGy8.js} +106 -2
- package/dist/redact-t-L3VGy8.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/{remarks-BNABzemt.js → remarks-CYFOsnz5.js} +46 -11
- package/dist/remarks-CYFOsnz5.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-U7fNt0UP.js → remote-0CHog7Oq.js} +6 -6
- package/dist/{remote-U7fNt0UP.js.map → remote-0CHog7Oq.js.map} +1 -1
- package/dist/resolve-Bm9ZyFR-.js +1 -0
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/{resolve-Cx3a9UQ2.js → resolve-DorSHDBs.js} +14 -11
- package/dist/resolve-DorSHDBs.js.map +1 -0
- package/dist/resources-DqdwihhT.js +62 -0
- package/dist/resources-DqdwihhT.js.map +1 -0
- package/dist/{results-CQKXmHym.js → results-DhMgRkR0.js} +9 -22
- package/dist/results-DhMgRkR0.js.map +1 -0
- package/dist/{resume-POpZa1ge.js → resume-C8Ihfy-C.js} +166 -17
- package/dist/resume-C8Ihfy-C.js.map +1 -0
- package/dist/{retry-CmG809Mp.js → retry-Bfk7pDti.js} +4 -4
- package/dist/{retry-CmG809Mp.js.map → retry-Bfk7pDti.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/routes-Cp4gAjBZ.js +427 -0
- package/dist/routes-Cp4gAjBZ.js.map +1 -0
- package/dist/{runtime-Xm9oeC7o.js → runtime-CPUH0Vk5.js} +46 -5
- package/dist/runtime-CPUH0Vk5.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-DrlL4yqb.js → sandbox-z8q_4Ywn.js} +3 -3
- package/dist/{sandbox-DrlL4yqb.js.map → sandbox-z8q_4Ywn.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-C00zd6zl.js → server-BsQZtuF9.js} +9 -9
- package/dist/{server-C00zd6zl.js.map → server-BsQZtuF9.js.map} +1 -1
- package/dist/server-ZaMzUNmV.js +1 -0
- package/dist/services-BG8xuU82.js +39 -0
- package/dist/services-BG8xuU82.js.map +1 -0
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-DDgr48Gn.js → source-9HWaTEgV.js} +2 -2
- package/dist/{source-DDgr48Gn.js.map → source-9HWaTEgV.js.map} +1 -1
- package/dist/source-BcBDj-R0.js +1 -0
- package/dist/src-B2eHTCnp.js +3 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-Bb3k9Kn_.js} +208 -59
- package/dist/src-Bb3k9Kn_.js.map +1 -0
- package/dist/src-BfOlH_Bu.js +2 -0
- package/dist/src-BuGc40IF.js +2 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C2K5_afU.js +2 -0
- package/dist/{src-YvblOrBU.js → src-C3OhA5pw.js} +87 -13
- package/dist/src-C3OhA5pw.js.map +1 -0
- package/dist/src-CSVJjghq.js +4 -0
- package/dist/{src-y0nOeRRE.js → src-CWfy8ct8.js} +1572 -487
- package/dist/src-CWfy8ct8.js.map +1 -0
- package/dist/src-CcrQQB0_.js +2 -0
- package/dist/src-CotUxxQV.js +2 -0
- package/dist/src-D-PwBaBB.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-DNWiU__5.js} +276 -41
- package/dist/src-DNWiU__5.js.map +1 -0
- package/dist/{src-KxS91P7L.js → src-DwcTn4JY.js} +79 -11
- package/dist/src-DwcTn4JY.js.map +1 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/{src-DvGns8cz.js → src-c3ZQu7Uu.js} +299 -17
- package/dist/src-c3ZQu7Uu.js.map +1 -0
- package/dist/src-jYqNkT-F.js +1 -0
- package/dist/start-Db5VRcRZ.js +660 -0
- package/dist/start-Db5VRcRZ.js.map +1 -0
- package/dist/state-CKIjZZRK.js +1 -0
- package/dist/{state-328JpnHA.js → state-DUYdzNA7.js} +21 -21
- package/dist/{state-328JpnHA.js.map → state-DUYdzNA7.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/suite-CH1aaAbs.js +2 -0
- package/dist/{suite-COglUhPd.js → suite-DjGxmDSY.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-DjGxmDSY.js.map} +1 -1
- package/dist/{summary-DDy9DfyJ.js → summary-BK5Sa_N0.js} +28 -7
- package/dist/summary-BK5Sa_N0.js.map +1 -0
- package/dist/sync-DUi_P2UJ.js +559 -0
- package/dist/sync-DUi_P2UJ.js.map +1 -0
- package/dist/{tarball-CESxtK4X.js → tarball-C8BSTAMk.js} +2 -2
- package/dist/{tarball-CESxtK4X.js.map → tarball-C8BSTAMk.js.map} +1 -1
- package/dist/tart-BQ_Ap0l4.js +508 -0
- package/dist/tart-BQ_Ap0l4.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-D1RttYfM.js +148 -0
- package/dist/tart-bake-D1RttYfM.js.map +1 -0
- package/dist/tart-bake-DjFcsG3N.js +1 -0
- package/dist/tart-cli-BOcluha1.js +1 -0
- package/dist/tart-cli-gA9xEaQL.js +89 -0
- package/dist/tart-cli-gA9xEaQL.js.map +1 -0
- package/dist/tart-hosts-CHNrfV6u.js +313 -0
- package/dist/tart-hosts-CHNrfV6u.js.map +1 -0
- package/dist/tart-hosts-CWse-ASo.js +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CLBjfF9u.js} +64 -6
- package/dist/tart-images-CLBjfF9u.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-krsmEo44.js +433 -0
- package/dist/tool-class-krsmEo44.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BXWZiMuC.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BXWZiMuC.js.map} +1 -1
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{types-DBwFsGem.d.ts → types-DKK8jj--.d.ts} +517 -45
- package/dist/ui-DKYjTVpn.js +1455 -0
- package/dist/ui-DKYjTVpn.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/vercel-models-Ci33llXz.js +21 -0
- package/dist/vercel-models-Ci33llXz.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +32 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-BROacjBv.js → views-B1zVLJsF.js} +6 -6
- package/dist/{views-BROacjBv.js.map → views-B1zVLJsF.js.map} +1 -1
- package/dist/views-DCjtw1jF.js +2 -0
- package/dist/{worker-entry-Smb4bNDc.js → worker-entry-DrAYYt0M.js} +37 -14
- package/dist/worker-entry-DrAYYt0M.js.map +1 -0
- package/dist/workspace-CnRbV8cM.js +1 -0
- package/dist/{workspace-DNDTEk8v.js → workspace-DwTe1zo6.js} +7 -7
- package/dist/{workspace-DNDTEk8v.js.map → workspace-DwTe1zo6.js.map} +1 -1
- package/docker/Dockerfile +1 -1
- package/package.json +15 -1
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C60z3wLq.js.map +0 -1
- package/dist/agent-BmsMXZXx.js.map +0 -1
- package/dist/agents-DXuFxtGC.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-DM2PNU4R.js.map +0 -1
- package/dist/attempt-host-7VviAJrl.js.map +0 -1
- package/dist/boat-BjLqFJIZ.js.map +0 -1
- package/dist/boat-DSNgZt-t.js +0 -322
- package/dist/boat-DSNgZt-t.js.map +0 -1
- package/dist/capture-BzotD2hs.js +0 -1
- package/dist/capture-Ccf0Hmyx.js.map +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-DTuLSlmW.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-DSL7I-SI.js.map +0 -1
- package/dist/command-Ff0T-1Hi.js.map +0 -1
- package/dist/commands-xAOi_VEk.js.map +0 -1
- package/dist/config-SvK8pjEK.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-BzOJ4Js4.js.map +0 -1
- package/dist/db-CQRZ3URX.js +0 -2
- package/dist/docker--mukta3L.js +0 -1
- package/dist/docker-DG_uFgn2.js +0 -246
- package/dist/docker-DG_uFgn2.js.map +0 -1
- package/dist/doctor-DWK4ttex.js.map +0 -1
- package/dist/environment-cPMCOk-h.js.map +0 -1
- package/dist/eval-source-C9b1h8L6.js.map +0 -1
- package/dist/evaluate-B12Tosvq.js.map +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-DMQ9-Vv5.js +0 -1
- package/dist/files-Di0xDbCc.js +0 -1
- package/dist/files-G5jf8wBp.js +0 -1
- package/dist/files-s28a6czq.js.map +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-D2jFze2v.js.map +0 -1
- package/dist/init-Bi1Q3lRi.js.map +0 -1
- package/dist/init-CUw65o36.js +0 -1
- package/dist/ink-list-BpfidQ1h.js.map +0 -1
- package/dist/kinds-Bh2I35id.js +0 -698
- package/dist/kinds-Bh2I35id.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CgYbO4Te.js.map +0 -1
- package/dist/live-DZLmj0iG.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/model-cgt5uvu8.js.map +0 -1
- package/dist/node-D7pAU--0.js.map +0 -1
- package/dist/options-DPUmk7v_.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-CsPilzpk.js.map +0 -1
- package/dist/plugins-BfnhazOv.js.map +0 -1
- package/dist/preflight-Bn3ianye.js +0 -1
- package/dist/preflight-GkpgrBxP.js.map +0 -1
- package/dist/prepare-DQEDXoDp.js +0 -268
- package/dist/prepare-DQEDXoDp.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-1p4HKYIp.js.map +0 -1
- package/dist/providers-B-ijCktf.js +0 -1
- package/dist/query-BYhyf0LR.js +0 -1
- package/dist/record-6IUzwP9K.js.map +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-BNABzemt.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/resolve-C14Pmtej.js +0 -1
- package/dist/resolve-Cx3a9UQ2.js.map +0 -1
- package/dist/results-CQKXmHym.js.map +0 -1
- package/dist/resume-POpZa1ge.js.map +0 -1
- package/dist/runtime-Xm9oeC7o.js.map +0 -1
- package/dist/server-C1UJcpUh.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-DJHmQv_j.js +0 -1
- package/dist/src-1dBuXzs4.js +0 -3
- package/dist/src-BKpioayC.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BY1cCaoJ.js +0 -2
- package/dist/src-BbZYaQ8b.js +0 -2
- package/dist/src-CDFH2vVN.js +0 -2
- package/dist/src-CMW5R9Q0.js +0 -1
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-CrV2WA2i.js +0 -2
- package/dist/src-CsFaawcZ.js +0 -2
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-wKwznPgt.js +0 -2
- package/dist/src-y0nOeRRE.js.map +0 -1
- package/dist/state-Ca55Gk_O.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/summary-DDy9DfyJ.js.map +0 -1
- package/dist/tart-cli-C-GsbVsQ.js +0 -1
- package/dist/tart-cli-CU0nFS-L.js +0 -55
- package/dist/tart-cli-CU0nFS-L.js.map +0 -1
- package/dist/tart-dWoJvWHc.js +0 -586
- package/dist/tart-dWoJvWHc.js.map +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js +0 -344
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-B0Yt_Q1S.js +0 -2
- package/dist/worker-entry-Smb4bNDc.js.map +0 -1
- package/dist/workspace-C0dDP75l.js +0 -1
package/README.md
CHANGED
|
@@ -27,11 +27,11 @@ Evals are TypeScript files in `evals/`, next to the tool they test. Each run wri
|
|
|
27
27
|
|
|
28
28
|
Cursor ⋅ Claude Code ⋅ Codex ⋅ Antigravity ⋅ OpenCode ⋅ Goose ⋅ pi ⋅ Grok Build
|
|
29
29
|
|
|
30
|
-
- **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](docs/configuration/agents/index.mdx)).
|
|
31
|
-
- **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](docs/configuration/environments/secrets.mdx)).
|
|
32
|
-
- **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](docs/results/compare/lift.mdx)).
|
|
33
|
-
- **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](docs/configuration/environments/index.mdx)).
|
|
34
|
-
- **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](docs/evaluations/runs/recording.mdx)).
|
|
30
|
+
- **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](https://github.com/invertase/reeval/blob/v0.22.0/docs/configuration/agents/index.mdx)).
|
|
31
|
+
- **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](https://github.com/invertase/reeval/blob/v0.22.0/docs/configuration/environments/secrets.mdx)).
|
|
32
|
+
- **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](https://github.com/invertase/reeval/blob/v0.22.0/docs/results/compare/lift.mdx)).
|
|
33
|
+
- **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](https://github.com/invertase/reeval/blob/v0.22.0/docs/configuration/environments/index.mdx)).
|
|
34
|
+
- **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/runs/recording.mdx)).
|
|
35
35
|
|
|
36
36
|
## Install
|
|
37
37
|
|
|
@@ -140,23 +140,23 @@ Environments load extensions from where teams already keep them:
|
|
|
140
140
|
|
|
141
141
|
## Examples
|
|
142
142
|
|
|
143
|
-
The recipes in [`docs/evaluations/recipes`](docs/evaluations/recipes) are complete evals for common jobs:
|
|
143
|
+
The recipes in [`docs/evaluations/recipes`](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes) are complete evals for common jobs:
|
|
144
144
|
|
|
145
|
-
- [Measure a skill](docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
|
|
146
|
-
- [Compare harnesses](docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
|
|
147
|
-
- [Measure MCP servers](docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
|
|
148
|
-
- [Check a recommendation](docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
|
|
149
|
-
- [Fail a pull request](docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
|
|
145
|
+
- [Measure a skill](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
|
|
146
|
+
- [Compare harnesses](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
|
|
147
|
+
- [Measure MCP servers](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
|
|
148
|
+
- [Check a recommendation](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
|
|
149
|
+
- [Fail a pull request](https://github.com/invertase/reeval/blob/v0.22.0/docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
|
|
150
150
|
|
|
151
|
-
The [`examples/`](examples) directory has 80 runnable projects, one idea each, from [`with-checks`](examples/with-checks) (deterministic checks) and [`with-skill-lift`](examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](examples/README.md) lists them all.
|
|
151
|
+
The [`examples/`](https://github.com/invertase/reeval/blob/v0.22.0/examples) directory has 80 runnable projects, one idea each, from [`with-checks`](https://github.com/invertase/reeval/blob/v0.22.0/examples/with-checks) (deterministic checks) and [`with-skill-lift`](https://github.com/invertase/reeval/blob/v0.22.0/examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](https://github.com/invertase/reeval/blob/v0.22.0/examples/README.md) lists them all.
|
|
152
152
|
|
|
153
153
|
## Documentation
|
|
154
154
|
|
|
155
|
-
The manual in [`docs/`](docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
|
|
155
|
+
The manual in [`docs/`](https://github.com/invertase/reeval/blob/v0.22.0/docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
|
|
156
156
|
|
|
157
157
|
## For coding agents
|
|
158
158
|
|
|
159
|
-
To write or update evals, load [`skills/reeval/SKILL.md`](skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
|
|
159
|
+
To write or update evals, load [`skills/reeval/SKILL.md`](https://github.com/invertase/reeval/blob/v0.22.0/skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
|
|
160
160
|
|
|
161
161
|
To preview a run without starting sandboxes or spending tokens, run `npx reeval run --dry-run --json`. It prints the planned evals, cells, and attempts as one JSON document and exits 0.
|
|
162
162
|
|
|
@@ -166,4 +166,4 @@ To read past runs without calling a model, run `npx reeval query failing_attempt
|
|
|
166
166
|
|
|
167
167
|
## License
|
|
168
168
|
|
|
169
|
-
[Apache-2.0](LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
|
|
169
|
+
[Apache-2.0](https://github.com/invertase/reeval/blob/v0.22.0/LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
|
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
// Writes pi's models.json for a
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// Writes pi's models.json for a model on an AI gateway, run in the sandbox before each turn, with
|
|
2
|
+
// the model in a provider of reeval's, on the API its provider's route names (`gateway.routes`),
|
|
3
|
+
// else Chat Completions. Its thinking levels, limits and prices are pi's catalog entry for it, on
|
|
4
|
+
// the gateway or its provider, else the host's description of it. A route with `thinkingConfig`
|
|
5
|
+
// sends each thinking level as the model's native `thinkingConfig`, so the model gets the level pi
|
|
6
|
+
// sends it there.
|
|
5
7
|
//
|
|
6
|
-
// node gateway-models.mjs <bootstrap dir> <models.json> <
|
|
7
|
-
import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
|
|
8
|
+
// node gateway-models.mjs <bootstrap dir> <models.json> <gateway json> <model> <model info json>
|
|
9
|
+
import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
|
|
8
10
|
import { dirname, join } from 'node:path'
|
|
9
11
|
|
|
10
|
-
const [bootstrapDir, out,
|
|
11
|
-
const
|
|
12
|
+
const [bootstrapDir, out, gatewayJson, model, infoJson] = process.argv.slice(2)
|
|
13
|
+
const gateway = JSON.parse(gatewayJson)
|
|
14
|
+
const info = JSON.parse(infoJson ?? 'null')
|
|
12
15
|
const fail = (why) => {
|
|
13
16
|
process.stderr.write(`reeval: ${why}\n`)
|
|
14
17
|
process.exit(1)
|
|
@@ -16,52 +19,83 @@ const fail = (why) => {
|
|
|
16
19
|
|
|
17
20
|
// pnpm puts pi's own dependencies beside it, so pi-ai is pi-coding-agent's sibling.
|
|
18
21
|
const agent = realpathSync(join(bootstrapDir, 'node_modules/@earendil-works/pi-coding-agent'))
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
)
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
`${id} takes a thinking budget, not a thinking level, so pi can't run it on ${provider.name}`,
|
|
22
|
+
const data = join(dirname(agent), 'pi-ai/dist/providers/data')
|
|
23
|
+
/** The chat models of one of pi's catalogs, whatever API each is on. */
|
|
24
|
+
const catalog = (name) => {
|
|
25
|
+
const file = join(data, `${name}.json`)
|
|
26
|
+
if (!/^[\w-]+$/.test(name) || !existsSync(file)) return []
|
|
27
|
+
return Object.values(JSON.parse(readFileSync(file, 'utf8'))).flatMap((models) =>
|
|
28
|
+
Object.entries(models)
|
|
29
|
+
.filter(([key]) => key.startsWith('chat:'))
|
|
30
|
+
.map(([, m]) => m),
|
|
29
31
|
)
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
const slash = model.indexOf('/')
|
|
35
|
+
const provider = slash < 0 ? '' : model.slice(0, slash)
|
|
36
|
+
const id = model.slice(slash + 1)
|
|
37
|
+
const route = gateway.routes[provider] ?? {}
|
|
38
|
+
const native = route.thinkingConfig
|
|
39
|
+
|
|
40
|
+
let entry
|
|
41
|
+
if (native) {
|
|
42
|
+
entry = catalog(native.catalog).find((m) => m.id === id)
|
|
43
|
+
if (!entry)
|
|
44
|
+
fail(
|
|
45
|
+
`pi's ${native.catalog} catalog has no model ${id}, so pi can't run ${model} on ${gateway.name}`,
|
|
46
|
+
)
|
|
47
|
+
if (entry.reasoning && !entry.thinkingLevelMap) {
|
|
48
|
+
fail(
|
|
49
|
+
`${id} takes a thinking budget, not a thinking level, so pi can't run it on ${gateway.name}`,
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
// A provider's own catalog names a version with a dash where a gateway dots it (claude-haiku-4-5).
|
|
54
|
+
const dashed = id.replace(/(\d)\.(\d)/g, '$1-$2')
|
|
55
|
+
entry =
|
|
56
|
+
catalog(gateway.catalog).find((m) => m.id === model) ??
|
|
57
|
+
catalog(provider).find((m) => m.id === id || m.id === dashed) ??
|
|
58
|
+
info ??
|
|
59
|
+
fail(
|
|
60
|
+
`neither pi's catalog nor models.dev describes ${model}, so pi can't run it on ${gateway.name}`,
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
|
|
32
64
|
const thinking = (level) => ({ thinkingConfig: { thinkingLevel: level, includeThoughts: true } })
|
|
33
65
|
const byLevel = {}
|
|
34
|
-
for (const level of entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
|
|
66
|
+
for (const level of native && entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
|
|
35
67
|
const mapped = entry.thinkingLevelMap[level]
|
|
36
68
|
if (mapped === null) continue
|
|
37
|
-
const
|
|
69
|
+
const sent = (typeof mapped === 'string' ? mapped : level).toLowerCase()
|
|
38
70
|
byLevel[level] = {
|
|
39
|
-
providerOptions: Object.fromEntries(
|
|
71
|
+
providerOptions: Object.fromEntries(native.upstreams.map((u) => [u, thinking(sent)])),
|
|
40
72
|
}
|
|
41
73
|
}
|
|
42
74
|
|
|
75
|
+
const api = route.api ?? 'openai-completions'
|
|
43
76
|
const models = {
|
|
44
77
|
providers: {
|
|
45
|
-
[
|
|
46
|
-
name:
|
|
47
|
-
baseUrl:
|
|
48
|
-
api
|
|
49
|
-
apiKey: `$${
|
|
50
|
-
|
|
51
|
-
// would map Gemini's levels onto its own.
|
|
78
|
+
[gateway.id]: {
|
|
79
|
+
name: gateway.name,
|
|
80
|
+
baseUrl: api === 'anthropic-messages' ? gateway.anthropic : gateway.openai,
|
|
81
|
+
api,
|
|
82
|
+
apiKey: `$${gateway.credentialEnv}`,
|
|
83
|
+
authHeader: true,
|
|
52
84
|
compat: {
|
|
53
|
-
|
|
85
|
+
// The level goes as the thinkingConfig above; the gateway's shared effort field would map
|
|
86
|
+
// the model's levels onto its own.
|
|
87
|
+
...(native ? { supportsReasoningEffort: false } : {}),
|
|
54
88
|
supportsStore: false,
|
|
55
89
|
supportsDeveloperRole: false,
|
|
56
90
|
},
|
|
57
91
|
models: [
|
|
58
92
|
{
|
|
59
93
|
id: model,
|
|
60
|
-
name: entry.name,
|
|
94
|
+
...(entry.name ? { name: entry.name } : {}),
|
|
61
95
|
reasoning: entry.reasoning,
|
|
62
96
|
...(entry.thinkingLevelMap ? { thinkingLevelMap: entry.thinkingLevelMap } : {}),
|
|
63
97
|
input: entry.input,
|
|
64
|
-
cost: entry.cost,
|
|
98
|
+
...(entry.cost ? { cost: entry.cost } : {}),
|
|
65
99
|
contextWindow: entry.contextWindow,
|
|
66
100
|
maxTokens: entry.maxTokens,
|
|
67
101
|
...(Object.keys(byLevel).length ? { samplingParamsByThinkingLevel: byLevel } : {}),
|
|
@@ -1,54 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { t as
|
|
4
|
-
import {
|
|
1
|
+
import { l as ownFile } from "./root-BzeoyfGd.js";
|
|
2
|
+
import { c as relativeFile } from "./ids-B-JKTCP8.js";
|
|
3
|
+
import { t as reevalJiti } from "./jiti-radm1Soi.js";
|
|
4
|
+
import { t as codeFrame } from "./errors-C1GBcWE2.js";
|
|
5
5
|
import { existsSync, readFileSync } from "node:fs";
|
|
6
6
|
import { delimiter, dirname, join, resolve } from "node:path";
|
|
7
|
-
import { stripVTControlCharacters } from "node:util";
|
|
8
7
|
import { spawnSync } from "node:child_process";
|
|
9
|
-
import {
|
|
8
|
+
import { stripVTControlCharacters } from "node:util";
|
|
10
9
|
import { DecodedMap, getOriginalPosition } from "@vitest/utils/source-map";
|
|
11
10
|
import { getTsconfig } from "get-tsconfig";
|
|
12
|
-
// src/jiti.ts
|
|
13
|
-
let babel;
|
|
14
|
-
/**
|
|
15
|
-
* jiti's own Babel transform. jiti requires it lazily from beside its code, which a compiled reeval
|
|
16
|
-
* doesn't have, so reeval requires the same file: from the jiti package on disk, or, compiled, from
|
|
17
|
-
* the copy embedded in it (`vendor/jiti/babel.cjs`), copied out to disk.
|
|
18
|
-
*/
|
|
19
|
-
const jitiTransform = (options) => {
|
|
20
|
-
if (!babel) {
|
|
21
|
-
const file = COMPILED ? join(ownDir("vendor/jiti"), "babel.cjs") : join(dirname(ownRequire.resolve("jiti/package.json")), "dist/babel.cjs");
|
|
22
|
-
babel = createRequire(file)(file);
|
|
23
|
-
}
|
|
24
|
-
return babel(options);
|
|
25
|
-
};
|
|
26
|
-
/**
|
|
27
|
-
* A jiti that imports from `id`. Compiled, a file's imports of reeval itself get this reeval's own
|
|
28
|
-
* modules: a copy on disk would register evals, and hold state, the executable can't see.
|
|
29
|
-
*/
|
|
30
|
-
async function reevalJiti(id, options = {}) {
|
|
31
|
-
const virtualModules = COMPILED ? await ownModules() : void 0;
|
|
32
|
-
return createJiti(id, {
|
|
33
|
-
transform: jitiTransform,
|
|
34
|
-
...options,
|
|
35
|
-
...virtualModules && { virtualModules }
|
|
36
|
-
});
|
|
37
|
-
}
|
|
38
|
-
async function ownModules() {
|
|
39
|
-
const exports = {
|
|
40
|
-
"": import("./index.js"),
|
|
41
|
-
"/harness": import("./harness.js"),
|
|
42
|
-
"/sandbox": import("./sandbox.js"),
|
|
43
|
-
"/sandbox/boat": import("./sandbox/boat.js"),
|
|
44
|
-
"/judge": import("./judge.js"),
|
|
45
|
-
"/gateway": import("./gateway.js"),
|
|
46
|
-
"/view": import("./view/index.js")
|
|
47
|
-
};
|
|
48
|
-
const modules = {};
|
|
49
|
-
for (const [subpath, module] of Object.entries(exports)) for (const name of PACKAGE_NAMES) modules[name + subpath] = await module;
|
|
50
|
-
return modules;
|
|
51
|
-
}
|
|
52
11
|
// src/engine/loader.ts
|
|
53
12
|
var TsconfigError = class extends Error {
|
|
54
13
|
name = "TsconfigError";
|
|
@@ -288,7 +247,7 @@ const LANGUAGE_ADAPTERS = [TYPESCRIPT_ADAPTER, {
|
|
|
288
247
|
extensions: [".eval.py"],
|
|
289
248
|
includeGlobs: ["evals/**/*.eval.py"],
|
|
290
249
|
remote: true,
|
|
291
|
-
load: (file, root) => import("./remote-
|
|
250
|
+
load: (file, root) => import("./remote-0CHog7Oq.js").then((m) => m.collectLanguageFile(file, root)),
|
|
292
251
|
runnerCommand: (root) => resolvePythonRunnerCommand(root),
|
|
293
252
|
setupHint: PYTHON_RUNNER_HINT
|
|
294
253
|
}];
|
|
@@ -312,6 +271,6 @@ function runnerEnv(root, adapter) {
|
|
|
312
271
|
REEVAL_PROJECT_ROOT: resolve(root)
|
|
313
272
|
};
|
|
314
273
|
}
|
|
315
|
-
export { originalText as a, originalStack as i, defaultIncludeGlobs as n,
|
|
274
|
+
export { originalText as a, originalStack as i, defaultIncludeGlobs as n, runnerEnv as r, adapterForFile as t };
|
|
316
275
|
|
|
317
|
-
//# sourceMappingURL=adapter-
|
|
276
|
+
//# sourceMappingURL=adapter-Di9H2etg.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter-Di9H2etg.js","names":[],"sources":["../src/engine/loader.ts","../src/language-runner/python-runner.ts","../src/language-runner/adapter.ts"],"mappings":";;;;;;;;;;;AAYA,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAgB;AAClB;AAOA,MAAM,8BAAc,IAAI,IAAyB;AAEjD,MAAM,aACJ;;AAGF,eAAsB,aAAa,MAAc,MAAgC;CAC/E,cAAc,QAAQ,IAAI,GAAG,IAAI;CACjC,MAAM,UAAU;EAAE,aAAa;EAAM,SAAS;EAAO,YAAY;CAAK;CACtE,MAAM,OAAO,MAAM,WAAW,MAAM,OAAO;CAC3C,MAAM,OAAO,MAAM,WAAW,MAAM;EAClC,GAAG;EACH,YAAY,SAAS;GACnB,MAAM,OAAO,KAAK,UAAU,IAAI;GAChC,MAAM,SAAS,WAAW,KAAK,IAAI;GACnC,IAAI,KAAK,YAAY,SAAS,IAAI;IAChC,MAAM,MAAM,KAAK,MAAM,OAAO,KAAK,OAAO,IAAI,QAAQ,CAAC,CAAC,SAAS,MAAM,CAAC;IAExE,MAAM,QACJ,IAAI,KAAK,QAAQ,WAAW,GAAG,2FAC5B;IACL,YAAY,IAAI,KAAK,UAAU;KAC7B,KAAK,IAAI,WAAW,KAAK,KAAK,QAAQ;KACtC;KACA,QAAQ,KAAK;KACb;IACF,CAAC;GACH;GACA,OAAO,EAAE,KAAK;EAChB;CACF,CAAC;CACD,IAAI;EACF,OAAO,MAAM,KAAK,OAAO,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,YAAY,OAAO,IAAI,KAAK,cAAc,KAAK;CACvD;AACF;;AAGA,SAAS,cAAc,OAAyB;CAC9C,MAAM,IAAI;CACV,IAAI,GAAG,SAAS,oBAAoB,OAAO;CAC3C,MAAM,QAAQ,wDAAwD,KAAK,EAAE,OAAO;CACpF,IAAI,OAAO,EAAE,UAAU,GAAG,MAAM,GAAG,iBAAiB,MAAM;CAC1D,OAAO;AACT;;;;;AAMA,SAAgB,YAAY,OAAgB,MAAuC;CACjF,MAAM,SAAS,8CAA8C,KAAM,OAAiB,WAAW,EAAE;CACjG,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,GAAG,MAAM,MAAM,MAAM,UAAU;CAErC,MAAM,SAAS,KAAK,QAAQ,8BAA8B,EAAE;CAC5D,MAAM,KAAK;EAAE,MAAM,OAAO,IAAI;EAAG,QAAQ,OAAO,MAAM,IAAI;CAAE;CAC5D,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,yBAAyB,UAAU,aAAa,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;CACpF,QAAQ,CAAC;CACT,uBAAO,IAAI,YACT,GAAG,OAAO,IAAI,aAAa,MAAM,IAAI,EAAE,GAAG,GAAG,KAAK,GAAG,GAAG,OAAO,GAAG,QAAQ,KAAK,UAAU,IAC3F;AACF;;AAGA,SAAgB,cAAc,OAAuB;CACnD,OAAO,MAAM,QACX,gDACC,OAAO,QAAgB,MAAc,MAAc,QAAgB,UAAkB;EACpF,MAAM,SAAS,YAAY,IAAI,IAAI;EACnC,MAAM,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS,MAAO,QAAQ,SAAS,IAAK;EAExE,IAAI,CAAC,UAAU,MAAM,GAAG,OAAO;EAC/B,MAAM,KAAK,oBAAoB,OAAO,KAAK;GAAE,MAAM,OAAO,IAAI;GAAG,QAAQ;EAAI,CAAC;EAC9E,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,WAAW,GAAG;EACpB,OAAO,GAAG,SAAS,KAAK,GAAG,SAAS,GAAG,OAAO,MAAM,UAAU,GAAG,MAAgB,IAAI,IAAI;CAC3F,CACF;AACF;;;;;;;AAQA,SAAgB,aAAa,MAAc,MAAmC;CAC5E,MAAM,YAAY,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;CAClE,KAAK,MAAM,UAAU,CAAC,GAAG,WAAW,GAAG,YAAY,OAAO,CAAC,GAAG;EAC5D,MAAM,KAAK,QAAQ,KAAK,QAAQ,IAAI,KAAK;EACzC,IAAI,CAAC,UAAU,KAAK,GAAG;EAGvB,MAAM,OAAO,KAAK,KAAK,SAAS;EAChC,IAAI,aAAa;EACjB,MAAM,MAAM,eAAe,QAAQ,IAAI;EACvC,OAAO,aAAa,MAAM,eAAe,QAAQ,aAAa,CAAC,MAAM,KAAK;EAC1E,OAAO,OAAO,OAAO,MAAM,eAAe,QAAQ,EAAE,GAAG,MAAM,OAAO,aAAa,CAAC;CACpF;AAEF;;AAGA,SAAS,eAAe,QAAqB,OAAuB;CAClE,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,IAAI;CACpD,MAAM,SAAU,MAAM,GAAG,EAAE,CAAC,CAAY;CACxC,MAAM,KAAK,oBAAoB,OAAO,KAAK;EAAE,MAAM,MAAM;EAAQ;CAAO,CAAC;CAGzE,OADoB,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,QAAQ,KAAK,CACxD,CAAC,CAAC,QAAQ,GAAG,SAAS,IAAI,KAAK,SAAS,GAAG,CAAC,KAAK,IAAI,UAAU;AAClF;;;;;AAMA,SAAS,OAAO,MAAc,MAAc,QAAwB;CAElE,MAAM,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,MAAM;CACpE,IAAI,KAAK,YAAY,KAAK,OAAO;CACjC,IAAI,QAAQ;CACZ,OAAO,QAAQ,KAAK,QAAQ,KAAK,KAAK,QAAQ,EAAY,GAAG;CAC7D,OAAO;AACT;;AAGA,SAAgB,cAAc,KAAa,MAAoB;CAC7D,IAAI;CACJ,IAAI;EACF,QAAQ,YAAY,GAAG;CACzB,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB;EACjC,MAAM,UAAU,4BAA4B,KAAK,OAAO;EACxD,MAAM,IAAI,cACR,UACI,4BAA4B,QAAQ,GAAG,yBACvC,4BAA4B,SAClC;CACF;CACA,IAAI,CAAC,OAAO;CACZ,IAAI;EAEF,KAAK,MAAM,kBAAkB,aAAa,MAAM,MAAM,MAAM,CAAC,CAAC,QAAQ,WAAW,EAAE,CAAC,CAAC;CACvF,SAAS,OAAO;EACd,MAAM,IAAI,cACR,4BAA4B,aAAa,MAAM,MAAM,IAAI,EAAE,KAAM,MAAgB,SACnF;CACF;AACF;;AAGA,SAAgB,kBAAkB,MAAsB;CACtD,OAAO,KACL,KAAK,OAAO,SAAS;EACnB,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,IAAI;GAC7B,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,KAAK,EAAE;EACzC;EACA,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,MAAM,CAAC;GAChC,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,MAAM,GAAG,GAAG;EAC9C;CAEF,CAAC,IACA,SAAU,aAAa,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,KAAA,CAClD;AACF;;AAGA,SAAS,KAAK,MAAc,IAA4D;CACtF,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,SAAU;EACjC,IAAI,KAAK,OAAO,MAAK;GACnB,IAAI,MAAM,IAAI;GACd,OAAO,MAAM,KAAK,UAAU,KAAK,SAAS,MAAK,OAAO,KAAK,SAAS,OAAO,IAAI;GAC/E,OAAO,KAAK,MAAM,GAAG,MAAM,CAAC;GAC5B,IAAI,MAAM;GACV;EACF;EACA,MAAM,MAAM,GAAG,KAAK,MAAM,CAAC,CAAC;EAC5B,IAAI,KAAK;GACP,OAAO,IAAI;GACX,KAAK,IAAI;EACX,OAAO,OAAO,KAAK;CACrB;CACA,OAAO;AACT;;AC7MA,MAAa,uBAAuB;AAEpC,MAAM,cAAc,CAAC,MAAM,oBAAoB;AAE/C,IAAI;AAEJ,SAAgB,cAAuB;CACrC,IAAI,aAAa,KAAA,GAAW,OAAO;CAMnC,WALe,UAAU,MAAM,CAAC,WAAW,GAAG;EAC5C,UAAU;EACV,OAAO;EACP,aAAa;CACf,CACgB,CAAC,CAAC,WAAW;CAC7B,OAAO;AACT;AAEA,SAAgB,cAAc,MAAuB;CACnD,OAAO,WAAW,KAAK,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,gBAAgB,CAAC;AACrF;AAEA,SAAS,WAAW,MAAkC;CACpD,MAAM,OAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;CAChD,IAAI,WAAW,IAAI,GAAG,OAAO;CAC7B,MAAM,MAAM,KAAK,MAAM,SAAS,WAAW,YAAY;CACvD,IAAI,WAAW,GAAG,GAAG,OAAO;AAE9B;;AAGA,SAAgB,2BAA2B,MAA6B;CACtE,MAAM,WAAW,QAAQ,IAAI,eAAe,KAAK;CACjD,IAAI,UAAU,OAAO,CAAC,UAAU,GAAG,WAAW;CAE9C,IAAI,cAAc,IAAI,KAAK,YAAY,GACrC,OAAO;EAAC;EAAM;EAAO;EAAa;EAAM;EAAU,GAAG;CAAW;CAGlE,MAAM,OAAO,WAAW,IAAI;CAC5B,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,WAAW;CAGtC,OAAO,CAAC,QAAQ,aAAa,UAAU,WAAW,WAAW,GAAG,WAAW;AAC7E;;AAGA,MAAa,qBAAqB;CAChC,kCAAkC,qBAAqB;CACvD;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;;AC7BV,MAAa,qBAAuC;CAClD,IAAI;CACJ,YAAY;EAAC;EAAY;EAAa;EAAa;EAAY;EAAa;CAAW;CACvF,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,aAAa,MAAM,IAAI;AAC/C;AAEA,MAAM,sBAA0C;CAC9C,MAAM,UAAU,KAAK,QAAQ,IAAI,GAAG,eAAe;CACnD,IAAI,WAAW,KAAK,SAAS,gBAAgB,CAAC,GAAG,OAAO;AAE1D;;AAaA,MAAa,oBAAgD,CAAC,oBAAoB;CAVhF,IAAI;CACJ,YAAY,CAAC,UAAU;CACvB,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,OAAO,uBAAc,CAAC,MAAM,MAAM,EAAE,oBAAoB,MAAM,IAAI,CAAC;CACzF,gBAAgB,SAAS,2BAA2B,IAAI;CACxD,WAAW;AAIqE,CAAc;AAEhG,SAAgB,eAAe,MAA+B;CAE5D,OADc,kBAAkB,MAAM,MAAM,EAAE,WAAW,MAAM,QAAQ,KAAK,SAAS,GAAG,CAAC,CAC9E,KAAK;AAClB;AAEA,SAAgB,sBAAgC;CAC9C,OAAO,kBAAkB,SAAS,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC;AAC7D;AAEA,SAAgB,UAAU,MAAc,SAA6C;CACnF,MAAM,MAAM,QAAQ,IAAI,qBAAqB,cAAc;CAC3D,IAAI,QAAQ,OAAO,YAAY,CAAC,KAAK,OAAO,QAAQ;CACpD,MAAM,MAAM,KAAK,KAAK,KAAK;CAC3B,OAAO;EACL,GAAG,QAAQ;EACX,YAAY;GAAC;GAAK;GAAK,QAAQ,IAAI;EAAU,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,SAAS;EAC7E,qBAAqB,QAAQ,IAAI;CACnC;AACF"}
|