reeval 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +803 -0
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/assets/gateway-models.mjs +65 -31
- package/dist/{adapter-C8xPuXYq.js → adapter-BdzhdRYo.js} +8 -6
- package/dist/adapter-BdzhdRYo.js.map +1 -0
- package/dist/{agent-CBoLndM7.js → agent-Dx-bDmlV.js} +179 -31
- package/dist/agent-Dx-bDmlV.js.map +1 -0
- package/dist/{agents-C6zhkxp3.js → agents-l0-k7vw3.js} +362 -102
- package/dist/agents-l0-k7vw3.js.map +1 -0
- package/dist/{app-files-DT5mJyV4.js → app-files-B-CQYB0c.js} +3 -3
- package/dist/app-files-B-CQYB0c.js.map +1 -0
- package/dist/{assemble-UPCRrtOz.js → assemble-CYMpN8LC.js} +1 -1
- package/dist/{assemble-t6h018An.js → assemble-CsZIcJo8.js} +171 -18
- package/dist/assemble-CsZIcJo8.js.map +1 -0
- package/dist/{attempt-CAdlwlNu.js → attempt-CKfKsqqw.js} +2 -2
- package/dist/{attempt-CAdlwlNu.js.map → attempt-CKfKsqqw.js.map} +1 -1
- package/dist/{attempt-host-CIm1-11m.js → attempt-host-BwSmNYkX.js} +17 -8
- package/dist/attempt-host-BwSmNYkX.js.map +1 -0
- package/dist/{boat-BjLqFJIZ.js → boat-CZpAreTB.js} +3 -3
- package/dist/{boat-BjLqFJIZ.js.map → boat-CZpAreTB.js.map} +1 -1
- package/dist/{boat-CJCtzfB1.js → boat-su1Y7LPd.js} +17 -14
- package/dist/boat-su1Y7LPd.js.map +1 -0
- package/dist/bridge/index.mjs +113 -8
- package/dist/{capture-iBzFiC0j.js → capture-BqIAeFyC.js} +55 -29
- package/dist/{capture-iBzFiC0j.js.map → capture-BqIAeFyC.js.map} +1 -1
- package/dist/capture-D0Ln5aCP.js +1 -0
- package/dist/{cases-z9E6DCd5.js → cases-CM-MQRQp.js} +133 -19
- package/dist/cases-CM-MQRQp.js.map +1 -0
- package/dist/cases-FvhbM8M5.js +1 -0
- package/dist/{cli-20Fpvs5H.js → cli-s_3opn_I.js} +219 -109
- package/dist/cli-s_3opn_I.js.map +1 -0
- package/dist/{cli-spec-Dsr79Qy1.js → cli-spec-DxzIjmq_.js} +109 -26
- package/dist/cli-spec-DxzIjmq_.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-C5fIzHQa.js +803 -0
- package/dist/client-C5fIzHQa.js.map +1 -0
- package/dist/{command-CRh-cb6Z.js → command-B5OAIjp7.js} +25 -21
- package/dist/command-B5OAIjp7.js.map +1 -0
- package/dist/{command-BLNVDiIg.js → command-BJ7KqczP.js} +10 -10
- package/dist/{command-BLNVDiIg.js.map → command-BJ7KqczP.js.map} +1 -1
- package/dist/{command-aUJvENwy.js → command-BXLZsEwQ.js} +16 -16
- package/dist/{command-aUJvENwy.js.map → command-BXLZsEwQ.js.map} +1 -1
- package/dist/{command-B5OyHpOu.js → command-DA_SHQUQ.js} +16 -16
- package/dist/{command-B5OyHpOu.js.map → command-DA_SHQUQ.js.map} +1 -1
- package/dist/{commands-CWy-WndG.js → commands-BKBamOcX.js} +174 -40
- package/dist/commands-BKBamOcX.js.map +1 -0
- package/dist/{commands-CfdBX-mc.js → commands-BPvAOEmb.js} +1 -1
- package/dist/{config-B9mB1kF5.js → config-C0URar0t.js} +255 -89
- package/dist/config-C0URar0t.js.map +1 -0
- package/dist/{context-Bkv3ndjq.js → context-Sj58aJEu.js} +2 -2
- package/dist/{context-Bkv3ndjq.js.map → context-Sj58aJEu.js.map} +1 -1
- package/dist/{cost-BdRgySQl.js → cost-C21gGrju.js} +19 -9
- package/dist/cost-C21gGrju.js.map +1 -0
- package/dist/cost-IM80LyGD.js +1 -0
- package/dist/{cursor-catalog-BSn9xjmo.js → cursor-catalog-FWgSo-T1.js} +4 -4
- package/dist/{cursor-catalog-BSn9xjmo.js.map → cursor-catalog-FWgSo-T1.js.map} +1 -1
- package/dist/db-Bc4L-Wcr.js +2 -0
- package/dist/{db-zgC1tvKB.js → db-BjAWxVAG.js} +142 -248
- package/dist/db-BjAWxVAG.js.map +1 -0
- package/dist/debug-CIXzB4wK.js.map +1 -1
- package/dist/{default-effort-BAy8ixd1.js → default-effort-CWlhZ6U1.js} +1 -1
- package/dist/{default-effort-DGhQmu95.js → default-effort-Ck2GYIxX.js} +4 -4
- package/dist/{default-effort-DGhQmu95.js.map → default-effort-Ck2GYIxX.js.map} +1 -1
- package/dist/{dist-CAFD1yfv.js → dist-DeqeCUZe.js} +15 -8
- package/dist/dist-DeqeCUZe.js.map +1 -0
- package/dist/{docker-DAMnm3L9.js → docker-CDdK6Ity.js} +49 -22
- package/dist/docker-CDdK6Ity.js.map +1 -0
- package/dist/{docker-Cqw-CyWz.js → docker-D74l67k8.js} +4 -4
- package/dist/{docker-Cqw-CyWz.js.map → docker-D74l67k8.js.map} +1 -1
- package/dist/{docker-Dr4XeDc8.js → docker-DMug3l-a.js} +1 -1
- package/dist/{doctor-B2PohneB.js → doctor-eFxyn6at.js} +85 -46
- package/dist/doctor-eFxyn6at.js.map +1 -0
- package/dist/{environment-DB8o3hGi.js → environment-CJIKz-Ud.js} +49 -11
- package/dist/environment-CJIKz-Ud.js.map +1 -0
- package/dist/{errors-BZgLSZtV.js → errors-C1GBcWE2.js} +2 -2
- package/dist/{errors-BZgLSZtV.js.map → errors-C1GBcWE2.js.map} +1 -1
- package/dist/{eval-source-D3sbAzt-.js → eval-source-CXIRJ91v.js} +24 -12
- package/dist/eval-source-CXIRJ91v.js.map +1 -0
- package/dist/{evaluate-CwKeN_fo.js → evaluate-DA4QsrHO.js} +792 -281
- package/dist/evaluate-DA4QsrHO.js.map +1 -0
- package/dist/{evaluate-COdzf7Hl.d.ts → evaluate-DHrU4Wld.d.ts} +2 -2
- package/dist/fake-bootstrap-CnwWG3kP.js +15 -0
- package/dist/fake-bootstrap-CnwWG3kP.js.map +1 -0
- package/dist/{files-BiFU16BW.js → files-BVVWAS0A.js} +6 -6
- package/dist/{files-BiFU16BW.js.map → files-BVVWAS0A.js.map} +1 -1
- package/dist/files-BaqXvB88.js +1 -0
- package/dist/{files-BEwfjD8J.js → files-Bey1VAex.js} +3 -3
- package/dist/{files-BEwfjD8J.js.map → files-Bey1VAex.js.map} +1 -1
- package/dist/files-BjOJw4sn.js +1 -0
- package/dist/{files-DgXyaAg6.js → files-BrGVs4gY.js} +2 -2
- package/dist/{files-DgXyaAg6.js.map → files-BrGVs4gY.js.map} +1 -1
- package/dist/files-CNHhSyb3.js +1 -0
- package/dist/files-CW1lChNh.js +1 -0
- package/dist/{files-BRzV7aPh.js → files-D6j-XpaK.js} +5 -5
- package/dist/{files-BRzV7aPh.js.map → files-D6j-XpaK.js.map} +1 -1
- package/dist/{files-Cl7BQ5gE.js → files-F44NRCN0.js} +143 -20
- package/dist/files-F44NRCN0.js.map +1 -0
- package/dist/{files-DlQshsks.js → files-g1watgRC.js} +2 -2
- package/dist/{files-DlQshsks.js.map → files-g1watgRC.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-fDiJ8UIV.js → git-Jqkf9SBa.js} +3 -3
- package/dist/{git-fDiJ8UIV.js.map → git-Jqkf9SBa.js.map} +1 -1
- package/dist/{harness-DXv2gnaT.js → harness-FcTyyPco.js} +17 -8
- package/dist/harness-FcTyyPco.js.map +1 -0
- package/dist/{harness-C2berMiM.js → harness-nwnyghJZ.js} +14 -11
- package/dist/{harness-C2berMiM.js.map → harness-nwnyghJZ.js.map} +1 -1
- package/dist/harness.d.ts +11 -8
- package/dist/harness.js +1 -1
- package/dist/{ids-C_advdJA.js → ids-B-JKTCP8.js} +2 -2
- package/dist/{ids-C_advdJA.js.map → ids-B-JKTCP8.js.map} +1 -1
- package/dist/{images-BNgcR77K.js → images-Dsc4Wi-F.js} +65 -30
- package/dist/images-Dsc4Wi-F.js.map +1 -0
- package/dist/index.d.ts +46 -7
- package/dist/index.js +6 -6
- package/dist/init-C0oUG3jh.js +1 -0
- package/dist/{init-DbuZVRrW.js → init-QB5ecPDA.js} +33 -14
- package/dist/init-QB5ecPDA.js.map +1 -0
- package/dist/{ink-list-BpfidQ1h.js → ink-list-mCJypk3d.js} +2 -2
- package/dist/{ink-list-BpfidQ1h.js.map → ink-list-mCJypk3d.js.map} +1 -1
- package/dist/{invocations-CxGNOlh0.js → invocations-BSr4lRzI.js} +5 -5
- package/dist/{invocations-CxGNOlh0.js.map → invocations-BSr4lRzI.js.map} +1 -1
- package/dist/{judge-COMXT1Hx.js → judge-eGH39PaP.js} +12 -10
- package/dist/{judge-COMXT1Hx.js.map → judge-eGH39PaP.js.map} +1 -1
- package/dist/judge.d.ts +3 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-CzgpsTqv.js → judgments-Cj8bFaDG.js} +2 -2
- package/dist/{judgments-CzgpsTqv.js.map → judgments-Cj8bFaDG.js.map} +1 -1
- package/dist/{layout-Cm85tQYk.js → layout-DfQa4mJs.js} +14 -11
- package/dist/layout-DfQa4mJs.js.map +1 -0
- package/dist/{list-DuThsE28.js → list-Dh6T0kdr.js} +23 -12
- package/dist/{list-DuThsE28.js.map → list-Dh6T0kdr.js.map} +1 -1
- package/dist/live-DwjWJTdn.js +1 -0
- package/dist/{live-CgYbO4Te.js → live-xTZs0c61.js} +2 -2
- package/dist/{live-CgYbO4Te.js.map → live-xTZs0c61.js.map} +1 -1
- package/dist/{live-CnGrRM6j.js → live-y1p3oNXH.js} +9 -9
- package/dist/{live-CnGrRM6j.js.map → live-y1p3oNXH.js.map} +1 -1
- package/dist/{model-CwwkrCb7.js → model-3pZZzloU.js} +7 -5
- package/dist/model-3pZZzloU.js.map +1 -0
- package/dist/{model-047Dcd-a.js → model-DEjwetkJ.js} +1 -1
- package/dist/{model-V3mMXp6x.js → model-DNpKiHqO.js} +6 -6
- package/dist/{model-V3mMXp6x.js.map → model-DNpKiHqO.js.map} +1 -1
- package/dist/model-calls-B0BoQfJ4.js +95 -0
- package/dist/model-calls-B0BoQfJ4.js.map +1 -0
- package/dist/{model-DJjLw-0r.js → model-jRKSQBg6.js} +5 -5
- package/dist/{model-DJjLw-0r.js.map → model-jRKSQBg6.js.map} +1 -1
- package/dist/model-nfewGTzB.js +1 -0
- package/dist/{node-D7pAU--0.js → node-DM-mLGdX.js} +7 -7
- package/dist/{node-D7pAU--0.js.map → node-DM-mLGdX.js.map} +1 -1
- package/dist/{opencode-providers-BGiXt8_B.js → opencode-providers-ChJTQRmz.js} +5 -2
- package/dist/{opencode-providers-BGiXt8_B.js.map → opencode-providers-ChJTQRmz.js.map} +1 -1
- package/dist/{openrouter-models-Cw1z8-sq.js → openrouter-models-C0aRo633.js} +2 -2
- package/dist/{openrouter-models-Cw1z8-sq.js.map → openrouter-models-C0aRo633.js.map} +1 -1
- package/dist/options-DpsJ_5CU.js +1 -0
- package/dist/{patch-DH4e_Gkd.js → patch-Bx-qp8Li.js} +2 -2
- package/dist/{patch-DH4e_Gkd.js.map → patch-Bx-qp8Li.js.map} +1 -1
- package/dist/pi-providers-HygmL1y-.js +552 -0
- package/dist/pi-providers-HygmL1y-.js.map +1 -0
- package/dist/{plain-BYUf_fUV.js → plain-Ctn8oUdp.js} +44 -10
- package/dist/plain-Ctn8oUdp.js.map +1 -0
- package/dist/{plan-Cgq9NCEX.js → plan-ChO0A-xn.js} +63 -12
- package/dist/plan-ChO0A-xn.js.map +1 -0
- package/dist/{plugins-BkmPxDiY.js → plugins-BioMWncI.js} +636 -9
- package/dist/plugins-BioMWncI.js.map +1 -0
- package/dist/{preflight-R-InZCQb.js → preflight-BUjk_6wT.js} +1 -1
- package/dist/{preflight-CtlloeOQ.js → preflight-CJSGzMzc.js} +8 -9
- package/dist/preflight-CJSGzMzc.js.map +1 -0
- package/dist/preflight-DD3fI4Hz.js +1 -0
- package/dist/{preflight-BElE_q0u.js → preflight-qFCXmxha.js} +4 -4
- package/dist/{preflight-BElE_q0u.js.map → preflight-qFCXmxha.js.map} +1 -1
- package/dist/{prepare-C7f5mPl5.js → prepare-D7oSSFNm.js} +9 -9
- package/dist/{prepare-C7f5mPl5.js.map → prepare-D7oSSFNm.js.map} +1 -1
- package/dist/{prepare-CzkAdbXm.js → prepare-xHU2BpN-.js} +2 -2
- package/dist/prices-B1vm_lPk.js +1 -0
- package/dist/{prices-5m3SzMZ3.js → prices-BrY_ihLs.js} +80 -22
- package/dist/prices-BrY_ihLs.js.map +1 -0
- package/dist/{provenance-zsc4--dP.js → provenance-B-flBAzR.js} +5 -5
- package/dist/{provenance-zsc4--dP.js.map → provenance-B-flBAzR.js.map} +1 -1
- package/dist/provider-DHT20Gl8.js +1 -0
- package/dist/{provider-BxzZq3Th.js → provider-ieWOn4YS.js} +2 -2
- package/dist/{provider-BxzZq3Th.js.map → provider-ieWOn4YS.js.map} +1 -1
- package/dist/provider-stop-CxzWW7Ft.js +69 -0
- package/dist/provider-stop-CxzWW7Ft.js.map +1 -0
- package/dist/{providers-B8xYMeah.js → providers-B6StVyfG.js} +215 -48
- package/dist/providers-B6StVyfG.js.map +1 -0
- package/dist/providers-Dljf6al6.js +1 -0
- package/dist/query-BAc2tQZj.js +1 -0
- package/dist/{query-Bb3SYICk.js → query-CkLmJv7R.js} +2 -2
- package/dist/{query-Bb3SYICk.js.map → query-CkLmJv7R.js.map} +1 -1
- package/dist/{questions-CWlR9ffu.js → questions-CJIh2FSX.js} +7 -7
- package/dist/{questions-CWlR9ffu.js.map → questions-CJIh2FSX.js.map} +1 -1
- package/dist/{record-6IUzwP9K.js → record-nVVujCW7.js} +2 -2
- package/dist/{record-6IUzwP9K.js.map → record-nVVujCW7.js.map} +1 -1
- package/dist/{redact-BhjtnjrU.js → redact-DL2BTykw.js} +17 -2
- package/dist/redact-DL2BTykw.js.map +1 -0
- package/dist/{remarks-vtkPmQb9.js → remarks-B3wjKscW.js} +46 -11
- package/dist/remarks-B3wjKscW.js.map +1 -0
- package/dist/{remarks-D9Lcd1fF.js → remarks-BpdM_2RC.js} +2 -1
- package/dist/remarks-BpdM_2RC.js.map +1 -0
- package/dist/remarks-CleoDlYw.js +1 -0
- package/dist/{remarks-DPCONFxZ.js → remarks-DqriPKxm.js} +3 -2
- package/dist/{remarks-DPCONFxZ.js.map → remarks-DqriPKxm.js.map} +1 -1
- package/dist/{remote-BgYWqogI.js → remote-C88amFRN.js} +6 -6
- package/dist/{remote-BgYWqogI.js.map → remote-C88amFRN.js.map} +1 -1
- package/dist/resolve-Bul8lHaa.js +205 -0
- package/dist/resolve-Bul8lHaa.js.map +1 -0
- package/dist/resolve-DW92wpIZ.js +1 -0
- package/dist/{resolve-Do0ZaYH6.js → resolve-G576PiFJ.js} +14 -11
- package/dist/resolve-G576PiFJ.js.map +1 -0
- package/dist/{results-JyuM64gy.js → results-zVHIW6xw.js} +9 -22
- package/dist/results-zVHIW6xw.js.map +1 -0
- package/dist/{resume-AW5oXzUc.js → resume-D_PcUXfG.js} +94 -16
- package/dist/resume-D_PcUXfG.js.map +1 -0
- package/dist/{retry-ChzISxmy.js → retry-BvTzCxdc.js} +4 -4
- package/dist/{retry-ChzISxmy.js.map → retry-BvTzCxdc.js.map} +1 -1
- package/dist/{root-Co0jhOT5.js → root-BzeoyfGd.js} +2 -2
- package/dist/{root-Co0jhOT5.js.map → root-BzeoyfGd.js.map} +1 -1
- package/dist/{vercel-CGlSBv63.js → routes-B0F2R-K7.js} +91 -56
- package/dist/routes-B0F2R-K7.js.map +1 -0
- package/dist/{runtime-BpW1f1Mx.js → runtime-B3r_pAq6.js} +4 -4
- package/dist/{runtime-BpW1f1Mx.js.map → runtime-B3r_pAq6.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +1 -1
- package/dist/{sandbox-kSgr7vo5.js → sandbox-Duh3z1TV.js} +3 -3
- package/dist/{sandbox-kSgr7vo5.js.map → sandbox-Duh3z1TV.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CGEsvVRG.js +1 -0
- package/dist/{server-Cn8VqO9S.js → server-DUWMRKik.js} +8 -8
- package/dist/{server-Cn8VqO9S.js.map → server-DUWMRKik.js.map} +1 -1
- package/dist/{shared-Cfi45Bvy.js → shared-DtZwIAfy.js} +7 -1
- package/dist/shared-DtZwIAfy.js.map +1 -0
- package/dist/{source-CYOhR98B.js → source-CHuN4Nfg.js} +2 -2
- package/dist/{source-CYOhR98B.js.map → source-CHuN4Nfg.js.map} +1 -1
- package/dist/source-Wz1Nrfue.js +1 -0
- package/dist/src--yl1OypK.js +2 -0
- package/dist/{src-Cb2MWozp.js → src-B-HuDqpH.js} +276 -41
- package/dist/src-B-HuDqpH.js.map +1 -0
- package/dist/{src-DAzaCTcN.js → src-B5r1d23Y.js} +176 -18
- package/dist/src-B5r1d23Y.js.map +1 -0
- package/dist/{src-Ck2r_EMT.js → src-BHJ4MWvY.js} +504 -119
- package/dist/src-BHJ4MWvY.js.map +1 -0
- package/dist/{src-Dw79VLa3.js → src-BI0v8qlZ.js} +202 -56
- package/dist/src-BI0v8qlZ.js.map +1 -0
- package/dist/src-BL-_0QoE.js +1 -0
- package/dist/{src-YvblOrBU.js → src-BqNzHHWn.js} +87 -13
- package/dist/src-BqNzHHWn.js.map +1 -0
- package/dist/{src-CV_7DkPa.js → src-C-ChdnG6.js} +102 -13
- package/dist/src-C-ChdnG6.js.map +1 -0
- package/dist/src-C-mOSczH.js +2 -0
- package/dist/src-CZaYu6ER.js +3 -0
- package/dist/src-CeBhwD8k.js +2 -0
- package/dist/src-CkEiiyON.js +2 -0
- package/dist/{src-KxS91P7L.js → src-CmA5Yev6.js} +79 -11
- package/dist/src-CmA5Yev6.js.map +1 -0
- package/dist/src-CwH_nEaw.js +2 -0
- package/dist/src-DA95rM6v.js +4 -0
- package/dist/{src-DvGns8cz.js → src-DL-EE4DU.js} +299 -17
- package/dist/src-DL-EE4DU.js.map +1 -0
- package/dist/src-De0KY5S_.js +2 -0
- package/dist/{src-BNQznj4z.js → src-boB6D-5T.js} +124 -45
- package/dist/src-boB6D-5T.js.map +1 -0
- package/dist/state-B3jex5KU.js +1 -0
- package/dist/{state-BOLt8pJg.js → state-DUBRDyMk.js} +22 -22
- package/dist/{state-BOLt8pJg.js.map → state-DUBRDyMk.js.map} +1 -1
- package/dist/store/local.d.ts +8 -0
- package/dist/store/local.js +16 -0
- package/dist/store/local.js.map +1 -0
- package/dist/store/s3.d.ts +10 -0
- package/dist/store/s3.js +18 -0
- package/dist/store/s3.js.map +1 -0
- package/dist/{suite-COglUhPd.js → suite-B9J6Afg6.js} +6 -5
- package/dist/{suite-COglUhPd.js.map → suite-B9J6Afg6.js.map} +1 -1
- package/dist/suite-Bhs9-Z7O.js +2 -0
- package/dist/{summary-DU5zaibx.js → summary-CSp4cfnr.js} +25 -7
- package/dist/{summary-DU5zaibx.js.map → summary-CSp4cfnr.js.map} +1 -1
- package/dist/sync-CxAY6Dki.js +559 -0
- package/dist/sync-CxAY6Dki.js.map +1 -0
- package/dist/tart-CcrrtSrp.js +704 -0
- package/dist/tart-CcrrtSrp.js.map +1 -0
- package/dist/tart-CzViW3WZ.js +186 -0
- package/dist/tart-CzViW3WZ.js.map +1 -0
- package/dist/tart-DmVuVsmX.js +1 -0
- package/dist/tart-bake-F-6seRC9.js +192 -0
- package/dist/tart-bake-F-6seRC9.js.map +1 -0
- package/dist/tart-bake-RK7RaT9E.js +1 -0
- package/dist/tart-cli-B86Jqo1t.js +89 -0
- package/dist/tart-cli-B86Jqo1t.js.map +1 -0
- package/dist/tart-cli-ClvRniLQ.js +1 -0
- package/dist/tart-hosts-Brii7zHM.js +1 -0
- package/dist/tart-hosts-DPyZsg-1.js +313 -0
- package/dist/tart-hosts-DPyZsg-1.js.map +1 -0
- package/dist/{tart-images-DZ_Mbh-g.js → tart-images-CWSf03w8.js} +64 -6
- package/dist/tart-images-CWSf03w8.js.map +1 -0
- package/dist/testing.d.ts +36 -2
- package/dist/testing.js +4 -4
- package/dist/tool-class-CVDdJy0H.js +433 -0
- package/dist/tool-class-CVDdJy0H.js.map +1 -0
- package/dist/{trace-Dx2NBE1O.js → trace-BpXByD7W.js} +13 -2
- package/dist/{trace-Dx2NBE1O.js.map → trace-BpXByD7W.js.map} +1 -1
- package/dist/{traits-DKnYkqeK.js → traits-CCD8XsmI.js} +2 -2
- package/dist/{traits-DKnYkqeK.js.map → traits-CCD8XsmI.js.map} +1 -1
- package/dist/{transcript-CCf07UmU.js → transcript-BP_2hji-.js} +3 -3
- package/dist/{transcript-CCf07UmU.js.map → transcript-BP_2hji-.js.map} +1 -1
- package/dist/{types-DBwFsGem.d.ts → types-BJ0wh8wC.d.ts} +408 -44
- package/dist/{types-C2mLpph3.js → types-CbcnJxCy.js} +7 -1
- package/dist/types-CbcnJxCy.js.map +1 -0
- package/dist/{usage-DJiDnh2D.js → usage-CwIRPPZ9.js} +16 -3
- package/dist/usage-CwIRPPZ9.js.map +1 -0
- package/dist/{vertex-Eb2wqahk.js → vertex-BEWxHoxH.js} +22 -6
- package/dist/{vertex-Eb2wqahk.js.map → vertex-BEWxHoxH.js.map} +1 -1
- package/dist/vertex-DXfcXYKn.js +1 -0
- package/dist/view/index.d.ts +27 -3
- package/dist/view/index.js +8 -8
- package/dist/{views-DAIiLiV3.js → views-DRxXCyIH.js} +6 -6
- package/dist/{views-DAIiLiV3.js.map → views-DRxXCyIH.js.map} +1 -1
- package/dist/views-i67V1ly3.js +2 -0
- package/dist/{worker-entry-BIROTPX1.js → worker-entry-C3Q4U63F.js} +12 -9
- package/dist/worker-entry-C3Q4U63F.js.map +1 -0
- package/dist/workspace-DhUOW8if.js +1 -0
- package/dist/{workspace-uuE51x35.js → workspace-P72SXLXr.js} +7 -7
- package/dist/{workspace-uuE51x35.js.map → workspace-P72SXLXr.js.map} +1 -1
- package/package.json +21 -10
- package/skills/eval-rubric/SKILL.md +10 -0
- package/skills/reeval/SKILL.md +370 -0
- package/skills/reeval/references/capture.md +48 -0
- package/skills/reeval/references/cli.md +162 -0
- package/skills/reeval/references/compare.md +180 -0
- package/skills/reeval/references/config.md +147 -0
- package/skills/reeval/references/environments.md +218 -0
- package/skills/reeval/references/evals.md +241 -0
- package/skills/reeval/references/mistakes.md +39 -0
- package/skills/reeval/references/vocabulary.md +19 -0
- package/skills/setup/SKILL.md +219 -0
- package/skills/setup/references/templates.md +258 -0
- package/dist/adapter-C8xPuXYq.js.map +0 -1
- package/dist/agent-CBoLndM7.js.map +0 -1
- package/dist/agents-C6zhkxp3.js.map +0 -1
- package/dist/app-files-DT5mJyV4.js.map +0 -1
- package/dist/assemble-t6h018An.js.map +0 -1
- package/dist/attempt-host-CIm1-11m.js.map +0 -1
- package/dist/boat-CJCtzfB1.js.map +0 -1
- package/dist/capture-DoxFUN3J.js +0 -1
- package/dist/cases-Dy0mifal.js +0 -1
- package/dist/cases-z9E6DCd5.js.map +0 -1
- package/dist/cli-20Fpvs5H.js.map +0 -1
- package/dist/cli-spec-Dsr79Qy1.js.map +0 -1
- package/dist/client-KM4C5r5_.js +0 -387
- package/dist/client-KM4C5r5_.js.map +0 -1
- package/dist/command-CRh-cb6Z.js.map +0 -1
- package/dist/commands-CWy-WndG.js.map +0 -1
- package/dist/config-B9mB1kF5.js.map +0 -1
- package/dist/cost-BdRgySQl.js.map +0 -1
- package/dist/cost-Cr26Alhq.js +0 -1
- package/dist/db-P2fXw4nx.js +0 -2
- package/dist/db-zgC1tvKB.js.map +0 -1
- package/dist/dist-CAFD1yfv.js.map +0 -1
- package/dist/docker-DAMnm3L9.js.map +0 -1
- package/dist/doctor-B2PohneB.js.map +0 -1
- package/dist/environment-DB8o3hGi.js.map +0 -1
- package/dist/eval-source-D3sbAzt-.js.map +0 -1
- package/dist/evaluate-CwKeN_fo.js.map +0 -1
- package/dist/files-B4KQUiM7.js +0 -1
- package/dist/files-C1Rk9FYA.js +0 -1
- package/dist/files-C3mCC8Mn.js +0 -1
- package/dist/files-Cl7BQ5gE.js.map +0 -1
- package/dist/files-DnlA1A7N.js +0 -1
- package/dist/harness-DXv2gnaT.js.map +0 -1
- package/dist/images-BNgcR77K.js.map +0 -1
- package/dist/init-4RPeD-K-.js +0 -1
- package/dist/init-DbuZVRrW.js.map +0 -1
- package/dist/kinds-C1oY3qyC.js +0 -697
- package/dist/kinds-C1oY3qyC.js.map +0 -1
- package/dist/layout-Cm85tQYk.js.map +0 -1
- package/dist/live-CJCK11nh.js +0 -1
- package/dist/matchers-CXLn-1jm.js +0 -148
- package/dist/matchers-CXLn-1jm.js.map +0 -1
- package/dist/model-Cv4GuTsZ.js +0 -1
- package/dist/model-CwwkrCb7.js.map +0 -1
- package/dist/options-CHubslQf.js +0 -1
- package/dist/plain-BYUf_fUV.js.map +0 -1
- package/dist/plan-Cgq9NCEX.js.map +0 -1
- package/dist/plugins-BkmPxDiY.js.map +0 -1
- package/dist/preflight-4gWSy0Pe.js +0 -1
- package/dist/preflight-CtlloeOQ.js.map +0 -1
- package/dist/prices-5m3SzMZ3.js.map +0 -1
- package/dist/provider-BnUoz4KB.js +0 -1
- package/dist/providers-B8xYMeah.js.map +0 -1
- package/dist/providers-CVu8e0xS.js +0 -1
- package/dist/query-DIfkUcPh.js +0 -1
- package/dist/redact-BhjtnjrU.js.map +0 -1
- package/dist/remarks-CAvTSq16.js +0 -1
- package/dist/remarks-D9Lcd1fF.js.map +0 -1
- package/dist/remarks-vtkPmQb9.js.map +0 -1
- package/dist/resolve-Do0ZaYH6.js.map +0 -1
- package/dist/resolve-amw_2Odz.js +0 -1
- package/dist/results-JyuM64gy.js.map +0 -1
- package/dist/resume-AW5oXzUc.js.map +0 -1
- package/dist/server-2_gUnVvf.js +0 -1
- package/dist/shared-Cfi45Bvy.js.map +0 -1
- package/dist/source-CY3etKzw.js +0 -1
- package/dist/src-6Zzyg4Qd.js +0 -2
- package/dist/src-BDhcPtnI.js +0 -2
- package/dist/src-BNQznj4z.js.map +0 -1
- package/dist/src-BRMsk9nh.js +0 -2
- package/dist/src-CV_7DkPa.js.map +0 -1
- package/dist/src-Cb2MWozp.js.map +0 -1
- package/dist/src-Ck2r_EMT.js.map +0 -1
- package/dist/src-DAzaCTcN.js.map +0 -1
- package/dist/src-DB7N8TYJ.js +0 -2
- package/dist/src-DPDaAYwh.js +0 -2
- package/dist/src-Dbk19BiP.js +0 -2
- package/dist/src-DtodMT59.js +0 -1
- package/dist/src-DvGns8cz.js.map +0 -1
- package/dist/src-Dw79VLa3.js.map +0 -1
- package/dist/src-KxS91P7L.js.map +0 -1
- package/dist/src-T-OeuJLr.js +0 -2
- package/dist/src-YvblOrBU.js.map +0 -1
- package/dist/src-zntEDlnx.js +0 -3
- package/dist/state-kimtIhBP.js +0 -1
- package/dist/suite-CKBa6MKt.js +0 -2
- package/dist/tart-BTVXp8rz.js +0 -373
- package/dist/tart-BTVXp8rz.js.map +0 -1
- package/dist/tart-cli-BIC9HqPg.js +0 -37
- package/dist/tart-cli-BIC9HqPg.js.map +0 -1
- package/dist/tart-cli-BMrkmBjI.js +0 -1
- package/dist/tart-images-DZ_Mbh-g.js.map +0 -1
- package/dist/tool-target-DS5d72eH.js +0 -40
- package/dist/tool-target-DS5d72eH.js.map +0 -1
- package/dist/types-C2mLpph3.js.map +0 -1
- package/dist/usage-DJiDnh2D.js.map +0 -1
- package/dist/vercel-CGlSBv63.js.map +0 -1
- package/dist/vertex-O88TCwgY.js +0 -1
- package/dist/views-DtRhj2Tn.js +0 -2
- package/dist/worker-entry-BIROTPX1.js.map +0 -1
- package/dist/workspace-CcQRT31C.js +0 -1
package/README.md
CHANGED
|
@@ -27,11 +27,11 @@ Evals are TypeScript files in `evals/`, next to the tool they test. Each run wri
|
|
|
27
27
|
|
|
28
28
|
Cursor ⋅ Claude Code ⋅ Codex ⋅ Antigravity ⋅ OpenCode ⋅ Goose ⋅ pi ⋅ Grok Build
|
|
29
29
|
|
|
30
|
-
- **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](docs/configuration/agents/index.mdx)).
|
|
31
|
-
- **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](docs/configuration/environments/secrets.mdx)).
|
|
32
|
-
- **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](docs/results/compare/lift.mdx)).
|
|
33
|
-
- **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](docs/configuration/environments/index.mdx)).
|
|
34
|
-
- **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](docs/evaluations/runs/recording.mdx)).
|
|
30
|
+
- **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](https://github.com/invertase/reeval/blob/v0.21.0/docs/configuration/agents/index.mdx)).
|
|
31
|
+
- **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](https://github.com/invertase/reeval/blob/v0.21.0/docs/configuration/environments/secrets.mdx)).
|
|
32
|
+
- **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](https://github.com/invertase/reeval/blob/v0.21.0/docs/results/compare/lift.mdx)).
|
|
33
|
+
- **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](https://github.com/invertase/reeval/blob/v0.21.0/docs/configuration/environments/index.mdx)).
|
|
34
|
+
- **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/runs/recording.mdx)).
|
|
35
35
|
|
|
36
36
|
## Install
|
|
37
37
|
|
|
@@ -140,23 +140,23 @@ Environments load extensions from where teams already keep them:
|
|
|
140
140
|
|
|
141
141
|
## Examples
|
|
142
142
|
|
|
143
|
-
The recipes in [`docs/evaluations/recipes`](docs/evaluations/recipes) are complete evals for common jobs:
|
|
143
|
+
The recipes in [`docs/evaluations/recipes`](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes) are complete evals for common jobs:
|
|
144
144
|
|
|
145
|
-
- [Measure a skill](docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
|
|
146
|
-
- [Compare harnesses](docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
|
|
147
|
-
- [Measure MCP servers](docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
|
|
148
|
-
- [Check a recommendation](docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
|
|
149
|
-
- [Fail a pull request](docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
|
|
145
|
+
- [Measure a skill](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
|
|
146
|
+
- [Compare harnesses](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
|
|
147
|
+
- [Measure MCP servers](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
|
|
148
|
+
- [Check a recommendation](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
|
|
149
|
+
- [Fail a pull request](https://github.com/invertase/reeval/blob/v0.21.0/docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
|
|
150
150
|
|
|
151
|
-
The [`examples/`](examples) directory has 80 runnable projects, one idea each, from [`with-checks`](examples/with-checks) (deterministic checks) and [`with-skill-lift`](examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](examples/README.md) lists them all.
|
|
151
|
+
The [`examples/`](https://github.com/invertase/reeval/blob/v0.21.0/examples) directory has 80 runnable projects, one idea each, from [`with-checks`](https://github.com/invertase/reeval/blob/v0.21.0/examples/with-checks) (deterministic checks) and [`with-skill-lift`](https://github.com/invertase/reeval/blob/v0.21.0/examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](https://github.com/invertase/reeval/blob/v0.21.0/examples/README.md) lists them all.
|
|
152
152
|
|
|
153
153
|
## Documentation
|
|
154
154
|
|
|
155
|
-
The manual in [`docs/`](docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
|
|
155
|
+
The manual in [`docs/`](https://github.com/invertase/reeval/blob/v0.21.0/docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
|
|
156
156
|
|
|
157
157
|
## For coding agents
|
|
158
158
|
|
|
159
|
-
To write or update evals, load [`skills/reeval/SKILL.md`](skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
|
|
159
|
+
To write or update evals, load [`skills/reeval/SKILL.md`](https://github.com/invertase/reeval/blob/v0.21.0/skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
|
|
160
160
|
|
|
161
161
|
To preview a run without starting sandboxes or spending tokens, run `npx reeval run --dry-run --json`. It prints the planned evals, cells, and attempts as one JSON document and exits 0.
|
|
162
162
|
|
|
@@ -166,4 +166,4 @@ To read past runs without calling a model, run `npx reeval query failing_attempt
|
|
|
166
166
|
|
|
167
167
|
## License
|
|
168
168
|
|
|
169
|
-
[Apache-2.0](LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
|
|
169
|
+
[Apache-2.0](https://github.com/invertase/reeval/blob/v0.21.0/LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
|
package/THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -7,8 +7,8 @@ with its own license.
|
|
|
7
7
|
|
|
8
8
|
| Package | Version | License | Inlined into |
|
|
9
9
|
|---|---|---|---|
|
|
10
|
-
| @ai-sdk/harness | 1.0.
|
|
11
|
-
| @ai-sdk/harness-claude-code | 1.0.
|
|
10
|
+
| @ai-sdk/harness | 1.0.172 | Apache-2.0 | dist/bridge |
|
|
11
|
+
| @ai-sdk/harness-claude-code | 1.0.176 | Apache-2.0 | dist, dist/bridge |
|
|
12
12
|
| @floating-ui/core | 1.8.0 | MIT | the studio app |
|
|
13
13
|
| @floating-ui/dom | 1.8.0 | MIT | the studio app |
|
|
14
14
|
| @floating-ui/react-dom | 2.1.9 | MIT | the studio app |
|
|
@@ -72,7 +72,7 @@ with its own license.
|
|
|
72
72
|
| shadcn/ui | | MIT | 11 source files |
|
|
73
73
|
| moby/profiles (Docker default seccomp profile) | | Apache-2.0 | 1 source files |
|
|
74
74
|
|
|
75
|
-
## @ai-sdk/harness@1.0.
|
|
75
|
+
## @ai-sdk/harness@1.0.172
|
|
76
76
|
|
|
77
77
|
- License: Apache-2.0
|
|
78
78
|
- Source: https://github.com/vercel/ai
|
|
@@ -94,12 +94,12 @@ See the License for the specific language governing permissions and
|
|
|
94
94
|
limitations under the License.
|
|
95
95
|
```
|
|
96
96
|
|
|
97
|
-
## @ai-sdk/harness-claude-code@1.0.
|
|
97
|
+
## @ai-sdk/harness-claude-code@1.0.176
|
|
98
98
|
|
|
99
99
|
- License: Apache-2.0
|
|
100
100
|
- Source: https://github.com/vercel/ai
|
|
101
101
|
- Inlined into: dist, dist/bridge
|
|
102
|
-
- Modified: dist/bridge/index.mjs, with patches/@ai-sdk__harness-claude-code@1.0.
|
|
102
|
+
- Modified: dist/bridge/index.mjs, with patches/@ai-sdk__harness-claude-code@1.0.176.patch
|
|
103
103
|
|
|
104
104
|
```text
|
|
105
105
|
Copyright 2023 Vercel, Inc.
|
|
@@ -1,14 +1,17 @@
|
|
|
1
|
-
// Writes pi's models.json for a
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// Writes pi's models.json for a model on an AI gateway, run in the sandbox before each turn, with
|
|
2
|
+
// the model in a provider of reeval's, on the API its provider's route names (`gateway.routes`),
|
|
3
|
+
// else Chat Completions. Its thinking levels, limits and prices are pi's catalog entry for it, on
|
|
4
|
+
// the gateway or its provider, else the host's description of it. A route with `thinkingConfig`
|
|
5
|
+
// sends each thinking level as the model's native `thinkingConfig`, so the model gets the level pi
|
|
6
|
+
// sends it there.
|
|
5
7
|
//
|
|
6
|
-
// node gateway-models.mjs <bootstrap dir> <models.json> <
|
|
7
|
-
import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
|
|
8
|
+
// node gateway-models.mjs <bootstrap dir> <models.json> <gateway json> <model> <model info json>
|
|
9
|
+
import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'
|
|
8
10
|
import { dirname, join } from 'node:path'
|
|
9
11
|
|
|
10
|
-
const [bootstrapDir, out,
|
|
11
|
-
const
|
|
12
|
+
const [bootstrapDir, out, gatewayJson, model, infoJson] = process.argv.slice(2)
|
|
13
|
+
const gateway = JSON.parse(gatewayJson)
|
|
14
|
+
const info = JSON.parse(infoJson ?? 'null')
|
|
12
15
|
const fail = (why) => {
|
|
13
16
|
process.stderr.write(`reeval: ${why}\n`)
|
|
14
17
|
process.exit(1)
|
|
@@ -16,52 +19,83 @@ const fail = (why) => {
|
|
|
16
19
|
|
|
17
20
|
// pnpm puts pi's own dependencies beside it, so pi-ai is pi-coding-agent's sibling.
|
|
18
21
|
const agent = realpathSync(join(bootstrapDir, 'node_modules/@earendil-works/pi-coding-agent'))
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
)
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
`${id} takes a thinking budget, not a thinking level, so pi can't run it on ${provider.name}`,
|
|
22
|
+
const data = join(dirname(agent), 'pi-ai/dist/providers/data')
|
|
23
|
+
/** The chat models of one of pi's catalogs, whatever API each is on. */
|
|
24
|
+
const catalog = (name) => {
|
|
25
|
+
const file = join(data, `${name}.json`)
|
|
26
|
+
if (!/^[\w-]+$/.test(name) || !existsSync(file)) return []
|
|
27
|
+
return Object.values(JSON.parse(readFileSync(file, 'utf8'))).flatMap((models) =>
|
|
28
|
+
Object.entries(models)
|
|
29
|
+
.filter(([key]) => key.startsWith('chat:'))
|
|
30
|
+
.map(([, m]) => m),
|
|
29
31
|
)
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
const slash = model.indexOf('/')
|
|
35
|
+
const provider = slash < 0 ? '' : model.slice(0, slash)
|
|
36
|
+
const id = model.slice(slash + 1)
|
|
37
|
+
const route = gateway.routes[provider] ?? {}
|
|
38
|
+
const native = route.thinkingConfig
|
|
39
|
+
|
|
40
|
+
let entry
|
|
41
|
+
if (native) {
|
|
42
|
+
entry = catalog(native.catalog).find((m) => m.id === id)
|
|
43
|
+
if (!entry)
|
|
44
|
+
fail(
|
|
45
|
+
`pi's ${native.catalog} catalog has no model ${id}, so pi can't run ${model} on ${gateway.name}`,
|
|
46
|
+
)
|
|
47
|
+
if (entry.reasoning && !entry.thinkingLevelMap) {
|
|
48
|
+
fail(
|
|
49
|
+
`${id} takes a thinking budget, not a thinking level, so pi can't run it on ${gateway.name}`,
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
// A provider's own catalog names a version with a dash where a gateway dots it (claude-haiku-4-5).
|
|
54
|
+
const dashed = id.replace(/(\d)\.(\d)/g, '$1-$2')
|
|
55
|
+
entry =
|
|
56
|
+
catalog(gateway.catalog).find((m) => m.id === model) ??
|
|
57
|
+
catalog(provider).find((m) => m.id === id || m.id === dashed) ??
|
|
58
|
+
info ??
|
|
59
|
+
fail(
|
|
60
|
+
`neither pi's catalog nor models.dev describes ${model}, so pi can't run it on ${gateway.name}`,
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
|
|
32
64
|
const thinking = (level) => ({ thinkingConfig: { thinkingLevel: level, includeThoughts: true } })
|
|
33
65
|
const byLevel = {}
|
|
34
|
-
for (const level of entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
|
|
66
|
+
for (const level of native && entry.reasoning ? ['minimal', 'low', 'medium', 'high'] : []) {
|
|
35
67
|
const mapped = entry.thinkingLevelMap[level]
|
|
36
68
|
if (mapped === null) continue
|
|
37
|
-
const
|
|
69
|
+
const sent = (typeof mapped === 'string' ? mapped : level).toLowerCase()
|
|
38
70
|
byLevel[level] = {
|
|
39
|
-
providerOptions: Object.fromEntries(
|
|
71
|
+
providerOptions: Object.fromEntries(native.upstreams.map((u) => [u, thinking(sent)])),
|
|
40
72
|
}
|
|
41
73
|
}
|
|
42
74
|
|
|
75
|
+
const api = route.api ?? 'openai-completions'
|
|
43
76
|
const models = {
|
|
44
77
|
providers: {
|
|
45
|
-
[
|
|
46
|
-
name:
|
|
47
|
-
baseUrl:
|
|
48
|
-
api
|
|
49
|
-
apiKey: `$${
|
|
50
|
-
|
|
51
|
-
// would map Gemini's levels onto its own.
|
|
78
|
+
[gateway.id]: {
|
|
79
|
+
name: gateway.name,
|
|
80
|
+
baseUrl: api === 'anthropic-messages' ? gateway.anthropic : gateway.openai,
|
|
81
|
+
api,
|
|
82
|
+
apiKey: `$${gateway.credentialEnv}`,
|
|
83
|
+
authHeader: true,
|
|
52
84
|
compat: {
|
|
53
|
-
|
|
85
|
+
// The level goes as the thinkingConfig above; the gateway's shared effort field would map
|
|
86
|
+
// the model's levels onto its own.
|
|
87
|
+
...(native ? { supportsReasoningEffort: false } : {}),
|
|
54
88
|
supportsStore: false,
|
|
55
89
|
supportsDeveloperRole: false,
|
|
56
90
|
},
|
|
57
91
|
models: [
|
|
58
92
|
{
|
|
59
93
|
id: model,
|
|
60
|
-
name: entry.name,
|
|
94
|
+
...(entry.name ? { name: entry.name } : {}),
|
|
61
95
|
reasoning: entry.reasoning,
|
|
62
96
|
...(entry.thinkingLevelMap ? { thinkingLevelMap: entry.thinkingLevelMap } : {}),
|
|
63
97
|
input: entry.input,
|
|
64
|
-
cost: entry.cost,
|
|
98
|
+
...(entry.cost ? { cost: entry.cost } : {}),
|
|
65
99
|
contextWindow: entry.contextWindow,
|
|
66
100
|
maxTokens: entry.maxTokens,
|
|
67
101
|
...(Object.keys(byLevel).length ? { samplingParamsByThinkingLevel: byLevel } : {}),
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { t as codeFrame } from "./errors-
|
|
1
|
+
import { a as PACKAGE_NAMES, c as ownDir, d as ownRequire, l as ownFile, n as COMPILED } from "./root-BzeoyfGd.js";
|
|
2
|
+
import { c as relativeFile } from "./ids-B-JKTCP8.js";
|
|
3
|
+
import { t as codeFrame } from "./errors-C1GBcWE2.js";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
5
|
import { existsSync, readFileSync } from "node:fs";
|
|
6
6
|
import { delimiter, dirname, join, resolve } from "node:path";
|
|
7
|
-
import { stripVTControlCharacters } from "node:util";
|
|
8
7
|
import { spawnSync } from "node:child_process";
|
|
8
|
+
import { stripVTControlCharacters } from "node:util";
|
|
9
9
|
import { createJiti } from "jiti";
|
|
10
10
|
import { DecodedMap, getOriginalPosition } from "@vitest/utils/source-map";
|
|
11
11
|
import { getTsconfig } from "get-tsconfig";
|
|
@@ -41,6 +41,8 @@ async function ownModules() {
|
|
|
41
41
|
"/harness": import("./harness.js"),
|
|
42
42
|
"/sandbox": import("./sandbox.js"),
|
|
43
43
|
"/sandbox/boat": import("./sandbox/boat.js"),
|
|
44
|
+
"/store/local": import("./store/local.js"),
|
|
45
|
+
"/store/s3": import("./store/s3.js"),
|
|
44
46
|
"/judge": import("./judge.js"),
|
|
45
47
|
"/gateway": import("./gateway.js"),
|
|
46
48
|
"/view": import("./view/index.js")
|
|
@@ -288,7 +290,7 @@ const LANGUAGE_ADAPTERS = [TYPESCRIPT_ADAPTER, {
|
|
|
288
290
|
extensions: [".eval.py"],
|
|
289
291
|
includeGlobs: ["evals/**/*.eval.py"],
|
|
290
292
|
remote: true,
|
|
291
|
-
load: (file, root) => import("./remote-
|
|
293
|
+
load: (file, root) => import("./remote-C88amFRN.js").then((m) => m.collectLanguageFile(file, root)),
|
|
292
294
|
runnerCommand: (root) => resolvePythonRunnerCommand(root),
|
|
293
295
|
setupHint: PYTHON_RUNNER_HINT
|
|
294
296
|
}];
|
|
@@ -314,4 +316,4 @@ function runnerEnv(root, adapter) {
|
|
|
314
316
|
}
|
|
315
317
|
export { originalText as a, originalStack as i, defaultIncludeGlobs as n, reevalJiti as o, runnerEnv as r, adapterForFile as t };
|
|
316
318
|
|
|
317
|
-
//# sourceMappingURL=adapter-
|
|
319
|
+
//# sourceMappingURL=adapter-BdzhdRYo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter-BdzhdRYo.js","names":[],"sources":["../src/jiti.ts","../src/engine/loader.ts","../src/language-runner/python-runner.ts","../src/language-runner/adapter.ts"],"mappings":";;;;;;;;;;;;AASA,IAAI;;;;;;AAOJ,MAAa,iBAA4B,YAAY;CACnD,IAAI,CAAC,OAAO;EACV,MAAM,OAAO,WACT,KAAK,OAAO,aAAa,GAAG,WAAW,IACvC,KAAK,QAAQ,WAAW,QAAQ,mBAAmB,CAAC,GAAG,gBAAgB;EAC3E,QAAQ,cAAc,IAAI,CAAC,CAAC,IAAI;CAClC;CACA,OAAO,MAAM,OAAO;AACtB;;;;;AAMA,eAAsB,WAAW,IAAY,UAAuB,CAAC,GAAkB;CACrF,MAAM,iBAAiB,WAAW,MAAM,WAAW,IAAI,KAAA;CACvD,OAAO,WAAW,IAAI;EACpB,WAAW;EACX,GAAG;EACH,GAAI,kBAAkB,EAAE,eAAe;CACzC,CAAC;AACH;AAEA,eAAe,aAA+C;CAC5D,MAAM,UAA4C;EAChD,IAAI,OAAO;EACX,YAAY,OAAO;EACnB,YAAY,OAAO;EACnB,iBAAiB,OAAO;EACxB,gBAAgB,OAAO;EACvB,aAAa,OAAO;EACpB,UAAU,OAAO;EACjB,YAAY,OAAO;EACnB,SAAS,OAAO;CAClB;CACA,MAAM,UAAmC,CAAC;CAC1C,KAAK,MAAM,CAAC,SAAS,WAAW,OAAO,QAAQ,OAAO,GACpD,KAAK,MAAM,QAAQ,eAAe,QAAQ,OAAO,WAAW,MAAM;CACpE,OAAO;AACT;;AC3CA,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAgB;AAClB;AAOA,MAAM,8BAAc,IAAI,IAAyB;AAEjD,MAAM,aACJ;;AAGF,eAAsB,aAAa,MAAc,MAAgC;CAC/E,cAAc,QAAQ,IAAI,GAAG,IAAI;CACjC,MAAM,UAAU;EAAE,aAAa;EAAM,SAAS;EAAO,YAAY;CAAK;CACtE,MAAM,OAAO,MAAM,WAAW,MAAM,OAAO;CAC3C,MAAM,OAAO,MAAM,WAAW,MAAM;EAClC,GAAG;EACH,YAAY,SAAS;GACnB,MAAM,OAAO,KAAK,UAAU,IAAI;GAChC,MAAM,SAAS,WAAW,KAAK,IAAI;GACnC,IAAI,KAAK,YAAY,SAAS,IAAI;IAChC,MAAM,MAAM,KAAK,MAAM,OAAO,KAAK,OAAO,IAAI,QAAQ,CAAC,CAAC,SAAS,MAAM,CAAC;IAExE,MAAM,QACJ,IAAI,KAAK,QAAQ,WAAW,GAAG,2FAC5B;IACL,YAAY,IAAI,KAAK,UAAU;KAC7B,KAAK,IAAI,WAAW,KAAK,KAAK,QAAQ;KACtC;KACA,QAAQ,KAAK;KACb;IACF,CAAC;GACH;GACA,OAAO,EAAE,KAAK;EAChB;CACF,CAAC;CACD,IAAI;EACF,OAAO,MAAM,KAAK,OAAO,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,YAAY,OAAO,IAAI,KAAK,cAAc,KAAK;CACvD;AACF;;AAGA,SAAS,cAAc,OAAyB;CAC9C,MAAM,IAAI;CACV,IAAI,GAAG,SAAS,oBAAoB,OAAO;CAC3C,MAAM,QAAQ,wDAAwD,KAAK,EAAE,OAAO;CACpF,IAAI,OAAO,EAAE,UAAU,GAAG,MAAM,GAAG,iBAAiB,MAAM;CAC1D,OAAO;AACT;;;;;AAMA,SAAgB,YAAY,OAAgB,MAAuC;CACjF,MAAM,SAAS,8CAA8C,KAAM,OAAiB,WAAW,EAAE;CACjG,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,GAAG,MAAM,MAAM,MAAM,UAAU;CAErC,MAAM,SAAS,KAAK,QAAQ,8BAA8B,EAAE;CAC5D,MAAM,KAAK;EAAE,MAAM,OAAO,IAAI;EAAG,QAAQ,OAAO,MAAM,IAAI;CAAE;CAC5D,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,yBAAyB,UAAU,aAAa,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;CACpF,QAAQ,CAAC;CACT,uBAAO,IAAI,YACT,GAAG,OAAO,IAAI,aAAa,MAAM,IAAI,EAAE,GAAG,GAAG,KAAK,GAAG,GAAG,OAAO,GAAG,QAAQ,KAAK,UAAU,IAC3F;AACF;;AAGA,SAAgB,cAAc,OAAuB;CACnD,OAAO,MAAM,QACX,gDACC,OAAO,QAAgB,MAAc,MAAc,QAAgB,UAAkB;EACpF,MAAM,SAAS,YAAY,IAAI,IAAI;EACnC,MAAM,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS,MAAO,QAAQ,SAAS,IAAK;EAExE,IAAI,CAAC,UAAU,MAAM,GAAG,OAAO;EAC/B,MAAM,KAAK,oBAAoB,OAAO,KAAK;GAAE,MAAM,OAAO,IAAI;GAAG,QAAQ;EAAI,CAAC;EAC9E,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,WAAW,GAAG;EACpB,OAAO,GAAG,SAAS,KAAK,GAAG,SAAS,GAAG,OAAO,MAAM,UAAU,GAAG,MAAgB,IAAI,IAAI;CAC3F,CACF;AACF;;;;;;;AAQA,SAAgB,aAAa,MAAc,MAAmC;CAC5E,MAAM,YAAY,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;CAClE,KAAK,MAAM,UAAU,CAAC,GAAG,WAAW,GAAG,YAAY,OAAO,CAAC,GAAG;EAC5D,MAAM,KAAK,QAAQ,KAAK,QAAQ,IAAI,KAAK;EACzC,IAAI,CAAC,UAAU,KAAK,GAAG;EAGvB,MAAM,OAAO,KAAK,KAAK,SAAS;EAChC,IAAI,aAAa;EACjB,MAAM,MAAM,eAAe,QAAQ,IAAI;EACvC,OAAO,aAAa,MAAM,eAAe,QAAQ,aAAa,CAAC,MAAM,KAAK;EAC1E,OAAO,OAAO,OAAO,MAAM,eAAe,QAAQ,EAAE,GAAG,MAAM,OAAO,aAAa,CAAC;CACpF;AAEF;;AAGA,SAAS,eAAe,QAAqB,OAAuB;CAClE,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,IAAI;CACpD,MAAM,SAAU,MAAM,GAAG,EAAE,CAAC,CAAY;CACxC,MAAM,KAAK,oBAAoB,OAAO,KAAK;EAAE,MAAM,MAAM;EAAQ;CAAO,CAAC;CAGzE,OADoB,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,QAAQ,KAAK,CACxD,CAAC,CAAC,QAAQ,GAAG,SAAS,IAAI,KAAK,SAAS,GAAG,CAAC,KAAK,IAAI,UAAU;AAClF;;;;;AAMA,SAAS,OAAO,MAAc,MAAc,QAAwB;CAElE,MAAM,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,MAAM;CACpE,IAAI,KAAK,YAAY,KAAK,OAAO;CACjC,IAAI,QAAQ;CACZ,OAAO,QAAQ,KAAK,QAAQ,KAAK,KAAK,QAAQ,EAAY,GAAG;CAC7D,OAAO;AACT;;AAGA,SAAgB,cAAc,KAAa,MAAoB;CAC7D,IAAI;CACJ,IAAI;EACF,QAAQ,YAAY,GAAG;CACzB,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB;EACjC,MAAM,UAAU,4BAA4B,KAAK,OAAO;EACxD,MAAM,IAAI,cACR,UACI,4BAA4B,QAAQ,GAAG,yBACvC,4BAA4B,SAClC;CACF;CACA,IAAI,CAAC,OAAO;CACZ,IAAI;EAEF,KAAK,MAAM,kBAAkB,aAAa,MAAM,MAAM,MAAM,CAAC,CAAC,QAAQ,WAAW,EAAE,CAAC,CAAC;CACvF,SAAS,OAAO;EACd,MAAM,IAAI,cACR,4BAA4B,aAAa,MAAM,MAAM,IAAI,EAAE,KAAM,MAAgB,SACnF;CACF;AACF;;AAGA,SAAgB,kBAAkB,MAAsB;CACtD,OAAO,KACL,KAAK,OAAO,SAAS;EACnB,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,IAAI;GAC7B,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,KAAK,EAAE;EACzC;EACA,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,MAAM,CAAC;GAChC,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,MAAM,GAAG,GAAG;EAC9C;CAEF,CAAC,IACA,SAAU,aAAa,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,KAAA,CAClD;AACF;;AAGA,SAAS,KAAK,MAAc,IAA4D;CACtF,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,SAAU;EACjC,IAAI,KAAK,OAAO,MAAK;GACnB,IAAI,MAAM,IAAI;GACd,OAAO,MAAM,KAAK,UAAU,KAAK,SAAS,MAAK,OAAO,KAAK,SAAS,OAAO,IAAI;GAC/E,OAAO,KAAK,MAAM,GAAG,MAAM,CAAC;GAC5B,IAAI,MAAM;GACV;EACF;EACA,MAAM,MAAM,GAAG,KAAK,MAAM,CAAC,CAAC;EAC5B,IAAI,KAAK;GACP,OAAO,IAAI;GACX,KAAK,IAAI;EACX,OAAO,OAAO,KAAK;CACrB;CACA,OAAO;AACT;;AC7MA,MAAa,uBAAuB;AAEpC,MAAM,cAAc,CAAC,MAAM,oBAAoB;AAE/C,IAAI;AAEJ,SAAgB,cAAuB;CACrC,IAAI,aAAa,KAAA,GAAW,OAAO;CAMnC,WALe,UAAU,MAAM,CAAC,WAAW,GAAG;EAC5C,UAAU;EACV,OAAO;EACP,aAAa;CACf,CACgB,CAAC,CAAC,WAAW;CAC7B,OAAO;AACT;AAEA,SAAgB,cAAc,MAAuB;CACnD,OAAO,WAAW,KAAK,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,gBAAgB,CAAC;AACrF;AAEA,SAAS,WAAW,MAAkC;CACpD,MAAM,OAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;CAChD,IAAI,WAAW,IAAI,GAAG,OAAO;CAC7B,MAAM,MAAM,KAAK,MAAM,SAAS,WAAW,YAAY;CACvD,IAAI,WAAW,GAAG,GAAG,OAAO;AAE9B;;AAGA,SAAgB,2BAA2B,MAA6B;CACtE,MAAM,WAAW,QAAQ,IAAI,eAAe,KAAK;CACjD,IAAI,UAAU,OAAO,CAAC,UAAU,GAAG,WAAW;CAE9C,IAAI,cAAc,IAAI,KAAK,YAAY,GACrC,OAAO;EAAC;EAAM;EAAO;EAAa;EAAM;EAAU,GAAG;CAAW;CAGlE,MAAM,OAAO,WAAW,IAAI;CAC5B,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,WAAW;CAGtC,OAAO,CAAC,QAAQ,aAAa,UAAU,WAAW,WAAW,GAAG,WAAW;AAC7E;;AAGA,MAAa,qBAAqB;CAChC,kCAAkC,qBAAqB;CACvD;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;;AC7BV,MAAa,qBAAuC;CAClD,IAAI;CACJ,YAAY;EAAC;EAAY;EAAa;EAAa;EAAY;EAAa;CAAW;CACvF,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,aAAa,MAAM,IAAI;AAC/C;AAEA,MAAM,sBAA0C;CAC9C,MAAM,UAAU,KAAK,QAAQ,IAAI,GAAG,eAAe;CACnD,IAAI,WAAW,KAAK,SAAS,gBAAgB,CAAC,GAAG,OAAO;AAE1D;;AAaA,MAAa,oBAAgD,CAAC,oBAAoB;CAVhF,IAAI;CACJ,YAAY,CAAC,UAAU;CACvB,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,OAAO,uBAAc,CAAC,MAAM,MAAM,EAAE,oBAAoB,MAAM,IAAI,CAAC;CACzF,gBAAgB,SAAS,2BAA2B,IAAI;CACxD,WAAW;AAIqE,CAAc;AAEhG,SAAgB,eAAe,MAA+B;CAE5D,OADc,kBAAkB,MAAM,MAAM,EAAE,WAAW,MAAM,QAAQ,KAAK,SAAS,GAAG,CAAC,CAC9E,KAAK;AAClB;AAEA,SAAgB,sBAAgC;CAC9C,OAAO,kBAAkB,SAAS,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC;AAC7D;AAEA,SAAgB,UAAU,MAAc,SAA6C;CACnF,MAAM,MAAM,QAAQ,IAAI,qBAAqB,cAAc;CAC3D,IAAI,QAAQ,OAAO,YAAY,CAAC,KAAK,OAAO,QAAQ;CACpD,MAAM,MAAM,KAAK,KAAK,KAAK;CAC3B,OAAO;EACL,GAAG,QAAQ;EACX,YAAY;GAAC;GAAK;GAAK,QAAQ,IAAI;EAAU,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,SAAS;EAC7E,qBAAqB,QAAQ,IAAI;CACnC;AACF"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
+
import { ct as SANDBOX_ROOT, dt as WORKSPACE_DIR } from "./src-BHJ4MWvY.js";
|
|
2
|
+
import { B as runsAgent, P as capabilitiesOf, U as sandboxKind, W as suggestedSandbox } from "./plugins-BioMWncI.js";
|
|
3
|
+
import { a as plural } from "./text-B8-W3Cvn.js";
|
|
4
|
+
import { S as brokerToken } from "./vertex-BEWxHoxH.js";
|
|
5
|
+
import { n as hasNetworkSession, t as InfraError } from "./types-CbcnJxCy.js";
|
|
6
|
+
import { C as writePluginFiles, S as unverifiedHarness, c as agentMcp, o as agentAuth, s as agentFor, u as authHint } from "./agents-l0-k7vw3.js";
|
|
1
7
|
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
|
-
import { t as addUsage } from "./usage-
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { $ as WORKSPACE_DIR, X as SANDBOX_ROOT } from "./src-Ck2r_EMT.js";
|
|
6
|
-
import { _ as unverifiedHarness, a as agentFor, i as agentAuth, o as agentMcp, s as authHint, v as writePluginFiles } from "./agents-C6zhkxp3.js";
|
|
7
|
-
import { n as hasNetworkSession, t as InfraError } from "./types-C2mLpph3.js";
|
|
8
|
-
import { i as debugTrace, r as debugLog } from "./attempt-CAdlwlNu.js";
|
|
8
|
+
import { a as tokenUsage, t as addUsage } from "./usage-CwIRPPZ9.js";
|
|
9
|
+
import { i as debugTrace, r as debugLog } from "./attempt-CKfKsqqw.js";
|
|
10
|
+
import { n as providerStop } from "./provider-stop-CxzWW7Ft.js";
|
|
9
11
|
import { HarnessAgent } from "@ai-sdk/harness/agent";
|
|
10
12
|
const SHELL_TOOLS = /* @__PURE__ */ new Set([
|
|
11
13
|
"bash",
|
|
@@ -29,6 +31,21 @@ function stallFlags(tool, durationMs) {
|
|
|
29
31
|
return {};
|
|
30
32
|
}
|
|
31
33
|
const record = (value) => isObjectLike(value) ? value : {};
|
|
34
|
+
/**
|
|
35
|
+
* A reasoning block recovered after the turn (Antigravity's, from its stored conversation) says
|
|
36
|
+
* in its provider metadata how long before its own part it started, and how long the agent says
|
|
37
|
+
* it took: it's placed there, not where it arrived.
|
|
38
|
+
*/
|
|
39
|
+
function recoveredTiming(part) {
|
|
40
|
+
for (const meta of Object.values(record(part.providerMetadata))) {
|
|
41
|
+
const { startedBeforeMs, durationMs } = record(meta);
|
|
42
|
+
if (typeof startedBeforeMs !== "number") continue;
|
|
43
|
+
return {
|
|
44
|
+
startedBeforeMs,
|
|
45
|
+
durationMs: typeof durationMs === "number" ? durationMs : null
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
32
49
|
/** The harness's synthetic "fileChange" twin of every edit, which always says "modify". */
|
|
33
50
|
const isFileChangeEcho = (part) => part.toolName === "fileChange" || str(part.toolCallId).startsWith("harness-file-change-");
|
|
34
51
|
function lines(text) {
|
|
@@ -284,18 +301,49 @@ function subagentParts(raw) {
|
|
|
284
301
|
return [];
|
|
285
302
|
});
|
|
286
303
|
}
|
|
304
|
+
const toolOutputOf = (value) => {
|
|
305
|
+
const said = record(value);
|
|
306
|
+
if (said.view !== "full") return {};
|
|
307
|
+
return {
|
|
308
|
+
view: "full",
|
|
309
|
+
...typeof said.bytes === "number" ? { bytes: said.bytes } : {}
|
|
310
|
+
};
|
|
311
|
+
};
|
|
312
|
+
/**
|
|
313
|
+
* What an adapter says about a tool result's output, as `toolOutput` in the result's
|
|
314
|
+
* `providerMetadata` under its own key: `view: 'full'` for output it read from where the CLI
|
|
315
|
+
* saved it (the model got a pointer or nothing), and `bytes`, the saved output's size in UTF-8,
|
|
316
|
+
* when it kept only part of it. An adapter that finds the saved output only after the result
|
|
317
|
+
* went out sends a `raw` part whose `toolOutput` also has the `toolCallId` and the `output` that
|
|
318
|
+
* replaces the result's.
|
|
319
|
+
*/
|
|
320
|
+
function adapterToolOutput(metadata) {
|
|
321
|
+
for (const entry of Object.values(record(metadata))) {
|
|
322
|
+
const said = toolOutputOf(record(entry).toolOutput);
|
|
323
|
+
if (said.view) return said;
|
|
324
|
+
}
|
|
325
|
+
return {};
|
|
326
|
+
}
|
|
287
327
|
/**
|
|
288
328
|
* Turns an AI SDK harness stream into reeval transcript events: text and reasoning blocks joined,
|
|
289
329
|
* each tool call merged with its result, edits normalized to `file-edit`, web tools given their
|
|
290
330
|
* query or URL, and the harness's duplicate `fileChange` events dropped. Cursor's tool durations
|
|
291
331
|
* come from its runner's own `tool_call` events, because tool results are only flushed at the
|
|
292
|
-
* end of a step.
|
|
332
|
+
* end of a step. Each tool event's `outputView` is `full` for the tools in `fullToolOutput` and
|
|
333
|
+
* for output an adapter read from where its CLI saved it, else `model`.
|
|
293
334
|
*/
|
|
294
|
-
function normalizeStream(parts, startedAt) {
|
|
335
|
+
function normalizeStream(parts, startedAt, options = {}) {
|
|
336
|
+
const fullByHarness = (tool) => options.fullToolOutput === "all" || options.fullToolOutput?.includes(tool) === true;
|
|
295
337
|
const timing = /* @__PURE__ */ new Map();
|
|
338
|
+
const replaced = /* @__PURE__ */ new Map();
|
|
296
339
|
for (const { ms, part } of parts) {
|
|
297
340
|
if (part.type !== "raw") continue;
|
|
298
341
|
const raw = record(part.rawValue);
|
|
342
|
+
const later = record(raw.toolOutput);
|
|
343
|
+
if (typeof later.toolCallId === "string" && "output" in later && toolOutputOf(later).view) replaced.set(later.toolCallId, {
|
|
344
|
+
...toolOutputOf(later),
|
|
345
|
+
output: later.output
|
|
346
|
+
});
|
|
299
347
|
const event = raw.type === "event" ? record(raw.event) : {};
|
|
300
348
|
const id = event.type === "tool_call" ? str(event.call_id) : "";
|
|
301
349
|
if (!id) continue;
|
|
@@ -327,7 +375,8 @@ function normalizeStream(parts, startedAt) {
|
|
|
327
375
|
};
|
|
328
376
|
const input = record(call.part.input);
|
|
329
377
|
const name = toolName(call.part);
|
|
330
|
-
const
|
|
378
|
+
const replacement = failed ? void 0 : replaced.get(id);
|
|
379
|
+
const raw = failed ? errorText(end.part.error) : replacement ? replacement.output : end.part.output;
|
|
331
380
|
const subagent = str(call.part.subagent);
|
|
332
381
|
const structured = record(end.part.structured);
|
|
333
382
|
const output = subagent && (name === "edit" || name === "write") && !failed ? typeof structured.filePath === "string" ? {
|
|
@@ -380,6 +429,7 @@ function normalizeStream(parts, startedAt) {
|
|
|
380
429
|
});
|
|
381
430
|
return;
|
|
382
431
|
}
|
|
432
|
+
const said = replacement ?? adapterToolOutput(end.part.providerMetadata);
|
|
383
433
|
out.push({
|
|
384
434
|
at: call.ms,
|
|
385
435
|
event: {
|
|
@@ -389,6 +439,8 @@ function normalizeStream(parts, startedAt) {
|
|
|
389
439
|
tool: name,
|
|
390
440
|
input: toolInput(call.part),
|
|
391
441
|
output,
|
|
442
|
+
outputView: said.view ?? (fullByHarness(name) ? "full" : "model"),
|
|
443
|
+
...said.bytes !== void 0 ? { outputTruncated: { bytes: said.bytes } } : {},
|
|
392
444
|
ok: !failed
|
|
393
445
|
}
|
|
394
446
|
});
|
|
@@ -416,13 +468,16 @@ function normalizeStream(parts, startedAt) {
|
|
|
416
468
|
}
|
|
417
469
|
break;
|
|
418
470
|
case "text-start":
|
|
419
|
-
case "reasoning-start":
|
|
471
|
+
case "reasoning-start": {
|
|
472
|
+
const recovered = part.type === "reasoning-start" ? recoveredTiming(part) : void 0;
|
|
420
473
|
blocks.set(str(part.id), {
|
|
421
474
|
type: part.type === "text-start" ? "text" : "reasoning",
|
|
422
|
-
ms,
|
|
423
|
-
text: ""
|
|
475
|
+
ms: recovered ? Math.max(0, ms - recovered.startedBeforeMs) : ms,
|
|
476
|
+
text: "",
|
|
477
|
+
...recovered ? { durationMs: recovered.durationMs } : {}
|
|
424
478
|
});
|
|
425
479
|
break;
|
|
480
|
+
}
|
|
426
481
|
case "text-delta":
|
|
427
482
|
case "reasoning-delta": {
|
|
428
483
|
const block = blocks.get(str(part.id));
|
|
@@ -439,7 +494,7 @@ function normalizeStream(parts, startedAt) {
|
|
|
439
494
|
...stamp(block.ms),
|
|
440
495
|
type: block.type,
|
|
441
496
|
text: block.text,
|
|
442
|
-
durationMs: ms - block.ms
|
|
497
|
+
durationMs: block.durationMs !== void 0 ? block.durationMs : ms - block.ms
|
|
443
498
|
}
|
|
444
499
|
});
|
|
445
500
|
break;
|
|
@@ -548,14 +603,16 @@ function normalizeStream(parts, startedAt) {
|
|
|
548
603
|
});
|
|
549
604
|
return out.sort((a, b) => a.at - b.at).map((e) => e.event);
|
|
550
605
|
}
|
|
606
|
+
/** Whether a harness's reasoning blocks carry their time: streamed, or recovered with it. */
|
|
607
|
+
const timesReasoning = (delivery) => delivery === "streamed" || delivery === "recovered";
|
|
551
608
|
/**
|
|
552
609
|
* A turn's reasoning time, from the reasoning blocks the harness streamed; null when it can't be
|
|
553
610
|
* known. A harness whose reasoning arrives whole (`whole`) or not at all (`none`) can't time it.
|
|
554
|
-
* A streamed turn with no reasoning block has spent none only when its usage says it
|
|
555
|
-
* reasoning tokens; otherwise the model may have reasoned without the harness showing it.
|
|
611
|
+
* A streamed or recovered turn with no reasoning block has spent none only when its usage says it
|
|
612
|
+
* used no reasoning tokens; otherwise the model may have reasoned without the harness showing it.
|
|
556
613
|
*/
|
|
557
614
|
function reasoningTime(delivery, events, usage) {
|
|
558
|
-
if (delivery
|
|
615
|
+
if (!timesReasoning(delivery)) return null;
|
|
559
616
|
const blocks = events.flatMap((e) => e.type === "reasoning" ? [e.durationMs] : []);
|
|
560
617
|
if (blocks.length === 0) return usage?.reasoning === 0 ? 0 : null;
|
|
561
618
|
if (blocks.some((ms) => ms === null)) return null;
|
|
@@ -583,17 +640,49 @@ function traceStreamPart(part, ms, tools) {
|
|
|
583
640
|
debugLog("harness", `tool ${call?.name ?? p.toolName ?? "tool"} ${how}${took} (${p.toolCallId})`);
|
|
584
641
|
} else if (p.type === "error") debugLog("harness", `stream error: ${errorText(p.error)}`);
|
|
585
642
|
}
|
|
586
|
-
/**
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
643
|
+
/**
|
|
644
|
+
* Each turn's cost from what the harness reports on its finish: a session's running total, as
|
|
645
|
+
* Claude Code's is, less what its earlier turns reported. A total that fell, or a session other
|
|
646
|
+
* than the attempt's first, can't be split into turns, so that turn has no reported cost; nor
|
|
647
|
+
* has one the harness put at $0, which it says for a model its own prices don't list.
|
|
648
|
+
*/
|
|
649
|
+
function turnCosts(reported, source) {
|
|
650
|
+
let session;
|
|
651
|
+
let before = 0;
|
|
652
|
+
return (harnessMetadata) => {
|
|
653
|
+
const said = reported?.(harnessMetadata);
|
|
654
|
+
if (!said) return void 0;
|
|
655
|
+
let usd = said.usd;
|
|
656
|
+
if (said.session !== void 0) {
|
|
657
|
+
const same = session === void 0 || session === said.session;
|
|
658
|
+
const fell = said.usd < before;
|
|
659
|
+
session ??= said.session;
|
|
660
|
+
if (!same) return void 0;
|
|
661
|
+
usd = Math.round((said.usd - before) * 1e9) / 1e9;
|
|
662
|
+
before = said.usd;
|
|
663
|
+
if (fell) return void 0;
|
|
664
|
+
}
|
|
665
|
+
return usd > 0 ? {
|
|
666
|
+
usd,
|
|
667
|
+
basis: "reported",
|
|
668
|
+
source
|
|
669
|
+
} : void 0;
|
|
670
|
+
};
|
|
671
|
+
}
|
|
672
|
+
/** How long reading a CLI's stored usage gets, past which the turn keeps none. */
|
|
673
|
+
const STORED_USAGE_MS = 5e3;
|
|
674
|
+
/** An adapter's usage, in the AI SDK's provider shape, as reeval's; null when it counts nothing. */
|
|
675
|
+
function harnessUsage(usage) {
|
|
676
|
+
const { inputTokens: input, outputTokens: output } = usage;
|
|
677
|
+
if (input.total === void 0 && output.total === void 0) return null;
|
|
678
|
+
const cacheRead = input.cacheRead ?? 0;
|
|
679
|
+
const cacheWrite = input.cacheWrite ?? 0;
|
|
591
680
|
return {
|
|
592
|
-
input:
|
|
593
|
-
output:
|
|
681
|
+
input: input.noCache ?? Math.max((input.total ?? 0) - cacheRead - cacheWrite, 0),
|
|
682
|
+
output: output.total ?? 0,
|
|
594
683
|
cacheRead,
|
|
595
684
|
cacheWrite,
|
|
596
|
-
reasoning:
|
|
685
|
+
reasoning: output.reasoning ?? null
|
|
597
686
|
};
|
|
598
687
|
}
|
|
599
688
|
/**
|
|
@@ -686,6 +775,7 @@ function createAgentHarness(descriptor, sandbox, vertex) {
|
|
|
686
775
|
const auth = agentAuth(catalog);
|
|
687
776
|
let servedModel = null;
|
|
688
777
|
let finishMetadata;
|
|
778
|
+
const turnCost = turnCosts(catalog.reportedCost, descriptor.kind);
|
|
689
779
|
const servers = agentMcp(catalog, mcp, plugins);
|
|
690
780
|
const model = catalog.modelId?.(descriptor.model) ?? descriptor.model;
|
|
691
781
|
const vertex = catalog.adc ? await writeAdc(sandbox, catalog.adc) : void 0;
|
|
@@ -737,21 +827,54 @@ function createAgentHarness(descriptor, sandbox, vertex) {
|
|
|
737
827
|
if (signal.aborted) throw error;
|
|
738
828
|
const message = error.message;
|
|
739
829
|
const hint = authHint(catalog, message, refusedSince(0));
|
|
740
|
-
throw new InfraError(`${hint ? `${hint}. ` : ""}${catalog.name} harness failed to start: ${message}
|
|
830
|
+
throw new InfraError(`${hint ? `${hint}. ` : ""}${catalog.name} harness failed to start: ${message}`, { transient: !hint && error.transient === true });
|
|
741
831
|
}
|
|
832
|
+
const network = sandbox.network;
|
|
833
|
+
let usageMark;
|
|
834
|
+
const storedUsage = async (since) => {
|
|
835
|
+
const store = await catalog.storedUsage?.();
|
|
836
|
+
if (!store) return null;
|
|
837
|
+
const command = store.command({
|
|
838
|
+
since,
|
|
839
|
+
...usageMark ? { mark: usageMark } : {}
|
|
840
|
+
});
|
|
841
|
+
const read = await Promise.resolve(network.run({
|
|
842
|
+
command,
|
|
843
|
+
abortSignal: AbortSignal.timeout(STORED_USAGE_MS)
|
|
844
|
+
})).then(({ exitCode, stdout, stderr }) => {
|
|
845
|
+
if (exitCode !== 0) throw new Error(`exit ${exitCode}: ${stderr.trim().slice(-300)}`);
|
|
846
|
+
return store.parse(stdout);
|
|
847
|
+
}).catch((error) => {
|
|
848
|
+
debugLog("harness", `reading ${catalog.name}'s stored usage failed: ${errorText(error)}`);
|
|
849
|
+
});
|
|
850
|
+
if (!read) {
|
|
851
|
+
debugLog("harness", `no usage in ${catalog.name}'s store since the turn began`);
|
|
852
|
+
return null;
|
|
853
|
+
}
|
|
854
|
+
if (read.mark) usageMark = read.mark;
|
|
855
|
+
const usage = harnessUsage(read.usage);
|
|
856
|
+
debugLog("harness", `usage read back from ${catalog.name}'s store: ${plural(read.calls, "model call")}${usage ? ` · ${usage.input} in · ${usage.output} out` : ""}`);
|
|
857
|
+
return usage;
|
|
858
|
+
};
|
|
859
|
+
let running;
|
|
860
|
+
let turn;
|
|
742
861
|
return {
|
|
743
862
|
async send(prompt, turnSignal) {
|
|
744
863
|
const startedAt = Date.now();
|
|
864
|
+
const current = { startedAt };
|
|
865
|
+
turn = current;
|
|
745
866
|
const parts = [];
|
|
746
867
|
const servedBefore = sandbox.servedRequests?.().length ?? 0;
|
|
747
868
|
const brokeredBefore = sandbox.brokeredStatuses?.().length ?? 0;
|
|
748
869
|
servedModel = null;
|
|
749
870
|
finishMetadata = void 0;
|
|
750
871
|
let stepUsage = null;
|
|
751
|
-
const
|
|
872
|
+
const steps = [];
|
|
873
|
+
const finish = async (text, status, extra = [], reported = stepUsage) => {
|
|
874
|
+
const usage = reported ?? await (current.salvaged ?? storedUsage(startedAt));
|
|
752
875
|
const statuses = sandbox.brokeredStatuses?.().slice(brokeredBefore) ?? [];
|
|
753
876
|
const refused = statuses.includes(401);
|
|
754
|
-
const events = [...normalizeStream(parts, startedAt), ...extra].map((e) => e.type === "reasoning" && catalog.reasoning
|
|
877
|
+
const events = [...normalizeStream(parts, startedAt, { ...catalog.fullToolOutput ? { fullToolOutput: catalog.fullToolOutput } : {} }), ...extra].map((e) => e.type === "reasoning" && !timesReasoning(catalog.reasoning) ? {
|
|
755
878
|
...e,
|
|
756
879
|
durationMs: null
|
|
757
880
|
} : e);
|
|
@@ -783,20 +906,30 @@ function createAgentHarness(descriptor, sandbox, vertex) {
|
|
|
783
906
|
status,
|
|
784
907
|
statuses
|
|
785
908
|
});
|
|
909
|
+
const stopped = rejected ? void 0 : providerStop({
|
|
910
|
+
errors: transcript.flatMap((e) => e.type === "error" ? [e.message] : []),
|
|
911
|
+
status,
|
|
912
|
+
statuses
|
|
913
|
+
});
|
|
914
|
+
const cost = turnCost(finishMetadata);
|
|
786
915
|
return {
|
|
787
916
|
text,
|
|
788
917
|
status,
|
|
789
918
|
usage,
|
|
919
|
+
...cost ? { cost } : {},
|
|
790
920
|
model: servedModel ?? sandbox.servedRequests?.().slice(servedBefore).at(-1) ?? null,
|
|
791
921
|
unreported: catalog.unreported?.(finishMetadata) ?? [],
|
|
922
|
+
steps,
|
|
792
923
|
reasoningMs: reasoningTime(catalog.reasoning, transcript, usage),
|
|
793
924
|
startedAt: new Date(startedAt).toISOString(),
|
|
794
925
|
durationMs: Date.now() - startedAt,
|
|
795
926
|
transcript,
|
|
796
927
|
...rejected ? { rejectedKey: rejected } : {},
|
|
928
|
+
...stopped ? { providerStop: stopped } : {},
|
|
797
929
|
...process.env.REEVAL_RAW_TRANSCRIPT ? { raw: parts } : {}
|
|
798
930
|
};
|
|
799
931
|
};
|
|
932
|
+
running = (note) => finish("", "aborted", [note]);
|
|
800
933
|
try {
|
|
801
934
|
const result = await agent.stream({
|
|
802
935
|
session,
|
|
@@ -811,9 +944,16 @@ function createAgentHarness(descriptor, sandbox, vertex) {
|
|
|
811
944
|
part
|
|
812
945
|
});
|
|
813
946
|
traceStreamPart(part, ms, tools);
|
|
814
|
-
if (part.type === "finish-step")
|
|
947
|
+
if (part.type === "finish-step") {
|
|
948
|
+
const step = tokenUsage(part.usage);
|
|
949
|
+
if (step) steps.push(step);
|
|
950
|
+
stepUsage = addUsage(stepUsage, step);
|
|
951
|
+
}
|
|
815
952
|
}
|
|
816
|
-
const
|
|
953
|
+
const reported = await result.totalUsage;
|
|
954
|
+
const total = tokenUsage(reported);
|
|
955
|
+
const mark = (await catalog.storedUsage?.())?.mark?.(reported.raw);
|
|
956
|
+
if (mark) usageMark = mark;
|
|
817
957
|
const status = turnSignal.aborted ? "aborted" : "completed";
|
|
818
958
|
return finish(await result.text, status, [], total ?? stepUsage);
|
|
819
959
|
} catch (error) {
|
|
@@ -826,8 +966,16 @@ function createAgentHarness(descriptor, sandbox, vertex) {
|
|
|
826
966
|
ts: new Date(startedAt + at).toISOString(),
|
|
827
967
|
atMs: at
|
|
828
968
|
}]);
|
|
969
|
+
} finally {
|
|
970
|
+
if (turn === current) turn = void 0;
|
|
829
971
|
}
|
|
830
972
|
},
|
|
973
|
+
stopped: (note) => running ? running(note) : Promise.reject(/* @__PURE__ */ new Error("No turn is running")),
|
|
974
|
+
salvage: async () => {
|
|
975
|
+
if (!turn) return;
|
|
976
|
+
turn.salvaged ??= storedUsage(turn.startedAt);
|
|
977
|
+
await turn.salvaged;
|
|
978
|
+
},
|
|
831
979
|
close: () => {
|
|
832
980
|
stopToken();
|
|
833
981
|
return session.destroy().catch(() => {});
|
|
@@ -837,4 +985,4 @@ function createAgentHarness(descriptor, sandbox, vertex) {
|
|
|
837
985
|
}
|
|
838
986
|
export { createAgentHarness };
|
|
839
987
|
|
|
840
|
-
//# sourceMappingURL=agent-
|
|
988
|
+
//# sourceMappingURL=agent-Dx-bDmlV.js.map
|