reeval 0.27.2 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +15 -15
- package/dist/{adapter-BuisZmtg.js → adapter-DpXw5J4g.js} +4 -4
- package/dist/{adapter-BuisZmtg.js.map → adapter-DpXw5J4g.js.map} +1 -1
- package/dist/agent-B1ndNDEu.js +272 -0
- package/dist/agent-B1ndNDEu.js.map +1 -0
- package/dist/agents-BP6F1c6y.d.ts +3 -0
- package/dist/agents-_3iNDMC1.js +400 -0
- package/dist/agents-_3iNDMC1.js.map +1 -0
- package/dist/{api-C8ZbQG7s.js → api-DIzSkDXZ.js} +2 -2
- package/dist/{api-C8ZbQG7s.js.map → api-DIzSkDXZ.js.map} +1 -1
- package/dist/{assemble-Co4noZvq.js → assemble-1bwPTJh1.js} +17 -17
- package/dist/{assemble-Co4noZvq.js.map → assemble-1bwPTJh1.js.map} +1 -1
- package/dist/assemble-B5nrfSs9.js +1 -0
- package/dist/{attach-DP3wc3A7.js → attach-D8MwHm3x.js} +5 -5
- package/dist/{attach-DP3wc3A7.js.map → attach-D8MwHm3x.js.map} +1 -1
- package/dist/{attempt-host-Bo8mbyuQ.js → attempt-host-lpJKcri8.js} +5 -5
- package/dist/{attempt-host-Bo8mbyuQ.js.map → attempt-host-lpJKcri8.js.map} +1 -1
- package/dist/{boat-obBLVWWQ.js → boat-DR71pK8X.js} +1 -1
- package/dist/{boat-BgdwW3_Q.js → boat-RFKQBduR.js} +8 -8
- package/dist/{boat-BgdwW3_Q.js.map → boat-RFKQBduR.js.map} +1 -1
- package/dist/{boat-images-CetjuF_c.js → boat-images-DvO8IPNz.js} +8 -8
- package/dist/{boat-images-CetjuF_c.js.map → boat-images-DvO8IPNz.js.map} +1 -1
- package/dist/{bundle-dK2eveab.js → bundle-Cnq5No9E.js} +20 -15
- package/dist/bundle-Cnq5No9E.js.map +1 -0
- package/dist/capture-L55SSOmN.js +1 -0
- package/dist/{capture-B8_Kx5R-.js → capture-sMi-R9PJ.js} +13 -13
- package/dist/{capture-B8_Kx5R-.js.map → capture-sMi-R9PJ.js.map} +1 -1
- package/dist/{cases-C6vnV-gn.js → cases-B_a7flE8.js} +3 -3
- package/dist/{cases-C6vnV-gn.js.map → cases-B_a7flE8.js.map} +1 -1
- package/dist/cases-D0-dA8q1.js +1 -0
- package/dist/{classify-BIJQ2Xqo.js → classify-gLl11KS8.js} +3 -3
- package/dist/{classify-BIJQ2Xqo.js.map → classify-gLl11KS8.js.map} +1 -1
- package/dist/{cli-eakKMZFR.js → cli-BscLXP46.js} +127 -114
- package/dist/cli-BscLXP46.js.map +1 -0
- package/dist/{cli-spec-DO-RQP-t.js → cli-spec-CYh8vLTE.js} +31 -9
- package/dist/cli-spec-CYh8vLTE.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/{cloud-BQbD0pT8.js → cloud-CcEcnyQh.js} +4 -4
- package/dist/{cloud-BQbD0pT8.js.map → cloud-CcEcnyQh.js.map} +1 -1
- package/dist/{cloud-files-DFLNNUdK.js → cloud-files-D_O9fbR-.js} +9 -9
- package/dist/{cloud-files-DFLNNUdK.js.map → cloud-files-D_O9fbR-.js.map} +1 -1
- package/dist/{cloud-project-D-5mk4oh.js → cloud-project-DLOjhvhB.js} +2 -2
- package/dist/{cloud-project-D-5mk4oh.js.map → cloud-project-DLOjhvhB.js.map} +1 -1
- package/dist/{cloud-project-CV3-AVEe.js → cloud-project-GRKSV6ye.js} +1 -1
- package/dist/{cloud-push-CjBRaPak.js → cloud-push-9n-MskNb.js} +1 -1
- package/dist/{cloud-push-bZL0TUQm.js → cloud-push-CXF92l8Z.js} +11 -11
- package/dist/{cloud-push-bZL0TUQm.js.map → cloud-push-CXF92l8Z.js.map} +1 -1
- package/dist/{cloud-run-BYvtsyiN.js → cloud-run-Bh7k0NT3.js} +1 -1
- package/dist/{cloud-run-CKcBID4J.js → cloud-run-D01Q6bI9.js} +39 -22
- package/dist/cloud-run-D01Q6bI9.js.map +1 -0
- package/dist/{cloud-secrets-OuKaQUlX.js → cloud-secrets-D75oFMh4.js} +75 -57
- package/dist/cloud-secrets-D75oFMh4.js.map +1 -0
- package/dist/cloud-secrets-IAkn5QAu.js +1 -0
- package/dist/{command-CPJuVTme.js → command-BnebWjYe.js} +7 -7
- package/dist/{command-CPJuVTme.js.map → command-BnebWjYe.js.map} +1 -1
- package/dist/{command-C6nA8Fwo.js → command-BvNM0r_3.js} +22 -22
- package/dist/{command-C6nA8Fwo.js.map → command-BvNM0r_3.js.map} +1 -1
- package/dist/{command-CU-A_HL3.js → command-C82cg19k.js} +2 -2
- package/dist/{command-CU-A_HL3.js.map → command-C82cg19k.js.map} +1 -1
- package/dist/{command-BC_zVYDg.js → command-CMP5tLCQ.js} +17 -17
- package/dist/{command-BC_zVYDg.js.map → command-CMP5tLCQ.js.map} +1 -1
- package/dist/{command-CcWqLUbc.js → command-du7_5bzq.js} +12 -12
- package/dist/{command-CcWqLUbc.js.map → command-du7_5bzq.js.map} +1 -1
- package/dist/{commands-CBbpjvzR.js → commands-CeC0GQb3.js} +1 -1
- package/dist/{commands-DzRyz-v9.js → commands-DvU8Kh5T.js} +18 -18
- package/dist/{commands-DzRyz-v9.js.map → commands-DvU8Kh5T.js.map} +1 -1
- package/dist/{config-Ozz1O7Dl.js → config-BUHRiR5M.js} +22 -25
- package/dist/config-BUHRiR5M.js.map +1 -0
- package/dist/{provenance-Cylt_YxT.d.ts → core-C2d0URWs.d.ts} +2 -2
- package/dist/{core-B3Qg8zwG.js → core-tpzE5NhW.js} +8 -8
- package/dist/{core-B3Qg8zwG.js.map → core-tpzE5NhW.js.map} +1 -1
- package/dist/{cost-DQ0ayy6h.js → cost-CSfMv_sp.js} +5 -2
- package/dist/cost-CSfMv_sp.js.map +1 -0
- package/dist/{cost-a_rUSdsf.js → cost-_uinO5Yg.js} +1 -1
- package/dist/{cost-Dp-xFGJ1.js → cost-rRhvzqJo.js} +2 -2
- package/dist/{cost-Dp-xFGJ1.js.map → cost-rRhvzqJo.js.map} +1 -1
- package/dist/{cursor-catalog-pRZm3jCP.js → cursor-catalog-jVblPRFf.js} +15 -5
- package/dist/cursor-catalog-jVblPRFf.js.map +1 -0
- package/dist/{daytona-Bg62QEsJ.js → daytona-B333cRQp.js} +10 -10
- package/dist/{daytona-Bg62QEsJ.js.map → daytona-B333cRQp.js.map} +1 -1
- package/dist/daytona-image-CPdhbuO3.js +1 -0
- package/dist/{daytona-image-DFsZlHaL.js → daytona-image-CZZ7KXhs.js} +4 -4
- package/dist/{daytona-image-DFsZlHaL.js.map → daytona-image-CZZ7KXhs.js.map} +1 -1
- package/dist/db-BPumcvp8.js +1 -0
- package/dist/{db-BQb7-_SX.js → db-t2g-ANto.js} +10 -10
- package/dist/{db-BQb7-_SX.js.map → db-t2g-ANto.js.map} +1 -1
- package/dist/{default-effort-DXPtXTsk.js → default-effort-DgRmptMy.js} +3 -2
- package/dist/default-effort-DgRmptMy.js.map +1 -0
- package/dist/{default-effort-C1Y0LOHo.js → default-effort-MTALS2Gg.js} +1 -1
- package/dist/{docker-BNZw-uhT.js → docker-8FHKmBSq.js} +1 -1
- package/dist/{docker-DzSu1y_e.js → docker-h8mYtGWF.js} +2 -2
- package/dist/{docker-DzSu1y_e.js.map → docker-h8mYtGWF.js.map} +1 -1
- package/dist/{doctor-TkGfyasW.js → doctor-duvpyGzv.js} +27 -26
- package/dist/doctor-duvpyGzv.js.map +1 -0
- package/dist/{env-8x1O8I1M.js → env-ARmqWMQ5.js} +3 -1
- package/dist/env-ARmqWMQ5.js.map +1 -0
- package/dist/{environment-C2i5ggoS.js → environment-ClhvHQzf.js} +4 -3
- package/dist/environment-ClhvHQzf.js.map +1 -0
- package/dist/{errors-xUi3AkBi.js → errors-BQxdlRgT.js} +2 -2
- package/dist/{errors-xUi3AkBi.js.map → errors-BQxdlRgT.js.map} +1 -1
- package/dist/{eval-source-CVx2dMCC.js → eval-source-B0BOsAj8.js} +5 -5
- package/dist/{eval-source-CVx2dMCC.js.map → eval-source-B0BOsAj8.js.map} +1 -1
- package/dist/{evaluate-Nnh0iPuo.d.ts → evaluate-Bhdj1cv0.d.ts} +1 -1
- package/dist/{evaluate-D9hHc00J.js → evaluate-CYk8DFen.js} +70 -55
- package/dist/evaluate-CYk8DFen.js.map +1 -0
- package/dist/events-C7tCnwxF.js +1 -0
- package/dist/{events-CQJ9VK-x.js → events-DrI9kjVf.js} +3 -3
- package/dist/{events-CQJ9VK-x.js.map → events-DrI9kjVf.js.map} +1 -1
- package/dist/{files-DxMU7Qa3.js → files-CLNAE1rM.js} +6 -6
- package/dist/{files-DxMU7Qa3.js.map → files-CLNAE1rM.js.map} +1 -1
- package/dist/files-CjabTKaR.js +1 -0
- package/dist/{files-C0k5nNUu.js → files-CmLgOH3w.js} +3 -3
- package/dist/{files-C0k5nNUu.js.map → files-CmLgOH3w.js.map} +1 -1
- package/dist/{files-CGzUGcN2.js → files-CsJWxr6c.js} +2 -2
- package/dist/{files-CGzUGcN2.js.map → files-CsJWxr6c.js.map} +1 -1
- package/dist/{files-FLbqNavk.js → files-DV9Tfusu.js} +7 -7
- package/dist/{files-FLbqNavk.js.map → files-DV9Tfusu.js.map} +1 -1
- package/dist/{files--mW-Ym3B.js → files-Dc2xVqx1.js} +1 -1
- package/dist/{files-lwByH3UH.js → files-DgRaHy0k.js} +5 -5
- package/dist/{files-lwByH3UH.js.map → files-DgRaHy0k.js.map} +1 -1
- package/dist/files-DrXNXtQR.js +1 -0
- package/dist/files-Ds_hi5Bh.js +1 -0
- package/dist/{files-f7mRGOM3.js → files-uFFwVJZG.js} +4 -4
- package/dist/{files-f7mRGOM3.js.map → files-uFFwVJZG.js.map} +1 -1
- package/dist/gateway.d.ts +1 -1
- package/dist/harness-Dcd3ffkY.js.map +1 -1
- package/dist/{harness-install-CTAsIRk9.js → harness-install-CnO_XOZX.js} +7 -5
- package/dist/harness-install-CnO_XOZX.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/{ids-D9QbaaVk.js → ids-cqB_4aeP.js} +3 -3
- package/dist/{ids-D9QbaaVk.js.map → ids-cqB_4aeP.js.map} +1 -1
- package/dist/{images-D-MMgo1d.js → images-DJhOleuR.js} +13 -13
- package/dist/{images-D-MMgo1d.js.map → images-DJhOleuR.js.map} +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +4 -4
- package/dist/{init-DJXSbI19.js → init-BWpCSCwu.js} +5 -5
- package/dist/{init-DJXSbI19.js.map → init-BWpCSCwu.js.map} +1 -1
- package/dist/init-DVPgHVjc.js +1 -0
- package/dist/{ink-list-BsPpwXuw.js → ink-list-Cu48wlm7.js} +2 -2
- package/dist/{ink-list-BsPpwXuw.js.map → ink-list-Cu48wlm7.js.map} +1 -1
- package/dist/{inputs-CTlr7diQ.js → inputs-BWGetwY1.js} +6 -6
- package/dist/{inputs-CTlr7diQ.js.map → inputs-BWGetwY1.js.map} +1 -1
- package/dist/internal/vm.d.ts +2 -2
- package/dist/{invocations-2vDCZ_7f.js → invocations-FUUh2r22.js} +3 -3
- package/dist/{invocations-2vDCZ_7f.js.map → invocations-FUUh2r22.js.map} +1 -1
- package/dist/{judge-3ABSMDa4.js → judge-rxVptB7O.js} +71 -24
- package/dist/judge-rxVptB7O.js.map +1 -0
- package/dist/judge.d.ts +3 -3
- package/dist/judge.js +1 -1
- package/dist/{judgments-CVCUWT-g.js → judgments-8zHsGW8o.js} +2 -2
- package/dist/{judgments-CVCUWT-g.js.map → judgments-8zHsGW8o.js.map} +1 -1
- package/dist/{kinds-DhH1c5u6.js → kinds-DHuWeXVE.js} +7 -7
- package/dist/{kinds-DhH1c5u6.js.map → kinds-DHuWeXVE.js.map} +1 -1
- package/dist/{labels-Dtyf5-pd.js → labels-CbwHtEHq.js} +2 -3
- package/dist/labels-CbwHtEHq.js.map +1 -0
- package/dist/{layout-ChWMCz4x.js → layout-zk3wcJ-D.js} +4 -4
- package/dist/{layout-ChWMCz4x.js.map → layout-zk3wcJ-D.js.map} +1 -1
- package/dist/{link-TH-DivdC.js → link-CAKz8dlf.js} +3 -3
- package/dist/{link-TH-DivdC.js.map → link-CAKz8dlf.js.map} +1 -1
- package/dist/{list-DSm2TZds.js → list-B_bnCkKF.js} +7 -7
- package/dist/{list-DSm2TZds.js.map → list-B_bnCkKF.js.map} +1 -1
- package/dist/{live-D3zHauJd.js → live-BXjhPeBS.js} +5 -5
- package/dist/{live-D3zHauJd.js.map → live-BXjhPeBS.js.map} +1 -1
- package/dist/{live-YXsT5z0d.js → live-CVlGZ5uP.js} +2 -2
- package/dist/{live-YXsT5z0d.js.map → live-CVlGZ5uP.js.map} +1 -1
- package/dist/{machine-service-Bs-09eUN.js → machine-service-CkNS0EDU.js} +3 -3
- package/dist/{machine-service-Bs-09eUN.js.map → machine-service-CkNS0EDU.js.map} +1 -1
- package/dist/{machine-service-DeE4-ueU.js → machine-service-SvzL_fG2.js} +1 -1
- package/dist/{machines-BxWMvCkU.js → machines-HP7b-Xu0.js} +23 -23
- package/dist/{machines-BxWMvCkU.js.map → machines-HP7b-Xu0.js.map} +1 -1
- package/dist/{markdown-COts-jxf.js → markdown-aVaKa5tB.js} +2 -2
- package/dist/{markdown-COts-jxf.js.map → markdown-aVaKa5tB.js.map} +1 -1
- package/dist/{model-U9bZgClN.js → model-CsOxcfcP.js} +25 -17
- package/dist/model-CsOxcfcP.js.map +1 -0
- package/dist/model-DHHW4Eva.js +1 -0
- package/dist/{model-WiM6J5cw.js → model-DmPDqMOT.js} +6 -5
- package/dist/model-DmPDqMOT.js.map +1 -0
- package/dist/{model-B89uRS9X.js → model-EiXnyEi4.js} +13 -13
- package/dist/model-EiXnyEi4.js.map +1 -0
- package/dist/model-Yu4XAseh.js +1 -0
- package/dist/{node-eEX-KpkO.js → node-ChfH6UTx.js} +4 -4
- package/dist/{node-eEX-KpkO.js.map → node-ChfH6UTx.js.map} +1 -1
- package/dist/{openrouter-models-B1sIOTD2.js → openrouter-models-D19R699I.js} +7 -8
- package/dist/openrouter-models-D19R699I.js.map +1 -0
- package/dist/options-BavfXTnZ.js +1 -0
- package/dist/{patch-CudvhNHJ.js → patch-CXAiVJJb.js} +2 -2
- package/dist/{patch-CudvhNHJ.js.map → patch-CXAiVJJb.js.map} +1 -1
- package/dist/pi-providers-C7r3J259.js +1 -0
- package/dist/{plan-CqBiH_vR.js → plan-XPBl8wPX.js} +10 -10
- package/dist/{plan-CqBiH_vR.js.map → plan-XPBl8wPX.js.map} +1 -1
- package/dist/{plugins-4DkJ-SSG.js → plugins-CVb68Csl.js} +6 -15
- package/dist/plugins-CVb68Csl.js.map +1 -0
- package/dist/{preflight-DshE4siP.js → preflight-D8tP_D1t.js} +9 -9
- package/dist/{preflight-DshE4siP.js.map → preflight-D8tP_D1t.js.map} +1 -1
- package/dist/{preflight-DTUzpuHa.js → preflight-DXXJrweW.js} +4 -4
- package/dist/{preflight-DTUzpuHa.js.map → preflight-DXXJrweW.js.map} +1 -1
- package/dist/preflight-DcUhwxb9.js +1 -0
- package/dist/{preflight-VrtpmE-m.js → preflight-XTBtCUST.js} +1 -1
- package/dist/{prepare-neo93PDB.js → prepare-BfkBJx-r.js} +11 -8
- package/dist/prepare-BfkBJx-r.js.map +1 -0
- package/dist/{prepare-B8mcpoSU.js → prepare-DncZcdrN.js} +2 -2
- package/dist/{prices-7Av-R8LY.js → prices-DZa1s6oE.js} +2 -2
- package/dist/{prices-7Av-R8LY.js.map → prices-DZa1s6oE.js.map} +1 -1
- package/dist/prices-DhqfDLI0.js +1 -0
- package/dist/{provenance-DH-yi6D5.js → provenance-BKmmc54i.js} +4 -4
- package/dist/{provenance-DH-yi6D5.js.map → provenance-BKmmc54i.js.map} +1 -1
- package/dist/provider-CImogcxo.js +1 -0
- package/dist/{provider-Cn7lmECy.js → provider-KoKRkRYc.js} +2 -2
- package/dist/{provider-Cn7lmECy.js.map → provider-KoKRkRYc.js.map} +1 -1
- package/dist/{providers-CkS7-mTp.js → providers-B1rFOe9J.js} +1 -1
- package/dist/{providers-CjS_fnib.js → providers-CDxtyguj.js} +26 -26
- package/dist/{providers-CjS_fnib.js.map → providers-CDxtyguj.js.map} +1 -1
- package/dist/{query-B9eLoFkA.js → query-Bb9N_XYo.js} +2 -2
- package/dist/{query-B9eLoFkA.js.map → query-Bb9N_XYo.js.map} +1 -1
- package/dist/query-CIn4l3KN.js +1 -0
- package/dist/{questions-OKDjvm6B.js → questions-B9CvSlT6.js} +6 -6
- package/dist/{questions-OKDjvm6B.js.map → questions-B9CvSlT6.js.map} +1 -1
- package/dist/{recipe-CRoyNF64.js → recipe-COFxUo3D.js} +3 -3
- package/dist/{recipe-CRoyNF64.js.map → recipe-COFxUo3D.js.map} +1 -1
- package/dist/{record-BWy9s9CV.js → record-CB1A6v0V.js} +2 -2
- package/dist/{record-BWy9s9CV.js.map → record-CB1A6v0V.js.map} +1 -1
- package/dist/{remarks-UZpf9-8r.js → remarks-Cf6sylYA.js} +10 -10
- package/dist/{remarks-UZpf9-8r.js.map → remarks-Cf6sylYA.js.map} +1 -1
- package/dist/{remarks-BbMdB_8I.js → remarks-vofg7p1p.js} +3 -3
- package/dist/{remarks-BbMdB_8I.js.map → remarks-vofg7p1p.js.map} +1 -1
- package/dist/{remote-Dr1gXMTr.js → remote-CiT9JKxm.js} +8 -7
- package/dist/remote-CiT9JKxm.js.map +1 -0
- package/dist/renderer/ascii.d.ts +1 -1
- package/dist/renderer/markdown.d.ts +1 -1
- package/dist/renderer/markdown.js +1 -1
- package/dist/{report-B-kavICs.js → report-BQTwKMyw.js} +3 -3
- package/dist/{report-B-kavICs.js.map → report-BQTwKMyw.js.map} +1 -1
- package/dist/{report-inputs-UUAME7ww.js → report-inputs-CfN15NUr.js} +6 -6
- package/dist/{report-inputs-UUAME7ww.js.map → report-inputs-CfN15NUr.js.map} +1 -1
- package/dist/{resolve-ihtFbkZH.js → resolve-C74KVA_E.js} +27 -17
- package/dist/resolve-C74KVA_E.js.map +1 -0
- package/dist/{resolve-C_UGu3ls.js → resolve-Cb2dNF75.js} +2 -2
- package/dist/{resolve-C_UGu3ls.js.map → resolve-Cb2dNF75.js.map} +1 -1
- package/dist/resolve-Dt5W-3CX.js +1 -0
- package/dist/{results-yZWhaeav.js → results-DFfbDGo8.js} +6 -6
- package/dist/{results-yZWhaeav.js.map → results-DFfbDGo8.js.map} +1 -1
- package/dist/{resume-Ds4K9mQc.js → resume-Upez3P-w.js} +6 -6
- package/dist/{resume-Ds4K9mQc.js.map → resume-Upez3P-w.js.map} +1 -1
- package/dist/{retry-Cjf40USW.js → retry-CnNc6eFZ.js} +5 -5
- package/dist/{retry-Cjf40USW.js.map → retry-CnNc6eFZ.js.map} +1 -1
- package/dist/routes-BKMw8O4e.js +629 -0
- package/dist/routes-BKMw8O4e.js.map +1 -0
- package/dist/{run-C_kA5Jzh.js → run-DwLQh41-.js} +3 -3
- package/dist/{run-C_kA5Jzh.js.map → run-DwLQh41-.js.map} +1 -1
- package/dist/{runtime-D1gU5fAJ.js → runtime-Ckm-qUYw.js} +6 -6
- package/dist/{runtime-D1gU5fAJ.js.map → runtime-Ckm-qUYw.js.map} +1 -1
- package/dist/sandbox/boat.d.ts +2 -2
- package/dist/sandbox/boat.js +1 -1
- package/dist/sandbox/daytona.d.ts +2 -2
- package/dist/sandbox/daytona.js +1 -1
- package/dist/{sandbox-Br9klmXq.js → sandbox-CIiuaavT.js} +2 -2
- package/dist/{sandbox-Br9klmXq.js.map → sandbox-CIiuaavT.js.map} +1 -1
- package/dist/sandbox.d.ts +3 -3
- package/dist/sandbox.js +1 -1
- package/dist/secrets-file-BFJSaPHI.js +169 -0
- package/dist/secrets-file-BFJSaPHI.js.map +1 -0
- package/dist/{server-2cICWjvT.js → server-BrBZ0_xA.js} +6 -6
- package/dist/{server-2cICWjvT.js.map → server-BrBZ0_xA.js.map} +1 -1
- package/dist/server-CzilMQ_m.js +1 -0
- package/dist/shared-7f8bQJP4.d.ts +2 -0
- package/dist/{skill-dirs-B7o83G5I.js → skill-dirs-DaHQ463p.js} +2 -2
- package/dist/skill-dirs-DaHQ463p.js.map +1 -0
- package/dist/{source-CFimUGjU.js → source-CfeQ4DL4.js} +2 -2
- package/dist/{source-CFimUGjU.js.map → source-CfeQ4DL4.js.map} +1 -1
- package/dist/source-DqFPDDZU.js +1 -0
- package/dist/{spans-BshoUrAD.js → spans-DUexyi3E.js} +2 -2
- package/dist/spans-DUexyi3E.js.map +1 -0
- package/dist/{src-ZaP7-0pi.js → src-B0ov9X7G.js} +1 -1
- package/dist/{src-CEValY4l.js → src-B3sBDfG2.js} +157 -114
- package/dist/src-B3sBDfG2.js.map +1 -0
- package/dist/src-BF1KIG3s.js +4 -0
- package/dist/{src-kjTrRKQJ.js → src-BRZvsbAC.js} +1 -1
- package/dist/{src-DP-gPQ5B.js → src-CJ-VZA2l.js} +1 -1
- package/dist/{src-B8t-GsRm.js → src-CMsTDH9Y.js} +1 -1
- package/dist/src-Cn5H3izr.js +3079 -0
- package/dist/src-Cn5H3izr.js.map +1 -0
- package/dist/{src-7kshnPZC.js → src-D2ILO3ov.js} +1 -1
- package/dist/{src-CRZg669b.js → src-DIk7y1HM.js} +1 -1
- package/dist/src-hPuCF6xC.js +1 -0
- package/dist/{src-B5r1d23Y.js → src-o0VTS7Rt.js} +5 -2
- package/dist/{src-B5r1d23Y.js.map → src-o0VTS7Rt.js.map} +1 -1
- package/dist/src-zu03lgt2.js +2 -0
- package/dist/ssh-pool-CYCX_JbC.js +1 -0
- package/dist/{ssh-pool-CANmGvdx.js → ssh-pool-CpPNPvOK.js} +7 -7
- package/dist/{ssh-pool-CANmGvdx.js.map → ssh-pool-CpPNPvOK.js.map} +1 -1
- package/dist/{start-D-xFuv0f.js → start-CCbSo9Hw.js} +13 -6
- package/dist/start-CCbSo9Hw.js.map +1 -0
- package/dist/startup-YENEnSa_.js +16 -0
- package/dist/startup-YENEnSa_.js.map +1 -0
- package/dist/{state-DcuHVqLB.js → state-BPIvAS8P.js} +5 -5
- package/dist/{state-DcuHVqLB.js.map → state-BPIvAS8P.js.map} +1 -1
- package/dist/state-Cqcd4_5G.js +1 -0
- package/dist/store/local.d.ts +1 -1
- package/dist/store/s3.d.ts +1 -1
- package/dist/{studio-share-qopKtSPe.js → studio-share-NQDwAWR_.js} +6 -6
- package/dist/{studio-share-qopKtSPe.js.map → studio-share-NQDwAWR_.js.map} +1 -1
- package/dist/suite-BvIl9-41.js +2 -0
- package/dist/{suite-DBbQs15H.js → suite-Bz0uCwd3.js} +4 -4
- package/dist/{suite-DBbQs15H.js.map → suite-Bz0uCwd3.js.map} +1 -1
- package/dist/{summary-CtTtjh3v.js → summary-Bx1qyFwL.js} +8 -8
- package/dist/{summary-CtTtjh3v.js.map → summary-Bx1qyFwL.js.map} +1 -1
- package/dist/{sync-DcFcjYLX.js → sync-BeiDYfR6.js} +6 -6
- package/dist/{sync-DcFcjYLX.js.map → sync-BeiDYfR6.js.map} +1 -1
- package/dist/{tart-CgW8e9RN.d.ts → tart-Brv7Huzv.d.ts} +2 -2
- package/dist/{tart-bake-BksL4gS9.js → tart-bake-Cdk8YaPG.js} +1 -1
- package/dist/{tart-bake-BEzKMVWv.js → tart-bake-Dhj-q9AS.js} +7 -7
- package/dist/{tart-bake-BEzKMVWv.js.map → tart-bake-Dhj-q9AS.js.map} +1 -1
- package/dist/{tart-cli-C05FdPQV.js → tart-cli-B1qZ4EPU.js} +2 -2
- package/dist/{tart-cli-C05FdPQV.js.map → tart-cli-B1qZ4EPU.js.map} +1 -1
- package/dist/{tart-cli-DVI2SWz4.js → tart-cli-B7YKODI8.js} +1 -1
- package/dist/{tart-D4VLu2qF.js → tart-ggAmpJpj.js} +7 -7
- package/dist/{tart-D4VLu2qF.js.map → tart-ggAmpJpj.js.map} +1 -1
- package/dist/{tart-hosts-GDk-lRnT.js → tart-hosts-BaeyUzzq.js} +4 -4
- package/dist/{tart-hosts-GDk-lRnT.js.map → tart-hosts-BaeyUzzq.js.map} +1 -1
- package/dist/{tart-hosts-DESXVcNv.js → tart-hosts-D0tEFNJz.js} +1 -1
- package/dist/{tart-images-CLpWt7Mm.js → tart-images-DOboVDfH.js} +2 -2
- package/dist/{tart-images-CLpWt7Mm.js.map → tart-images-DOboVDfH.js.map} +1 -1
- package/dist/testing.d.ts +3 -3
- package/dist/testing.js +4 -4
- package/dist/{tool-class-CB-m4PiF.js → tool-class-DzpZJJWK.js} +2 -2
- package/dist/{tool-class-CB-m4PiF.js.map → tool-class-DzpZJJWK.js.map} +1 -1
- package/dist/{trace-CJQLlAbB.js → trace-BY_rkR5K.js} +2 -2
- package/dist/{trace-CJQLlAbB.js.map → trace-BY_rkR5K.js.map} +1 -1
- package/dist/{types-Buut5yGe.js → types-ByoVW0_V.js} +3 -8
- package/dist/types-ByoVW0_V.js.map +1 -0
- package/dist/{types-CYz4dqAd.d.ts → types-CseRWbr7.d.ts} +1 -1
- package/dist/{types-BZaItiui.d.ts → types-DtRJb-jv.d.ts} +203 -199
- package/dist/{ui-Df1vnh7L.js → ui-DsGOlh5c.js} +5 -5
- package/dist/{ui-Df1vnh7L.js.map → ui-DsGOlh5c.js.map} +1 -1
- package/dist/{usage-CwIRPPZ9.js → usage-Dg_IyU0o.js} +3 -5
- package/dist/usage-Dg_IyU0o.js.map +1 -0
- package/dist/vercel-models-DZxE5QJe.js +14 -0
- package/dist/vercel-models-DZxE5QJe.js.map +1 -0
- package/dist/vertex-C7IbGLJg.js +2 -0
- package/dist/{vertex-DQQGhUn7.js → vertex-Plo5WUxj.js} +20 -91
- package/dist/vertex-Plo5WUxj.js.map +1 -0
- package/dist/view/index.d.ts +3 -3
- package/dist/view/index.js +12 -12
- package/dist/view/report/index.d.ts +3 -3
- package/dist/view/report/index.js +3 -3
- package/dist/views-BrNK-NV6.js +2 -0
- package/dist/{views-BP8ck_JR.js → views-Dv-m2rqd.js} +6 -6
- package/dist/{views-BP8ck_JR.js.map → views-Dv-m2rqd.js.map} +1 -1
- package/dist/{vm-container-Bx3bIlDn.js → vm-container-DOkT5Iki.js} +6 -6
- package/dist/{vm-container-Bx3bIlDn.js.map → vm-container-DOkT5Iki.js.map} +1 -1
- package/dist/{vm-images-CimgmVk6.js → vm-images-BKGyBt9y.js} +13 -13
- package/dist/{vm-images-CimgmVk6.js.map → vm-images-BKGyBt9y.js.map} +1 -1
- package/dist/{worker-entry-CY_GX-KW.js → worker-entry-Dt0KPdT9.js} +3 -3
- package/dist/{worker-entry-CY_GX-KW.js.map → worker-entry-Dt0KPdT9.js.map} +1 -1
- package/dist/{workspace-2wz0BP-V.js → workspace-DZs1aV-E.js} +6 -6
- package/dist/{workspace-2wz0BP-V.js.map → workspace-DZs1aV-E.js.map} +1 -1
- package/dist/workspace-Dtfq6vqG.js +1 -0
- package/package.json +1 -1
- package/dist/agent-ofh_n1XS.js +0 -987
- package/dist/agent-ofh_n1XS.js.map +0 -1
- package/dist/agents-D4H9bKHD.d.ts +0 -2
- package/dist/agents-DGr0kCFD.js +0 -1408
- package/dist/agents-DGr0kCFD.js.map +0 -1
- package/dist/assemble-HOUMVnjQ.js +0 -1
- package/dist/bundle-dK2eveab.js.map +0 -1
- package/dist/capture-BEGBn9e9.js +0 -1
- package/dist/cases-Bsp-G-ZL.js +0 -1
- package/dist/cli-eakKMZFR.js.map +0 -1
- package/dist/cli-spec-DO-RQP-t.js.map +0 -1
- package/dist/cloud-run-CKcBID4J.js.map +0 -1
- package/dist/cloud-secrets-D_IiyGgA.js +0 -1
- package/dist/cloud-secrets-OuKaQUlX.js.map +0 -1
- package/dist/config-Ozz1O7Dl.js.map +0 -1
- package/dist/cost-DQ0ayy6h.js.map +0 -1
- package/dist/cursor-catalog-pRZm3jCP.js.map +0 -1
- package/dist/daytona-image-D8TFfSxM.js +0 -1
- package/dist/db-Djz9zixD.js +0 -1
- package/dist/default-effort-DXPtXTsk.js.map +0 -1
- package/dist/doctor-TkGfyasW.js.map +0 -1
- package/dist/env-8x1O8I1M.js.map +0 -1
- package/dist/environment-C2i5ggoS.js.map +0 -1
- package/dist/evaluate-D9hHc00J.js.map +0 -1
- package/dist/events-CwWpBkkc.js +0 -1
- package/dist/files-1PVfWWXB.js +0 -1
- package/dist/files-CsPOd60N.js +0 -1
- package/dist/files-DOz7Qr5r.js +0 -1
- package/dist/harness-install-CTAsIRk9.js.map +0 -1
- package/dist/init-DkTAnHaf.js +0 -1
- package/dist/judge-3ABSMDa4.js.map +0 -1
- package/dist/labels-Dtyf5-pd.js.map +0 -1
- package/dist/model-B89uRS9X.js.map +0 -1
- package/dist/model-BGc_y-K_.js +0 -1
- package/dist/model-Doy9vFnq.js +0 -1
- package/dist/model-U9bZgClN.js.map +0 -1
- package/dist/model-WiM6J5cw.js.map +0 -1
- package/dist/openrouter-models-B1sIOTD2.js.map +0 -1
- package/dist/options-BK7nMvou.js +0 -1
- package/dist/plugins-4DkJ-SSG.js.map +0 -1
- package/dist/preflight-CjR6YpXu.js +0 -1
- package/dist/prepare-neo93PDB.js.map +0 -1
- package/dist/prices-CZWSNosm.js +0 -1
- package/dist/provider-B24hZX95.js +0 -1
- package/dist/provider-stop-DjBMfZW5.js +0 -69
- package/dist/provider-stop-DjBMfZW5.js.map +0 -1
- package/dist/query-3PjR8hU2.js +0 -1
- package/dist/remote-Dr1gXMTr.js.map +0 -1
- package/dist/resolve-CKRG9k6Z.js +0 -1
- package/dist/resolve-ihtFbkZH.js.map +0 -1
- package/dist/routes-Cp4gAjBZ.js +0 -427
- package/dist/routes-Cp4gAjBZ.js.map +0 -1
- package/dist/secrets-file-B1Yj1WjD.js +0 -100
- package/dist/secrets-file-B1Yj1WjD.js.map +0 -1
- package/dist/server-CzTVzKWG.js +0 -1
- package/dist/shared-B3iLKAiu.d.ts +0 -1
- package/dist/skill-dirs-B7o83G5I.js.map +0 -1
- package/dist/source-CydZOoBv.js +0 -1
- package/dist/spans-BshoUrAD.js.map +0 -1
- package/dist/src-2MDgX3cP.js +0 -3
- package/dist/src-CEValY4l.js.map +0 -1
- package/dist/src-CH8ZNcTe.js +0 -1
- package/dist/src-Con14kZF.js +0 -2
- package/dist/ssh-pool-Du_BZWR1.js +0 -1
- package/dist/start-D-xFuv0f.js.map +0 -1
- package/dist/state-C007zNcM.js +0 -1
- package/dist/suite-BQNoCyRE.js +0 -2
- package/dist/types-Buut5yGe.js.map +0 -1
- package/dist/usage-CwIRPPZ9.js.map +0 -1
- package/dist/vercel-models-Ci33llXz.js +0 -21
- package/dist/vercel-models-Ci33llXz.js.map +0 -1
- package/dist/vertex-DQQGhUn7.js.map +0 -1
- package/dist/vertex-DtWe0Xf6.js +0 -1
- package/dist/views-_yc6K-9E.js +0 -2
- package/dist/workspace-CQxNEvkp.js +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.29.0](https://github.com/reeval-ai/reeval/compare/v0.28.0...v0.29.0) (2026-11-15)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* **cloud:** carry credential alternatives as secretChoices and plan question models ([#1001](https://github.com/reeval-ai/reeval/issues/1001)) ([01e6dc0](https://github.com/reeval-ai/reeval/commit/01e6dc0eebaf3469296cafa18f4a2d54ca9f79f1))
|
|
9
|
+
|
|
10
|
+
## [0.28.0](https://github.com/reeval-ai/reeval/compare/v0.27.2...v0.28.0) (2026-11-15)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### ⚠ BREAKING CHANGES
|
|
14
|
+
|
|
15
|
+
* **cli:** take reeval cloud provider set|delete <provider> and let the platform decide which exist ([#989](https://github.com/reeval-ai/reeval/issues/989))
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
* **agents:** add the agent() session API and host-side output ([#997](https://github.com/reeval-ai/reeval/issues/997)) ([44745c2](https://github.com/reeval-ai/reeval/commit/44745c2d5ac28fdbf59df486b571d1f3e7313158))
|
|
20
|
+
* **agents:** add the docker() runner and harness setup install ([#998](https://github.com/reeval-ai/reeval/issues/998)) ([ae3a500](https://github.com/reeval-ai/reeval/commit/ae3a500164d3417d4db16d75682e994e279f01db))
|
|
21
|
+
* **agents:** add the Runner interface and local() runner ([#985](https://github.com/reeval-ai/reeval/issues/985)) ([6b576b1](https://github.com/reeval-ai/reeval/commit/6b576b1a5889d221544d07364b1488d7b98038a8))
|
|
22
|
+
* **cloud:** add reeval cloud gateway set|delete <gateway> and list ([#992](https://github.com/reeval-ai/reeval/issues/992)) ([9f2170a](https://github.com/reeval-ai/reeval/commit/9f2170aa153dbc145fc13548633f5cb0172ae31b))
|
|
23
|
+
* **cloud:** stop a cloud run before upload when the project has no key for its gateway ([#1000](https://github.com/reeval-ai/reeval/issues/1000)) ([635dc80](https://github.com/reeval-ai/reeval/commit/635dc80f2fb1ca0a1b494632ccdaf42c974a7f6b))
|
|
24
|
+
* **judge:** let reeval's own model calls use a brokered key on its host ([#993](https://github.com/reeval-ai/reeval/issues/993)) ([c17f168](https://github.com/reeval-ai/reeval/commit/c17f168307f3ca84c934df1285b459333c32c231))
|
|
25
|
+
* **secrets:** fill held secrets through an external filler ([#1003](https://github.com/reeval-ai/reeval/issues/1003)) ([9f50631](https://github.com/reeval-ai/reeval/commit/9f50631778d8e3bc69e69e718611345416a62881))
|
|
26
|
+
* **secrets:** relay a held secret's requests, and broker one with no hosts where each use goes ([#995](https://github.com/reeval-ai/reeval/issues/995)) ([7412e2a](https://github.com/reeval-ai/reeval/commit/7412e2a38f49808314b1237c8e0aabc0f55569ae))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
### Bug Fixes
|
|
30
|
+
|
|
31
|
+
* **cli:** take reeval cloud provider set|delete <provider> and let the platform decide which exist ([#989](https://github.com/reeval-ai/reeval/issues/989)) ([681b4d6](https://github.com/reeval-ai/reeval/commit/681b4d60b52742e0d9731df595a06df664cb2c07))
|
|
32
|
+
|
|
3
33
|
## [0.27.2](https://github.com/reeval-ai/reeval/compare/v0.27.1...v0.27.2) (2026-11-14)
|
|
4
34
|
|
|
5
35
|
|
package/README.md
CHANGED
|
@@ -27,11 +27,11 @@ Evals are TypeScript files in `evals/`, next to the tool they test. Each run wri
|
|
|
27
27
|
|
|
28
28
|
Cursor ⋅ Claude Code ⋅ Codex ⋅ Antigravity ⋅ OpenCode ⋅ Goose ⋅ pi ⋅ Grok Build
|
|
29
29
|
|
|
30
|
-
- **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](https://github.com/reeval-ai/reeval/blob/v0.
|
|
31
|
-
- **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](https://github.com/reeval-ai/reeval/blob/v0.
|
|
32
|
-
- **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](https://github.com/reeval-ai/reeval/blob/v0.
|
|
33
|
-
- **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](https://github.com/reeval-ai/reeval/blob/v0.
|
|
34
|
-
- **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](https://github.com/reeval-ai/reeval/blob/v0.
|
|
30
|
+
- **Real agents**: each harness runs the agent itself, with its own tools, skills, and rules, as it works on a developer's machine ([Choose an agent](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/configuration/agents/index.mdx)).
|
|
31
|
+
- **Keys stay on the host**: each attempt starts in a fresh sandbox that holds only a placeholder key. reeval's egress gateway swaps in the real key on requests to that provider and nowhere else ([Pass secrets](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/configuration/environments/secrets.mdx)).
|
|
32
|
+
- **Compare a change**: one eval runs a baseline and its variants with the same prompt and checks, then prints whether each variant helped, made it worse, or did the same ([Measure lift against a baseline](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/results/compare/lift.mdx)).
|
|
33
|
+
- **Pinned environments**: an environment starts from a workspace, such as a git repository at a fixed ref, then adds files, setup steps, rules, skills, and MCP servers, so every attempt starts from the same state ([Define an environment](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/configuration/environments/index.mdx)).
|
|
34
|
+
- **History in Git**: recorded runs sit beside the code. `reeval query` reads them with SQL, and `reeval studio` shows each attempt's transcript, diff, and checks ([Record runs in Git](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/runs/recording.mdx)).
|
|
35
35
|
|
|
36
36
|
## Install
|
|
37
37
|
|
|
@@ -140,23 +140,23 @@ Environments load extensions from where teams already keep them:
|
|
|
140
140
|
|
|
141
141
|
## Examples
|
|
142
142
|
|
|
143
|
-
The recipes in [`docs/evaluations/recipes`](https://github.com/reeval-ai/reeval/blob/v0.
|
|
143
|
+
The recipes in [`docs/evaluations/recipes`](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/recipes) are complete evals for common jobs:
|
|
144
144
|
|
|
145
|
-
- [Measure a skill](https://github.com/reeval-ai/reeval/blob/v0.
|
|
146
|
-
- [Compare harnesses](https://github.com/reeval-ai/reeval/blob/v0.
|
|
147
|
-
- [Measure MCP servers](https://github.com/reeval-ai/reeval/blob/v0.
|
|
148
|
-
- [Check a recommendation](https://github.com/reeval-ai/reeval/blob/v0.
|
|
149
|
-
- [Fail a pull request](https://github.com/reeval-ai/reeval/blob/v0.
|
|
145
|
+
- [Measure a skill](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/recipes/measure-a-skill.mdx): with and without a skill, gated on a minimum lift.
|
|
146
|
+
- [Compare harnesses](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/recipes/compare-harnesses.mdx): one suite on several agents at once, with pass rate and effort per agent.
|
|
147
|
+
- [Measure MCP servers](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/recipes/measure-mcp-servers.mdx): check the agent calls the server's tools, and whether answers improve.
|
|
148
|
+
- [Check a recommendation](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/recipes/test-recommendations.mdx): ask which library to use, and let a judge decide whether the agent picks the right one.
|
|
149
|
+
- [Fail a pull request](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/evaluations/recipes/gate-pull-requests.mdx): run on every pull request and fail the check on a regression.
|
|
150
150
|
|
|
151
|
-
The [`examples/`](https://github.com/reeval-ai/reeval/blob/v0.
|
|
151
|
+
The [`examples/`](https://github.com/reeval-ai/reeval/blob/v0.29.0/examples) directory has 80 runnable projects, one idea each, from [`with-checks`](https://github.com/reeval-ai/reeval/blob/v0.29.0/examples/with-checks) (deterministic checks) and [`with-skill-lift`](https://github.com/reeval-ai/reeval/blob/v0.29.0/examples/with-skill-lift) (does a skill help?) to sandboxes, egress, judges, captures and real-world tasks. Most run on a fake agent with no keys or Docker; [`examples/README.md`](https://github.com/reeval-ai/reeval/blob/v0.29.0/examples/README.md) lists them all.
|
|
152
152
|
|
|
153
153
|
## Documentation
|
|
154
154
|
|
|
155
|
-
The manual in [`docs/`](https://github.com/reeval-ai/reeval/blob/v0.
|
|
155
|
+
The manual in [`docs/`](https://github.com/reeval-ai/reeval/blob/v0.29.0/docs/overview.mdx) covers harnesses, sandboxes, environments, secrets and egress, checks and judges, profiles, history, and every CLI command and flag.
|
|
156
156
|
|
|
157
157
|
## For coding agents
|
|
158
158
|
|
|
159
|
-
To write or update evals, load [`skills/reeval/SKILL.md`](https://github.com/reeval-ai/reeval/blob/v0.
|
|
159
|
+
To write or update evals, load [`skills/reeval/SKILL.md`](https://github.com/reeval-ai/reeval/blob/v0.29.0/skills/reeval/SKILL.md). For every command, flag, and exit code, run `npx reeval --help`.
|
|
160
160
|
|
|
161
161
|
To preview a run without starting sandboxes or spending tokens, run `npx reeval run --dry-run --json`. It prints the planned evals, cells, and attempts as one JSON document and exits 0.
|
|
162
162
|
|
|
@@ -166,4 +166,4 @@ To read past runs without calling a model, run `npx reeval query failing_attempt
|
|
|
166
166
|
|
|
167
167
|
## License
|
|
168
168
|
|
|
169
|
-
[Apache-2.0](https://github.com/reeval-ai/reeval/blob/v0.
|
|
169
|
+
[Apache-2.0](https://github.com/reeval-ai/reeval/blob/v0.29.0/LICENSE). Copyright 2026 Invertase Limited. The npm package ships third-party licenses and attributions in `THIRD_PARTY_NOTICES.md`. Questions about licensing: [oss@invertase.io](mailto:oss@invertase.io).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { l as relativeFile } from "./ids-
|
|
1
|
+
import { l as relativeFile } from "./ids-cqB_4aeP.js";
|
|
2
2
|
import { t as reevalJiti } from "./jiti-aNfS_lSj.js";
|
|
3
|
-
import { t as codeFrame } from "./errors-
|
|
3
|
+
import { t as codeFrame } from "./errors-BQxdlRgT.js";
|
|
4
4
|
import { l as ownFile } from "./root-Co0jhOT5.js";
|
|
5
5
|
import { existsSync, readFileSync } from "node:fs";
|
|
6
6
|
import { delimiter, dirname, join, resolve } from "node:path";
|
|
@@ -247,7 +247,7 @@ const LANGUAGE_ADAPTERS = [TYPESCRIPT_ADAPTER, {
|
|
|
247
247
|
extensions: [".eval.py"],
|
|
248
248
|
includeGlobs: ["evals/**/*.eval.py"],
|
|
249
249
|
remote: true,
|
|
250
|
-
load: (file, root) => import("./remote-
|
|
250
|
+
load: (file, root) => import("./remote-CiT9JKxm.js").then((m) => m.collectLanguageFile(file, root)),
|
|
251
251
|
runnerCommand: (root) => resolvePythonRunnerCommand(root),
|
|
252
252
|
setupHint: PYTHON_RUNNER_HINT
|
|
253
253
|
}];
|
|
@@ -273,4 +273,4 @@ function runnerEnv(root, adapter) {
|
|
|
273
273
|
}
|
|
274
274
|
export { originalText as a, originalStack as i, defaultIncludeGlobs as n, runnerEnv as r, adapterForFile as t };
|
|
275
275
|
|
|
276
|
-
//# sourceMappingURL=adapter-
|
|
276
|
+
//# sourceMappingURL=adapter-DpXw5J4g.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter-
|
|
1
|
+
{"version":3,"file":"adapter-DpXw5J4g.js","names":[],"sources":["../src/engine/loader.ts","../src/language-runner/python-runner.ts","../src/language-runner/adapter.ts"],"mappings":";;;;;;;;;;;AAYA,IAAa,gBAAb,cAAmC,MAAM;CACvC,OAAgB;AAClB;AAOA,MAAM,8BAAc,IAAI,IAAyB;AAEjD,MAAM,aACJ;;AAGF,eAAsB,aAAa,MAAc,MAAgC;CAC/E,cAAc,QAAQ,IAAI,GAAG,IAAI;CACjC,MAAM,UAAU;EAAE,aAAa;EAAM,SAAS;EAAO,YAAY;CAAK;CACtE,MAAM,OAAO,MAAM,WAAW,MAAM,OAAO;CAC3C,MAAM,OAAO,MAAM,WAAW,MAAM;EAClC,GAAG;EACH,YAAY,SAAS;GACnB,MAAM,OAAO,KAAK,UAAU,IAAI;GAChC,MAAM,SAAS,WAAW,KAAK,IAAI;GACnC,IAAI,KAAK,YAAY,SAAS,IAAI;IAChC,MAAM,MAAM,KAAK,MAAM,OAAO,KAAK,OAAO,IAAI,QAAQ,CAAC,CAAC,SAAS,MAAM,CAAC;IAExE,MAAM,QACJ,IAAI,KAAK,QAAQ,WAAW,GAAG,2FAC5B;IACL,YAAY,IAAI,KAAK,UAAU;KAC7B,KAAK,IAAI,WAAW,KAAK,KAAK,QAAQ;KACtC;KACA,QAAQ,KAAK;KACb;IACF,CAAC;GACH;GACA,OAAO,EAAE,KAAK;EAChB;CACF,CAAC;CACD,IAAI;EACF,OAAO,MAAM,KAAK,OAAO,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,YAAY,OAAO,IAAI,KAAK,cAAc,KAAK;CACvD;AACF;;AAGA,SAAS,cAAc,OAAyB;CAC9C,MAAM,IAAI;CACV,IAAI,GAAG,SAAS,oBAAoB,OAAO;CAC3C,MAAM,QAAQ,wDAAwD,KAAK,EAAE,OAAO;CACpF,IAAI,OAAO,EAAE,UAAU,GAAG,MAAM,GAAG,iBAAiB,MAAM;CAC1D,OAAO;AACT;;;;;AAMA,SAAgB,YAAY,OAAgB,MAAuC;CACjF,MAAM,SAAS,8CAA8C,KAAM,OAAiB,WAAW,EAAE;CACjG,IAAI,CAAC,QAAQ,OAAO,KAAA;CACpB,MAAM,GAAG,MAAM,MAAM,MAAM,UAAU;CAErC,MAAM,SAAS,KAAK,QAAQ,8BAA8B,EAAE;CAC5D,MAAM,KAAK;EAAE,MAAM,OAAO,IAAI;EAAG,QAAQ,OAAO,MAAM,IAAI;CAAE;CAC5D,IAAI,QAAQ;CACZ,IAAI;EACF,QAAQ,yBAAyB,UAAU,aAAa,MAAM,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC;CACpF,QAAQ,CAAC;CACT,uBAAO,IAAI,YACT,GAAG,OAAO,IAAI,aAAa,MAAM,IAAI,EAAE,GAAG,GAAG,KAAK,GAAG,GAAG,OAAO,GAAG,QAAQ,KAAK,UAAU,IAC3F;AACF;;AAGA,SAAgB,cAAc,OAAuB;CACnD,OAAO,MAAM,QACX,gDACC,OAAO,QAAgB,MAAc,MAAc,QAAgB,UAAkB;EACpF,MAAM,SAAS,YAAY,IAAI,IAAI;EACnC,MAAM,MAAM,OAAO,MAAM,IAAI,KAAK,SAAS,MAAO,QAAQ,SAAS,IAAK;EAExE,IAAI,CAAC,UAAU,MAAM,GAAG,OAAO;EAC/B,MAAM,KAAK,oBAAoB,OAAO,KAAK;GAAE,MAAM,OAAO,IAAI;GAAG,QAAQ;EAAI,CAAC;EAC9E,IAAI,CAAC,IAAI,OAAO;EAChB,MAAM,WAAW,GAAG;EACpB,OAAO,GAAG,SAAS,KAAK,GAAG,SAAS,GAAG,OAAO,MAAM,UAAU,GAAG,MAAgB,IAAI,IAAI;CAC3F,CACF;AACF;;;;;;;AAQA,SAAgB,aAAa,MAAc,MAAmC;CAC5E,MAAM,YAAY,SAAS,KAAA,IAAY,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;CAClE,KAAK,MAAM,UAAU,CAAC,GAAG,WAAW,GAAG,YAAY,OAAO,CAAC,GAAG;EAC5D,MAAM,KAAK,QAAQ,KAAK,QAAQ,IAAI,KAAK;EACzC,IAAI,CAAC,UAAU,KAAK,GAAG;EAGvB,MAAM,OAAO,KAAK,KAAK,SAAS;EAChC,IAAI,aAAa;EACjB,MAAM,MAAM,eAAe,QAAQ,IAAI;EACvC,OAAO,aAAa,MAAM,eAAe,QAAQ,aAAa,CAAC,MAAM,KAAK;EAC1E,OAAO,OAAO,OAAO,MAAM,eAAe,QAAQ,EAAE,GAAG,MAAM,OAAO,aAAa,CAAC;CACpF;AAEF;;AAGA,SAAS,eAAe,QAAqB,OAAuB;CAClE,MAAM,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,IAAI;CACpD,MAAM,SAAU,MAAM,GAAG,EAAE,CAAC,CAAY;CACxC,MAAM,KAAK,oBAAoB,OAAO,KAAK;EAAE,MAAM,MAAM;EAAQ;CAAO,CAAC;CAGzE,OADoB,OAAO,OAAO,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,QAAQ,KAAK,CACxD,CAAC,CAAC,QAAQ,GAAG,SAAS,IAAI,KAAK,SAAS,GAAG,CAAC,KAAK,IAAI,UAAU;AAClF;;;;;AAMA,SAAS,OAAO,MAAc,MAAc,QAAwB;CAElE,MAAM,OAAO,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,MAAM;CACpE,IAAI,KAAK,YAAY,KAAK,OAAO;CACjC,IAAI,QAAQ;CACZ,OAAO,QAAQ,KAAK,QAAQ,KAAK,KAAK,QAAQ,EAAY,GAAG;CAC7D,OAAO;AACT;;AAGA,SAAgB,cAAc,KAAa,MAAoB;CAC7D,IAAI;CACJ,IAAI;EACF,QAAQ,YAAY,GAAG;CACzB,SAAS,OAAO;EACd,MAAM,UAAW,MAAgB;EACjC,MAAM,UAAU,4BAA4B,KAAK,OAAO;EACxD,MAAM,IAAI,cACR,UACI,4BAA4B,QAAQ,GAAG,yBACvC,4BAA4B,SAClC;CACF;CACA,IAAI,CAAC,OAAO;CACZ,IAAI;EAEF,KAAK,MAAM,kBAAkB,aAAa,MAAM,MAAM,MAAM,CAAC,CAAC,QAAQ,WAAW,EAAE,CAAC,CAAC;CACvF,SAAS,OAAO;EACd,MAAM,IAAI,cACR,4BAA4B,aAAa,MAAM,MAAM,IAAI,EAAE,KAAM,MAAgB,SACnF;CACF;AACF;;AAGA,SAAgB,kBAAkB,MAAsB;CACtD,OAAO,KACL,KAAK,OAAO,SAAS;EACnB,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,IAAI;GAC7B,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,KAAK,EAAE;EACzC;EACA,IAAI,KAAK,WAAW,IAAI,GAAG;GACzB,MAAM,MAAM,KAAK,QAAQ,MAAM,CAAC;GAChC,OAAO,CAAC,MAAM,IAAI,KAAK,SAAS,MAAM,GAAG,GAAG;EAC9C;CAEF,CAAC,IACA,SAAU,aAAa,KAAK,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,KAAA,CAClD;AACF;;AAGA,SAAS,KAAK,MAAc,IAA4D;CACtF,IAAI,MAAM;CACV,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,SAAU;EACjC,IAAI,KAAK,OAAO,MAAK;GACnB,IAAI,MAAM,IAAI;GACd,OAAO,MAAM,KAAK,UAAU,KAAK,SAAS,MAAK,OAAO,KAAK,SAAS,OAAO,IAAI;GAC/E,OAAO,KAAK,MAAM,GAAG,MAAM,CAAC;GAC5B,IAAI,MAAM;GACV;EACF;EACA,MAAM,MAAM,GAAG,KAAK,MAAM,CAAC,CAAC;EAC5B,IAAI,KAAK;GACP,OAAO,IAAI;GACX,KAAK,IAAI;EACX,OAAO,OAAO,KAAK;CACrB;CACA,OAAO;AACT;;AC7MA,MAAa,uBAAuB;AAEpC,MAAM,cAAc,CAAC,MAAM,oBAAoB;AAE/C,IAAI;AAEJ,SAAgB,cAAuB;CACrC,IAAI,aAAa,KAAA,GAAW,OAAO;CAMnC,WALe,UAAU,MAAM,CAAC,WAAW,GAAG;EAC5C,UAAU;EACV,OAAO;EACP,aAAa;CACf,CACgB,CAAC,CAAC,WAAW;CAC7B,OAAO;AACT;AAEA,SAAgB,cAAc,MAAuB;CACnD,OAAO,WAAW,KAAK,MAAM,SAAS,CAAC,KAAK,WAAW,KAAK,MAAM,gBAAgB,CAAC;AACrF;AAEA,SAAS,WAAW,MAAkC;CACpD,MAAM,OAAO,KAAK,MAAM,SAAS,OAAO,QAAQ;CAChD,IAAI,WAAW,IAAI,GAAG,OAAO;CAC7B,MAAM,MAAM,KAAK,MAAM,SAAS,WAAW,YAAY;CACvD,IAAI,WAAW,GAAG,GAAG,OAAO;AAE9B;;AAGA,SAAgB,2BAA2B,MAA6B;CACtE,MAAM,WAAW,QAAQ,IAAI,eAAe,KAAK;CACjD,IAAI,UAAU,OAAO,CAAC,UAAU,GAAG,WAAW;CAE9C,IAAI,cAAc,IAAI,KAAK,YAAY,GACrC,OAAO;EAAC;EAAM;EAAO;EAAa;EAAM;EAAU,GAAG;CAAW;CAGlE,MAAM,OAAO,WAAW,IAAI;CAC5B,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,WAAW;CAGtC,OAAO,CAAC,QAAQ,aAAa,UAAU,WAAW,WAAW,GAAG,WAAW;AAC7E;;AAGA,MAAa,qBAAqB;CAChC,kCAAkC,qBAAqB;CACvD;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;;AC7BV,MAAa,qBAAuC;CAClD,IAAI;CACJ,YAAY;EAAC;EAAY;EAAa;EAAa;EAAY;EAAa;CAAW;CACvF,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,aAAa,MAAM,IAAI;AAC/C;AAEA,MAAM,sBAA0C;CAC9C,MAAM,UAAU,KAAK,QAAQ,IAAI,GAAG,eAAe;CACnD,IAAI,WAAW,KAAK,SAAS,gBAAgB,CAAC,GAAG,OAAO;AAE1D;;AAaA,MAAa,oBAAgD,CAAC,oBAAoB;CAVhF,IAAI;CACJ,YAAY,CAAC,UAAU;CACvB,cAAc,CAAC,oBAAoB;CACnC,QAAQ;CACR,OAAO,MAAM,SAAS,OAAO,uBAAc,CAAC,MAAM,MAAM,EAAE,oBAAoB,MAAM,IAAI,CAAC;CACzF,gBAAgB,SAAS,2BAA2B,IAAI;CACxD,WAAW;AAIqE,CAAc;AAEhG,SAAgB,eAAe,MAA+B;CAE5D,OADc,kBAAkB,MAAM,MAAM,EAAE,WAAW,MAAM,QAAQ,KAAK,SAAS,GAAG,CAAC,CAC9E,KAAK;AAClB;AAEA,SAAgB,sBAAgC;CAC9C,OAAO,kBAAkB,SAAS,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC;AAC7D;AAEA,SAAgB,UAAU,MAAc,SAA6C;CACnF,MAAM,MAAM,QAAQ,IAAI,qBAAqB,cAAc;CAC3D,IAAI,QAAQ,OAAO,YAAY,CAAC,KAAK,OAAO,QAAQ;CACpD,MAAM,MAAM,KAAK,KAAK,KAAK;CAC3B,OAAO;EACL,GAAG,QAAQ;EACX,YAAY;GAAC;GAAK;GAAK,QAAQ,IAAI;EAAU,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,SAAS;EAC7E,qBAAqB,QAAQ,IAAI;CACnC;AACF"}
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
import { A as errorText, t as agent } from "./src-Cn5H3izr.js";
|
|
2
|
+
import { At as SANDBOX_ROOT, Nt as WORKSPACE_DIR } from "./src-B3sBDfG2.js";
|
|
3
|
+
import { n as shellQuote } from "./process-C_nh1H7X.js";
|
|
4
|
+
import { p as brokerToken, t as SANDBOX_ACCESS_TOKEN } from "./vertex-Plo5WUxj.js";
|
|
5
|
+
import { r as modelLimits } from "./prices-DZa1s6oE.js";
|
|
6
|
+
import { c as runsAgent, d as sandboxKind, f as suggestedSandbox, n as capabilitiesOf } from "./kinds-DHuWeXVE.js";
|
|
7
|
+
import { i as hasNetworkSession, t as InfraError } from "./types-ByoVW0_V.js";
|
|
8
|
+
import { r as startEnv } from "./startup-YENEnSa_.js";
|
|
9
|
+
import { a as agentFor, i as agentAuth, s as authHint, v as writePluginFiles } from "./agents-_3iNDMC1.js";
|
|
10
|
+
// src/environment/runner.ts
|
|
11
|
+
const PLATFORMS = {
|
|
12
|
+
Linux: "linux",
|
|
13
|
+
Darwin: "darwin"
|
|
14
|
+
};
|
|
15
|
+
const ARCHES = {
|
|
16
|
+
x86_64: "x64",
|
|
17
|
+
amd64: "x64",
|
|
18
|
+
aarch64: "arm64",
|
|
19
|
+
arm64: "arm64"
|
|
20
|
+
};
|
|
21
|
+
/** A stream of `from`'s once it has started; it errors when starting fails. */
|
|
22
|
+
function later(from) {
|
|
23
|
+
let reader;
|
|
24
|
+
return new ReadableStream({
|
|
25
|
+
async pull(controller) {
|
|
26
|
+
reader ??= (await from).getReader();
|
|
27
|
+
const { done, value } = await reader.read();
|
|
28
|
+
if (done) controller.close();
|
|
29
|
+
else controller.enqueue(value);
|
|
30
|
+
},
|
|
31
|
+
async cancel(reason) {
|
|
32
|
+
reader ??= await from.then((s) => s.getReader()).catch(() => void 0);
|
|
33
|
+
await reader?.cancel(reason);
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
/** A session's process as a {@link Proc}, which is returned before the process has started. */
|
|
38
|
+
function procOf(started) {
|
|
39
|
+
started.catch(() => {});
|
|
40
|
+
let writer;
|
|
41
|
+
const stdin = async () => {
|
|
42
|
+
const { stdin } = await started;
|
|
43
|
+
if (!stdin) throw new Error("the process wasn't spawned with stdin: 'pipe'");
|
|
44
|
+
writer ??= stdin.getWriter();
|
|
45
|
+
return writer;
|
|
46
|
+
};
|
|
47
|
+
return {
|
|
48
|
+
stdout: later(started.then((p) => p.stdout)),
|
|
49
|
+
stderr: later(started.then((p) => p.stderr)),
|
|
50
|
+
write: async (chunk) => (await stdin()).write(typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk),
|
|
51
|
+
end: async () => (await stdin()).close(),
|
|
52
|
+
kill: async () => {
|
|
53
|
+
await (await started.catch(() => void 0))?.kill();
|
|
54
|
+
},
|
|
55
|
+
exited: started.then(async (p) => ({
|
|
56
|
+
code: (await p.wait()).exitCode,
|
|
57
|
+
signal: null
|
|
58
|
+
}))
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** What the session's machine is, from `uname`, for {@link RunnerInfo}. */
|
|
62
|
+
async function probe(network, home) {
|
|
63
|
+
const { exitCode, stdout, stderr } = await network.run({ command: "uname -s; uname -m" });
|
|
64
|
+
const [system = "", machine = ""] = stdout.trim().split("\n");
|
|
65
|
+
const platform = PLATFORMS[system.trim()];
|
|
66
|
+
const arch = ARCHES[machine.trim()];
|
|
67
|
+
if (exitCode !== 0 || !platform || !arch) throw new Error(`can't tell the sandbox's platform from uname: ${stdout.trim() || stderr.trim()}`);
|
|
68
|
+
return {
|
|
69
|
+
platform: `${platform}-${arch}`,
|
|
70
|
+
home,
|
|
71
|
+
cwd: network.defaultWorkingDirectory,
|
|
72
|
+
shell: "/bin/sh"
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* `sandbox`'s harness session as a {@link Runner}, with what the session the driver builds over
|
|
77
|
+
* it says of itself: the sandbox session's id, description and ports. The sandbox owns the
|
|
78
|
+
* machine, so closing the runner leaves it running.
|
|
79
|
+
*/
|
|
80
|
+
function runnerOf(sandbox) {
|
|
81
|
+
const { network } = sandbox;
|
|
82
|
+
let info;
|
|
83
|
+
return {
|
|
84
|
+
runner: {
|
|
85
|
+
info: () => {
|
|
86
|
+
info ??= probe(network, sandbox.home);
|
|
87
|
+
return info;
|
|
88
|
+
},
|
|
89
|
+
spawn(cmd, o = {}) {
|
|
90
|
+
const command = typeof cmd === "string" ? cmd : cmd.map(shellQuote).join(" ");
|
|
91
|
+
const spawn = network.spawn;
|
|
92
|
+
return procOf(Promise.resolve().then(() => spawn({
|
|
93
|
+
command,
|
|
94
|
+
...o.cwd && { workingDirectory: o.cwd },
|
|
95
|
+
...o.env && { env: o.env },
|
|
96
|
+
...o.signal && { abortSignal: o.signal },
|
|
97
|
+
...o.stdin === "pipe" && { stdin: true }
|
|
98
|
+
})));
|
|
99
|
+
},
|
|
100
|
+
readFile: (path) => Promise.resolve(network.readBinaryFile({ path })),
|
|
101
|
+
async writeFile(path, data, o) {
|
|
102
|
+
const content = typeof data === "string" ? new TextEncoder().encode(data) : data;
|
|
103
|
+
await network.writeBinaryFile({
|
|
104
|
+
path,
|
|
105
|
+
content
|
|
106
|
+
});
|
|
107
|
+
if (o?.mode === void 0) return;
|
|
108
|
+
const chmod = await network.run({ command: `chmod ${o.mode.toString(8)} ${shellQuote(path)}` });
|
|
109
|
+
if (chmod.exitCode !== 0) throw new Error(`chmod ${path}: ${chmod.stderr.trim()}`);
|
|
110
|
+
},
|
|
111
|
+
async forwardPort(port) {
|
|
112
|
+
const { url } = await network.getPortEndpoint({
|
|
113
|
+
port,
|
|
114
|
+
protocol: "http"
|
|
115
|
+
});
|
|
116
|
+
const { hostname, port: hostPort } = new URL(url);
|
|
117
|
+
return {
|
|
118
|
+
host: hostname,
|
|
119
|
+
port: Number(hostPort),
|
|
120
|
+
close: async () => {}
|
|
121
|
+
};
|
|
122
|
+
},
|
|
123
|
+
close: async () => {},
|
|
124
|
+
network: {
|
|
125
|
+
...network.setNetworkPolicy && { setNetworkPolicy: network.setNetworkPolicy.bind(network) },
|
|
126
|
+
...network.addRequestTransformations && { addRequestTransformations: network.addRequestTransformations.bind(network) },
|
|
127
|
+
...network.setRequestTransformations && { setRequestTransformations: network.setRequestTransformations.bind(network) }
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
session: {
|
|
131
|
+
id: network.id,
|
|
132
|
+
description: network.description,
|
|
133
|
+
ports: [...network.ports]
|
|
134
|
+
}
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// src/harness/transcript.ts
|
|
138
|
+
/** The workspace's root, slot 0's (`/reeval/workspace/`) or a shared machine's slot's (`/reeval-s2/…`). */
|
|
139
|
+
const WORKSPACE_PATH = new RegExp(`^${SANDBOX_ROOT}(?:-s\\d+)?/${WORKSPACE_DIR}/`);
|
|
140
|
+
// src/harness/agent.ts
|
|
141
|
+
/** The route's service account, written beside the workspace; returns where it is. */
|
|
142
|
+
async function writeAdc(sandbox, adc, log) {
|
|
143
|
+
await sandbox.writeFile(`../${adc.file}`, adc.content);
|
|
144
|
+
const credentialsFile = `${sandbox.workdir.replace(/[\\/][^\\/]*$/, "")}/${adc.file}`;
|
|
145
|
+
log.log("env", `Vertex AI service account → ${credentialsFile}`);
|
|
146
|
+
return credentialsFile;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Why `agent` can't run in `sandbox`: its kind has no harness session, can't keep the agent's
|
|
150
|
+
* credential out of it, or runs a platform the agent's bootstrap has no build for.
|
|
151
|
+
*/
|
|
152
|
+
function agentSandboxProblem(agent, sandbox) {
|
|
153
|
+
const fits = runsAgent(agent.platforms);
|
|
154
|
+
const can = capabilitiesOf(sandbox);
|
|
155
|
+
if (fits(can)) return void 0;
|
|
156
|
+
const lacks = [
|
|
157
|
+
...can.harnessSession ? [] : ["has no harness session"],
|
|
158
|
+
...can.brokers ? [] : [`can't keep ${agent.credential} out of it`],
|
|
159
|
+
...!can.harnessSession || agent.platforms.includes(can.platform) ? [] : [`runs ${can.platform}, not ${agent.platforms.join(" or ")}`]
|
|
160
|
+
];
|
|
161
|
+
const fix = suggestedSandbox(fits);
|
|
162
|
+
return `The ${agent.name} harness can't run in the ${sandboxKind(sandbox).provider(sandbox)} sandbox, which ${lacks.join(" and ")}: ${fix ? `set sandbox: ${fix} for this eval, or ` : ""}use fakeHarness() with fakeSandbox()`;
|
|
163
|
+
}
|
|
164
|
+
/** The core's spec of a harness descriptor: which agent, on which model. */
|
|
165
|
+
function harnessSpec(descriptor) {
|
|
166
|
+
const { kind, model, effort } = descriptor;
|
|
167
|
+
if (kind === "cursor") return {
|
|
168
|
+
kind,
|
|
169
|
+
model,
|
|
170
|
+
...effort ? { effort } : {},
|
|
171
|
+
...descriptor.params ? { params: descriptor.params } : {}
|
|
172
|
+
};
|
|
173
|
+
if (kind === "claude-code") return {
|
|
174
|
+
kind,
|
|
175
|
+
model,
|
|
176
|
+
...effort ? { effort } : {}
|
|
177
|
+
};
|
|
178
|
+
return {
|
|
179
|
+
kind,
|
|
180
|
+
model,
|
|
181
|
+
...effort ? { effort } : {}
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Where the descriptor's gateway sends its model calls. On Vertex, the attempt's `vertex()`
|
|
186
|
+
* settings, the service account written beside the workspace, and the placeholder the gateway
|
|
187
|
+
* swaps for the minted token.
|
|
188
|
+
*/
|
|
189
|
+
function harnessRoute(descriptor, vertex, credentialsFile) {
|
|
190
|
+
const { gateway } = descriptor;
|
|
191
|
+
if (gateway === "vertex" && vertex) return {
|
|
192
|
+
kind: "vertex",
|
|
193
|
+
...vertex,
|
|
194
|
+
...credentialsFile ? { credentialsFile } : {},
|
|
195
|
+
accessToken: SANDBOX_ACCESS_TOKEN
|
|
196
|
+
};
|
|
197
|
+
if (gateway === "vercel" || gateway === "openrouter") return { kind: gateway };
|
|
198
|
+
return { kind: "direct" };
|
|
199
|
+
}
|
|
200
|
+
/** The fetch the adapter's own calls from the host take: Cursor's through a filled key's filler. */
|
|
201
|
+
async function adapterFetch(descriptor) {
|
|
202
|
+
if (descriptor.kind !== "cursor") return {};
|
|
203
|
+
return (await import("./cursor-catalog-jVblPRFf.js")).cursorFetch();
|
|
204
|
+
}
|
|
205
|
+
/** A real agent through `@reeval/agents`, in a sandbox that gives it a harness session. */
|
|
206
|
+
function createAgentHarness(descriptor, sandbox, vertex) {
|
|
207
|
+
const catalog = agentFor(descriptor, vertex);
|
|
208
|
+
const problem = agentSandboxProblem(catalog, sandbox);
|
|
209
|
+
if (problem) throw new InfraError(problem);
|
|
210
|
+
return { async start(sandbox, signal, { skills, mcp, plugins }, log) {
|
|
211
|
+
const auth = agentAuth(catalog);
|
|
212
|
+
const adc = catalog.adc ? await writeAdc(sandbox, catalog.adc, log) : void 0;
|
|
213
|
+
for (const path of await writePluginFiles(sandbox, catalog, plugins)) log.log("env", `plugin settings → ${path}`);
|
|
214
|
+
if (!hasNetworkSession(sandbox)) throw new InfraError(`The ${catalog.name} harness got a sandbox with no harness session`);
|
|
215
|
+
let stopToken = () => {};
|
|
216
|
+
if (catalog.token) {
|
|
217
|
+
try {
|
|
218
|
+
stopToken = await brokerToken(catalog.token, sandbox.network, {
|
|
219
|
+
onRefresh: (t) => log.log("broker", `access token refreshed, valid until ${new Date(t.expiresAt).toISOString()}`),
|
|
220
|
+
onError: (error) => log.log("broker", `access token refresh failed, retrying: ${errorText(error)}`)
|
|
221
|
+
});
|
|
222
|
+
} catch (error) {
|
|
223
|
+
throw new InfraError(`Couldn't mint an access token from ${catalog.credential} for ${catalog.name}: ${errorText(error)}. Run gcloud auth application-default login, or set GOOGLE_APPLICATION_CREDENTIALS.`);
|
|
224
|
+
}
|
|
225
|
+
log.log("broker", `gateway adds an access token from ${catalog.credential}`);
|
|
226
|
+
}
|
|
227
|
+
const { runner, session } = runnerOf(sandbox);
|
|
228
|
+
let started;
|
|
229
|
+
try {
|
|
230
|
+
started = await agent({
|
|
231
|
+
harness: harnessSpec(descriptor),
|
|
232
|
+
route: harnessRoute(descriptor, vertex, adc),
|
|
233
|
+
runner,
|
|
234
|
+
session,
|
|
235
|
+
credentials: auth,
|
|
236
|
+
install: "auto",
|
|
237
|
+
workDir: WORKSPACE_DIR,
|
|
238
|
+
skills,
|
|
239
|
+
mcp,
|
|
240
|
+
plugins,
|
|
241
|
+
env: startEnv(),
|
|
242
|
+
describeModel: modelLimits,
|
|
243
|
+
...await adapterFetch(descriptor),
|
|
244
|
+
requireBrokering: true,
|
|
245
|
+
traffic: {
|
|
246
|
+
statuses: () => sandbox.brokeredStatuses?.() ?? [],
|
|
247
|
+
servedModels: () => sandbox.servedRequests?.() ?? []
|
|
248
|
+
},
|
|
249
|
+
authHint: (message, refused) => authHint(catalog, message, refused),
|
|
250
|
+
workspace: WORKSPACE_PATH,
|
|
251
|
+
raw: Boolean(process.env.REEVAL_RAW_TRANSCRIPT),
|
|
252
|
+
log,
|
|
253
|
+
signal,
|
|
254
|
+
onEvent: (event) => {
|
|
255
|
+
if (event.type === "warning") log.log("harness", event.message);
|
|
256
|
+
}
|
|
257
|
+
}).start();
|
|
258
|
+
} catch (error) {
|
|
259
|
+
stopToken();
|
|
260
|
+
throw error;
|
|
261
|
+
}
|
|
262
|
+
return {
|
|
263
|
+
send: (prompt, turnSignal) => started.send(prompt, { signal: turnSignal }).result,
|
|
264
|
+
stopped: (note) => started.stopped(note),
|
|
265
|
+
salvage: () => started.salvage(),
|
|
266
|
+
close: () => started.close().finally(stopToken)
|
|
267
|
+
};
|
|
268
|
+
} };
|
|
269
|
+
}
|
|
270
|
+
export { createAgentHarness };
|
|
271
|
+
|
|
272
|
+
//# sourceMappingURL=agent-B1ndNDEu.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-B1ndNDEu.js","names":[],"sources":["../src/environment/runner.ts","../src/harness/transcript.ts","../src/harness/agent.ts"],"mappings":";;;;;;;;;;AAUA,MAAM,YAAgD;CAAE,OAAO;CAAS,QAAQ;AAAS;AACzF,MAAM,SAA0C;CAC9C,QAAQ;CACR,OAAO;CACP,SAAS;CACT,OAAO;AACT;;AAGA,SAAS,MAAM,MAAuE;CACpF,IAAI;CACJ,OAAO,IAAI,eAAe;EACxB,MAAM,KAAK,YAAY;GACrB,YAAY,MAAM,KAAA,CAAM,UAAU;GAClC,MAAM,EAAE,MAAM,UAAU,MAAM,OAAO,KAAK;GAC1C,IAAI,MAAM,WAAW,MAAM;QACtB,WAAW,QAAQ,KAAK;EAC/B;EACA,MAAM,OAAO,QAAQ;GACnB,WAAW,MAAM,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,YAAY,KAAA,CAAS;GACtE,MAAM,QAAQ,OAAO,MAAM;EAC7B;CACF,CAAC;AACH;;AAGA,SAAS,OAAO,SAA0E;CACxF,QAAQ,YAAY,CAAC,CAAC;CACtB,IAAI;CACJ,MAAM,QAAQ,YAAY;EACxB,MAAM,EAAE,UAAU,MAAM;EACxB,IAAI,CAAC,OAAO,MAAM,IAAI,MAAM,+CAA+C;EAC3E,WAAW,MAAM,UAAU;EAC3B,OAAO;CACT;CACA,OAAO;EACL,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,MAAM,CAAC;EAC3C,QAAQ,MAAM,QAAQ,MAAM,MAAM,EAAE,MAAM,CAAC;EAC3C,OAAO,OAAO,WACX,MAAM,MAAM,EAAA,CAAG,MAAM,OAAO,UAAU,WAAW,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK,IAAI,KAAK;EAC3F,KAAK,aAAa,MAAM,MAAM,EAAA,CAAG,MAAM;EACvC,MAAM,YAAY;GAEhB,OAAM,MADa,QAAQ,YAAY,KAAA,CAAS,EAAA,EACpC,KAAK;EACnB;EACA,QAAQ,QAAQ,KAAK,OAAO,OAAO;GAAE,OAAO,MAAM,EAAE,KAAK,EAAA,CAAG;GAAU,QAAQ;EAAK,EAAE;CACvF;AACF;;AAGA,eAAe,MAAM,SAAkB,MAAmC;CACxE,MAAM,EAAE,UAAU,QAAQ,WAAW,MAAM,QAAQ,IAAI,EAAE,SAAS,qBAAqB,CAAC;CACxF,MAAM,CAAC,SAAS,IAAI,UAAU,MAAM,OAAO,KAAK,CAAC,CAAC,MAAM,IAAI;CAC5D,MAAM,WAAW,UAAU,OAAO,KAAK;CACvC,MAAM,OAAO,OAAO,QAAQ,KAAK;CACjC,IAAI,aAAa,KAAK,CAAC,YAAY,CAAC,MAClC,MAAM,IAAI,MACR,iDAAiD,OAAO,KAAK,KAAK,OAAO,KAAK,GAChF;CAEF,OAAO;EACL,UAAU,GAAG,SAAS,GAAG;EACzB;EACA,KAAK,QAAQ;EACb,OAAO;CACT;AACF;;;;;;AAOA,SAAgB,SAAS,SAGvB;CACA,MAAM,EAAE,YAAY;CACpB,IAAI;CAuDJ,OAAO;EACL,QAAA;GAtDA,YAAY;IACV,SAAS,MAAM,SAAS,QAAQ,IAAI;IACpC,OAAO;GACT;GACA,MAAM,KAAK,IAAI,CAAC,GAAG;IACjB,MAAM,UAAU,OAAO,QAAQ,WAAW,MAAM,IAAI,IAAI,UAAU,CAAC,CAAC,KAAK,GAAG;IAC5E,MAAM,QAAQ,QAAQ;IAOtB,OAAO,OACL,QAAQ,QAAQ,CAAC,CAAC,WAChB,MAAM;KACJ;KACA,GAAI,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI;KACvC,GAAI,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI;KAC1B,GAAI,EAAE,UAAU,EAAE,aAAa,EAAE,OAAO;KACxC,GAAI,EAAE,UAAU,UAAU,EAAE,OAAO,KAAK;IAC1C,CAAC,CACH,CACF;GACF;GACA,WAAW,SAAS,QAAQ,QAAQ,QAAQ,eAAe,EAAE,KAAK,CAAC,CAAC;GACpE,MAAM,UAAU,MAAM,MAAM,GAAG;IAC7B,MAAM,UAAU,OAAO,SAAS,WAAW,IAAI,YAAY,CAAC,CAAC,OAAO,IAAI,IAAI;IAC5E,MAAM,QAAQ,gBAAgB;KAAE;KAAM;IAAQ,CAAC;IAC/C,IAAI,GAAG,SAAS,KAAA,GAAW;IAC3B,MAAM,QAAQ,MAAM,QAAQ,IAAI,EAC9B,SAAS,SAAS,EAAE,KAAK,SAAS,CAAC,EAAE,GAAG,WAAW,IAAI,IACzD,CAAC;IACD,IAAI,MAAM,aAAa,GAAG,MAAM,IAAI,MAAM,SAAS,KAAK,IAAI,MAAM,OAAO,KAAK,GAAG;GACnF;GACA,MAAM,YAAY,MAAM;IACtB,MAAM,EAAE,QAAQ,MAAM,QAAQ,gBAAgB;KAAE;KAAM,UAAU;IAAO,CAAC;IACxE,MAAM,EAAE,UAAU,MAAM,aAAa,IAAI,IAAI,GAAG;IAChD,OAAO;KAAE,MAAM;KAAU,MAAM,OAAO,QAAQ;KAAG,OAAO,YAAY,CAAC;IAAE;GACzE;GACA,OAAO,YAAY,CAAC;GACpB,SAAS;IACP,GAAI,QAAQ,oBAAoB,EAC9B,kBAAkB,QAAQ,iBAAiB,KAAK,OAAO,EACzD;IACA,GAAI,QAAQ,6BAA6B,EACvC,2BAA2B,QAAQ,0BAA0B,KAAK,OAAO,EAC3E;IACA,GAAI,QAAQ,6BAA6B,EACvC,2BAA2B,QAAQ,0BAA0B,KAAK,OAAO,EAC3E;GACF;EAGK;EACL,SAAS;GAAE,IAAI,QAAQ;GAAI,aAAa,QAAQ;GAAa,OAAO,CAAC,GAAG,QAAQ,KAAK;EAAE;CACzF;AACF;;;AC/IA,MAAa,iBAAiB,IAAI,OAAO,IAAI,aAAa,cAAc,cAAc,EAAE;;;ACsBxF,eAAe,SAAS,SAAkB,KAAiB,KAAgC;CACzF,MAAM,QAAQ,UAAU,MAAM,IAAI,QAAQ,IAAI,OAAO;CACrD,MAAM,kBAAkB,GAAG,QAAQ,QAAQ,QAAQ,iBAAiB,EAAE,EAAE,GAAG,IAAI;CAC/E,IAAI,IAAI,OAAO,+BAA+B,iBAAiB;CAC/D,OAAO;AACT;;;;;AAMA,SAAgB,oBAAoB,OAAc,SAAgD;CAChG,MAAM,OAAO,UAAU,MAAM,SAAS;CACtC,MAAM,MAAM,eAAe,OAAO;CAClC,IAAI,KAAK,GAAG,GAAG,OAAO,KAAA;CAEtB,MAAM,QAAQ;EACZ,GAAI,IAAI,iBAAiB,CAAC,IAAI,CAAC,wBAAwB;EACvD,GAAI,IAAI,UAAU,CAAC,IAAI,CAAC,cAAc,MAAM,WAAW,WAAW;EAClE,GAAI,CAAC,IAAI,kBAAkB,MAAM,UAAU,SAAS,IAAI,QAAQ,IAC5D,CAAC,IACD,CAAC,QAAQ,IAAI,SAAS,QAAQ,MAAM,UAAU,KAAK,MAAM,GAAG;CAClE;CACA,MAAM,MAAM,iBAAiB,IAAI;CACjC,OAAO,OAAO,MAAM,KAAK,4BAA4B,YAAY,OAAO,CAAC,CAAC,SAAS,OAAO,EAAE,kBAAkB,MAAM,KAAK,OAAO,EAAE,IAAI,MAAM,gBAAgB,IAAI,uBAAuB,GAAG;AAC5L;;AAGA,SAAgB,YAAY,YAAiD;CAC3E,MAAM,EAAE,MAAM,OAAO,WAAW;CAChC,IAAI,SAAS,UACX,OAAO;EACL;EACA;EACA,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;EAC3B,GAAI,WAAW,SAAS,EAAE,QAAQ,WAAW,OAAO,IAAI,CAAC;CAC3D;CAEF,IAAI,SAAS,eAAe,OAAO;EAAE;EAAM;EAAO,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;CAAG;CAChF,OAAO;EAAE;EAAM;EAAO,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;CAAG;AACtD;;;;;;AAOA,SAAgB,aACd,YACA,QACA,iBACY;CACZ,MAAM,EAAE,YAAY;CACpB,IAAI,YAAY,YAAY,QAC1B,OAAO;EACL,MAAM;EACN,GAAG;EACH,GAAI,kBAAkB,EAAE,gBAAgB,IAAI,CAAC;EAC7C,aAAa;CACf;CAEF,IAAI,YAAY,YAAY,YAAY,cAAc,OAAO,EAAE,MAAM,QAAQ;CAC7E,OAAO,EAAE,MAAM,SAAS;AAC1B;;AAGA,eAAe,aAAa,YAAuE;CACjG,IAAI,WAAW,SAAS,UAAU,OAAO,CAAC;CAC1C,QAAQ,MAAM,OAAO,gCAAA,CAAwB,YAAY;AAC3D;;AAyBA,SAAgB,mBACd,YACA,SACA,QACS;CACT,MAAM,UAAU,SAAS,YAAY,MAAM;CAC3C,MAAM,UAAU,oBAAoB,SAAS,OAAO;CACpD,IAAI,SAAS,MAAM,IAAI,WAAW,OAAO;CACzC,OAAO,EACL,MAAM,MAAM,SAAS,QAAQ,EAAE,QAAQ,KAAK,WAAW,KAAK;EAC1D,MAAM,OAAO,UAAU,OAAO;EAC9B,MAAM,MAAM,QAAQ,MAAM,MAAM,SAAS,SAAS,QAAQ,KAAK,GAAG,IAAI,KAAA;EACtE,KAAK,MAAM,QAAQ,MAAM,iBAAiB,SAAS,SAAS,OAAO,GACjE,IAAI,IAAI,OAAO,qBAAqB,MAAM;EAE5C,IAAI,CAAC,kBAAkB,OAAO,GAC5B,MAAM,IAAI,WAAW,OAAO,QAAQ,KAAK,+CAA+C;EAE1F,IAAI,kBAAkB,CAAC;EACvB,IAAI,QAAQ,OAAO;GACjB,IAAI;IACF,YAAY,MAAM,YAAY,QAAQ,OAAO,QAAQ,SAAS;KAC5D,YAAY,MACV,IAAI,IACF,UACA,uCAAuC,IAAI,KAAK,EAAE,SAAS,CAAC,CAAC,YAAY,GAC3E;KACF,UAAU,UACR,IAAI,IAAI,UAAU,0CAA0C,UAAU,KAAK,GAAG;IAClF,CAAC;GACH,SAAS,OAAO;IACd,MAAM,IAAI,WACR,sCAAsC,QAAQ,WAAW,OAAO,QAAQ,KAAK,IAAI,UAAU,KAAK,EAAE,oFACpG;GACF;GACA,IAAI,IAAI,UAAU,qCAAqC,QAAQ,YAAY;EAC7E;EACA,MAAM,EAAE,QAAQ,YAAY,SAAS,OAAO;EAC5C,IAAI;EACJ,IAAI;GACF,UAAU,MAAM,MAAM;IACpB,SAAS,YAAY,UAAU;IAC/B,OAAO,aAAa,YAAY,QAAQ,GAAG;IAC3C;IACA;IACA,aAAa;IAEb,SAAS;IACT,SAAS;IACT;IACA;IACA;IACA,KAAK,SAAS;IACd,eAAe;IACf,GAAI,MAAM,aAAa,UAAU;IAEjC,kBAAkB;IAClB,SAAS;KACP,gBAAgB,QAAQ,mBAAmB,KAAK,CAAC;KACjD,oBAAoB,QAAQ,iBAAiB,KAAK,CAAC;IACrD;IACA,WAAW,SAAS,YAAY,SAAS,SAAS,SAAS,OAAO;IAClE,WAAW;IACX,KAAK,QAAQ,QAAQ,IAAI,qBAAqB;IAC9C;IACA;IACA,UAAU,UAAU;KAClB,IAAI,MAAM,SAAS,WAAW,IAAI,IAAI,WAAW,MAAM,OAAO;IAChE;GACF,CAAC,CAAC,CAAC,MAAM;EACX,SAAS,OAAO;GACd,UAAU;GACV,MAAM;EACR;EACA,OAAO;GACL,OAAO,QAAQ,eAAe,QAAQ,KAAK,QAAQ,EAAE,QAAQ,WAAW,CAAC,CAAC,CAAC;GAC3E,UAAU,SAAS,QAAQ,QAAQ,IAAI;GACvC,eAAe,QAAQ,QAAQ;GAC/B,aAAa,QAAQ,MAAM,CAAC,CAAC,QAAQ,SAAS;EAChD;CACF,EACF;AACF"}
|