reeval 0.17.0 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES.md +19 -19
- package/dist/{adapter-CqUt4A0w.js → adapter-DyvDj4ZN.js} +9 -13
- package/dist/adapter-DyvDj4ZN.js.map +1 -0
- package/dist/{agent-BtAbRTFq.js → agent-v036GEgG.js} +167 -30
- package/dist/agent-v036GEgG.js.map +1 -0
- package/dist/{analysis-C2-aEbr7.js → analysis-adKPz-UJ.js} +5 -6
- package/dist/{analysis-C2-aEbr7.js.map → analysis-adKPz-UJ.js.map} +1 -1
- package/dist/app-files-hpRT5lj_.js +10 -0
- package/dist/app-files-hpRT5lj_.js.map +1 -0
- package/dist/assemble-DTXwCL3m.js +1 -0
- package/dist/{assemble-J9rkE29m.js → assemble-Dlh6Xv3U.js} +23 -23
- package/dist/assemble-Dlh6Xv3U.js.map +1 -0
- package/dist/{attempt-CpbZ9arV.js → attempt-CtKGvaQR.js} +4 -5
- package/dist/{attempt-CpbZ9arV.js.map → attempt-CtKGvaQR.js.map} +1 -1
- package/dist/{attempt-host-DDedP7Qv.js → attempt-host-DErH-DYV.js} +9 -11
- package/dist/{attempt-host-DDedP7Qv.js.map → attempt-host-DErH-DYV.js.map} +1 -1
- package/dist/bridge/index.mjs +9 -5
- package/dist/{budget-DBUrH4jX.js → budget-BcxLYY5L.js} +2 -3
- package/dist/budget-BcxLYY5L.js.map +1 -0
- package/dist/capture-BT2fAmpZ.js +1 -0
- package/dist/{capture-DheNqeYS.js → capture-Bdcx9Gap.js} +52 -56
- package/dist/capture-Bdcx9Gap.js.map +1 -0
- package/dist/{capture-Bvc4FtWT.js → capture-DMBZE9cs.js} +2 -3
- package/dist/{capture-Bvc4FtWT.js.map → capture-DMBZE9cs.js.map} +1 -1
- package/dist/{cli-KTHjNFoo.js → cli-CGd51thd.js} +118 -94
- package/dist/cli-CGd51thd.js.map +1 -0
- package/dist/{cli-spec-CPWT1mAG.js → cli-spec-ChkleEVm.js} +2 -3
- package/dist/{cli-spec-CPWT1mAG.js.map → cli-spec-ChkleEVm.js.map} +1 -1
- package/dist/cli.js +3 -4
- package/dist/{client-DMmOkbRA.js → client-CF-DOjOU.js} +6 -8
- package/dist/{client-DMmOkbRA.js.map → client-CF-DOjOU.js.map} +1 -1
- package/dist/{colors-Y6O3mtl_.js → colors-DcXBvAlU.js} +2 -3
- package/dist/{colors-Y6O3mtl_.js.map → colors-DcXBvAlU.js.map} +1 -1
- package/dist/{command-Ce03LLtg.js → command-CsfcwMyc.js} +22 -24
- package/dist/command-CsfcwMyc.js.map +1 -0
- package/dist/{command-nNN0mx65.js → command-CtTdCF3B.js} +17 -21
- package/dist/command-CtTdCF3B.js.map +1 -0
- package/dist/{command-1FBECMs-.js → command-CyIb9BSC.js} +29 -30
- package/dist/command-CyIb9BSC.js.map +1 -0
- package/dist/{command-CztRuwUM.js → command-TLmY-rYp.js} +22 -27
- package/dist/command-TLmY-rYp.js.map +1 -0
- package/dist/{commands-DP13JX9i.js → commands-C54S45d7.js} +51 -21
- package/dist/{commands-DP13JX9i.js.map → commands-C54S45d7.js.map} +1 -1
- package/dist/{commands-ZwQUezI1.js → commands-jO9Jj35r.js} +1 -1
- package/dist/{config-BjjDryd5.js → config-ImY6AlRE.js} +112 -74
- package/dist/config-ImY6AlRE.js.map +1 -0
- package/dist/{context-Dgn48BqT.js → context-BajfiH0S.js} +4 -5
- package/dist/{context-Dgn48BqT.js.map → context-BajfiH0S.js.map} +1 -1
- package/dist/{contrasts-DSsRZpR8.js → contrasts-DtSX8E_V.js} +4 -5
- package/dist/{contrasts-DSsRZpR8.js.map → contrasts-DtSX8E_V.js.map} +1 -1
- package/dist/{core-tLN-z1Be.js → core-CxY40JAs.js} +2 -3
- package/dist/{core-tLN-z1Be.js.map → core-CxY40JAs.js.map} +1 -1
- package/dist/cost-CMzY9VXv.js +1 -0
- package/dist/{cost-ChPa8-Fp.js → cost-e4IppZpJ.js} +5 -6
- package/dist/{cost-ChPa8-Fp.js.map → cost-e4IppZpJ.js.map} +1 -1
- package/dist/{cursor-catalog-D7bFLEAD.js → cursor-catalog-Bw6-pJjR.js} +6 -7
- package/dist/{cursor-catalog-D7bFLEAD.js.map → cursor-catalog-Bw6-pJjR.js.map} +1 -1
- package/dist/db-ZIk6S2CD.js +2 -0
- package/dist/{db-B9Ityuit.js → db-sHGpaLo9.js} +28 -27
- package/dist/db-sHGpaLo9.js.map +1 -0
- package/dist/{debug-CgG3T071.js → debug-B5jDImcW.js} +5 -6
- package/dist/{debug-CgG3T071.js.map → debug-B5jDImcW.js.map} +1 -1
- package/dist/debug-BnXV7JbX.js +1 -0
- package/dist/{default-effort-DK8gospa.js → default-effort-BZRa-SQP.js} +12 -9
- package/dist/default-effort-BZRa-SQP.js.map +1 -0
- package/dist/{default-effort-DmXHpqo9.js → default-effort-fpdbYVKm.js} +1 -1
- package/dist/{dist-CYrBwyHA.js → dist-ByYk_wAs.js} +59 -292
- package/dist/dist-ByYk_wAs.js.map +1 -0
- package/dist/{docker-SKpYOlGi.js → docker--2ytOHcu.js} +20 -11
- package/dist/docker--2ytOHcu.js.map +1 -0
- package/dist/{docker-BLCNfUie.js → docker-CIzHzwlC.js} +1 -1
- package/dist/{docker-hwp3EucJ.js → docker-CWuSRj7a.js} +5 -6
- package/dist/{docker-hwp3EucJ.js.map → docker-CWuSRj7a.js.map} +1 -1
- package/dist/{doctor-CT0Sgjng.js → doctor-eE65YpEv.js} +26 -27
- package/dist/{doctor-CT0Sgjng.js.map → doctor-eE65YpEv.js.map} +1 -1
- package/dist/{env-CARj5Axq.js → env-T86lAqUD.js} +3 -4
- package/dist/{env-CARj5Axq.js.map → env-T86lAqUD.js.map} +1 -1
- package/dist/{environment-CZdercoj.js → environment-Cy3FVLR5.js} +10 -13
- package/dist/environment-Cy3FVLR5.js.map +1 -0
- package/dist/{errors-BZVKVKYd.js → errors-BQV__TzH.js} +3 -4
- package/dist/{errors-BZVKVKYd.js.map → errors-BQV__TzH.js.map} +1 -1
- package/dist/{errors-B6BSMs03.js → errors-ttMTnIYX.js} +2 -3
- package/dist/{errors-B6BSMs03.js.map → errors-ttMTnIYX.js.map} +1 -1
- package/dist/{eval-source-nZD7-FSz.js → eval-source-BTsTRiBW.js} +8 -9
- package/dist/{eval-source-nZD7-FSz.js.map → eval-source-BTsTRiBW.js.map} +1 -1
- package/dist/{evaluate-fhC1PCd_.js → evaluate-CFFdyYZX.js} +118 -89
- package/dist/evaluate-CFFdyYZX.js.map +1 -0
- package/dist/{evaluate-CNi0lZ7n.d.ts → evaluate-DnGOaGQc.d.ts} +2 -4
- package/dist/files-B3d-8Fe0.js +1 -0
- package/dist/{files-Bk1mC3AN.js → files-BGlHnC46.js} +3 -4
- package/dist/{files-Bk1mC3AN.js.map → files-BGlHnC46.js.map} +1 -1
- package/dist/{files-I1i6Xyh1.js → files-CIX-pLl4.js} +7 -8
- package/dist/{files-I1i6Xyh1.js.map → files-CIX-pLl4.js.map} +1 -1
- package/dist/files-COW5JBJM.js +1 -0
- package/dist/{files-BGy05Dlr.js → files-CQU08uh4.js} +6 -7
- package/dist/{files-BGy05Dlr.js.map → files-CQU08uh4.js.map} +1 -1
- package/dist/files-Do4Rp3yZ.js +1 -0
- package/dist/{files-D1y8Pg4Z.js → files-p-uTdOjA.js} +4 -5
- package/dist/{files-D1y8Pg4Z.js.map → files-p-uTdOjA.js.map} +1 -1
- package/dist/{files-BIGxpL08.js → files-tnMgu7-X.js} +13 -16
- package/dist/files-tnMgu7-X.js.map +1 -0
- package/dist/{gate-BC6SdQZs.js → gate-BoDKeuJu.js} +4 -6
- package/dist/{gate-BC6SdQZs.js.map → gate-BoDKeuJu.js.map} +1 -1
- package/dist/gateway.d.ts +1 -3
- package/dist/gateway.js +1 -2
- package/dist/{git-DR02tHtU.js → git-BWVkG4Yl.js} +4 -5
- package/dist/{git-DR02tHtU.js.map → git-BWVkG4Yl.js.map} +1 -1
- package/dist/{harness-ylJLvrbp.js → harness-Ch2P81sx.js} +12 -13
- package/dist/harness-Ch2P81sx.js.map +1 -0
- package/dist/harness.d.ts +1 -3
- package/dist/harness.js +1 -1
- package/dist/{ids-Bp6ut7ZD.js → ids-DuK6xl4w.js} +2 -3
- package/dist/{ids-Bp6ut7ZD.js.map → ids-DuK6xl4w.js.map} +1 -1
- package/dist/{images---X8Wu9d.js → images-BVZHDf_z.js} +9 -10
- package/dist/{images---X8Wu9d.js.map → images-BVZHDf_z.js.map} +1 -1
- package/dist/index.d.ts +2 -12
- package/dist/index.js +5 -5
- package/dist/{init-CA95ZnkV.js → init-BWnMbUnc.js} +8 -9
- package/dist/{init-CA95ZnkV.js.map → init-BWnMbUnc.js.map} +1 -1
- package/dist/init-s1jN-PCI.js +1 -0
- package/dist/{invocations-BdoROAfb.js → invocations-BGRs1r_C.js} +6 -8
- package/dist/{invocations-BdoROAfb.js.map → invocations-BGRs1r_C.js.map} +1 -1
- package/dist/{judge-Dp7FPJ1P.js → judge-ByEUlzOM.js} +10 -15
- package/dist/judge-ByEUlzOM.js.map +1 -0
- package/dist/judge.d.ts +1 -5
- package/dist/judge.js +2 -3
- package/dist/live-ColieOai.js +120 -0
- package/dist/live-ColieOai.js.map +1 -0
- package/dist/live-DmJix4c8.js +1 -0
- package/dist/{live-CBFkdK1K.js → live-d_5a5iUv.js} +4 -5
- package/dist/{live-CBFkdK1K.js.map → live-d_5a5iUv.js.map} +1 -1
- package/dist/{matchers-Cv74mJRJ.js → matchers-B4ZtlJyP.js} +5 -6
- package/dist/{matchers-Cv74mJRJ.js.map → matchers-B4ZtlJyP.js.map} +1 -1
- package/dist/model-8uZBYqV5.js +1 -0
- package/dist/{model-DkJVmVd2.js → model-CFSjV-HJ.js} +6 -7
- package/dist/{model-DkJVmVd2.js.map → model-CFSjV-HJ.js.map} +1 -1
- package/dist/{model-6dfKFkQd.js → model-DBpdRXtO.js} +9 -11
- package/dist/model-DBpdRXtO.js.map +1 -0
- package/dist/{model-CN4ccjNz.js → model-DOXnWH65.js} +1 -1
- package/dist/{model-Be7fPvgi.js → model-DOiCiMCD.js} +23 -18
- package/dist/model-DOiCiMCD.js.map +1 -0
- package/dist/{node-D_Zkwhq2.js → node-CwP1G178.js} +11 -13
- package/dist/node-CwP1G178.js.map +1 -0
- package/dist/{object-BkXGPKWY.js → object-CeQ8dgUC.js} +2 -3
- package/dist/{object-BkXGPKWY.js.map → object-CeQ8dgUC.js.map} +1 -1
- package/dist/{opencode-providers-DPgdqRfg.js → opencode-providers-yNKyXD49.js} +57 -21
- package/dist/opencode-providers-yNKyXD49.js.map +1 -0
- package/dist/{openrouter-models-CR00FWAa.js → openrouter-models-C8nzPKG2.js} +3 -4
- package/dist/{openrouter-models-CR00FWAa.js.map → openrouter-models-C8nzPKG2.js.map} +1 -1
- package/dist/{patch-CquGAGX5.js → patch-C7yBztZj.js} +2 -3
- package/dist/{patch-CquGAGX5.js.map → patch-C7yBztZj.js.map} +1 -1
- package/dist/{patch-BJNdOZsf.js → patch-DvLr8wCZ.js} +5 -6
- package/dist/{patch-BJNdOZsf.js.map → patch-DvLr8wCZ.js.map} +1 -1
- package/dist/{paths-BmR-7tx3.js → paths-Dwmi7L4X.js} +2 -3
- package/dist/{paths-BmR-7tx3.js.map → paths-Dwmi7L4X.js.map} +1 -1
- package/dist/{plan-C1UpGZTk.js → plan-CIfOFhth.js} +11 -12
- package/dist/{plan-C1UpGZTk.js.map → plan-CIfOFhth.js.map} +1 -1
- package/dist/{plugins-Cx6ReJWE.js → plugins-BEs2Am3L.js} +8 -10
- package/dist/plugins-BEs2Am3L.js.map +1 -0
- package/dist/{preflight-BZ3L7P3a.js → preflight-50Qw5e5K.js} +4 -5
- package/dist/{preflight-BZ3L7P3a.js.map → preflight-50Qw5e5K.js.map} +1 -1
- package/dist/{preflight-CUooKgtT.js → preflight-BAIRsTpl.js} +1 -1
- package/dist/{prepare-BbBoTeDj.js → prepare-8XhzhLHX.js} +10 -11
- package/dist/{prepare-BbBoTeDj.js.map → prepare-8XhzhLHX.js.map} +1 -1
- package/dist/{prepare-CEZaEwaq.js → prepare-8nvfDFQx.js} +2 -2
- package/dist/{prices-QaS-kjSQ.js → prices-C4BRuSe8.js} +5 -6
- package/dist/{prices-QaS-kjSQ.js.map → prices-C4BRuSe8.js.map} +1 -1
- package/dist/{provenance-BWvTIlQM.js → provenance-Cc8fFE7Z.js} +6 -7
- package/dist/{provenance-BWvTIlQM.js.map → provenance-Cc8fFE7Z.js.map} +1 -1
- package/dist/provider-DCdTHQpk.js +1 -0
- package/dist/{provider-CmAYTQQs.js → provider-DZd8t_XW.js} +3 -4
- package/dist/{provider-CmAYTQQs.js.map → provider-DZd8t_XW.js.map} +1 -1
- package/dist/{providers-DdyZW_ru.js → providers-CP_bBtYm.js} +1 -1
- package/dist/{providers-B3yqakc7.js → providers-DZSTlC9R.js} +8 -9
- package/dist/{providers-B3yqakc7.js.map → providers-DZSTlC9R.js.map} +1 -1
- package/dist/{query-CZcqpOI9.js → query-DXpxaaTH.js} +5 -6
- package/dist/{query-CZcqpOI9.js.map → query-DXpxaaTH.js.map} +1 -1
- package/dist/query-kFPuqBv1.js +1 -0
- package/dist/{questions-CFU7-crW.js → questions-C_Y-oXW6.js} +10 -11
- package/dist/{questions-CFU7-crW.js.map → questions-C_Y-oXW6.js.map} +1 -1
- package/dist/{record-Bu1Xi6MQ.js → record-DhzwkSXc.js} +27 -4
- package/dist/record-DhzwkSXc.js.map +1 -0
- package/dist/{redact-DVIaahLv.js → redact-BEEGc37e.js} +8 -7
- package/dist/{redact-DVIaahLv.js.map → redact-BEEGc37e.js.map} +1 -1
- package/dist/{remarks-eBcrIZRu.js → remarks-0EXMh6AD.js} +2 -3
- package/dist/{remarks-eBcrIZRu.js.map → remarks-0EXMh6AD.js.map} +1 -1
- package/dist/remarks-BL18Id09.js +1 -0
- package/dist/{remarks-Dj_x9l5n.js → remarks-DeZZT8PK.js} +4 -6
- package/dist/{remarks-Dj_x9l5n.js.map → remarks-DeZZT8PK.js.map} +1 -1
- package/dist/{remarks-A3j-SkJ_.js → remarks-E4rpi8Ym.js} +16 -17
- package/dist/remarks-E4rpi8Ym.js.map +1 -0
- package/dist/{remote-EIFHdMd8.js → remote-DZ00A8V_.js} +10 -13
- package/dist/remote-DZ00A8V_.js.map +1 -0
- package/dist/resolve-BJSay360.js +1 -0
- package/dist/{resolve-8H5rz8mi.js → resolve-B_JlQ0La.js} +11 -12
- package/dist/{resolve-8H5rz8mi.js.map → resolve-B_JlQ0La.js.map} +1 -1
- package/dist/{results-E7JdVjq2.js → results-BMJHaQbi.js} +8 -9
- package/dist/{results-E7JdVjq2.js.map → results-BMJHaQbi.js.map} +1 -1
- package/dist/{resume-Blu4yIcq.js → resume-CexsWuGK.js} +8 -10
- package/dist/resume-CexsWuGK.js.map +1 -0
- package/dist/{root-Dlff-zp8.js → root-Co0jhOT5.js} +2 -3
- package/dist/{root-Dlff-zp8.js.map → root-Co0jhOT5.js.map} +1 -1
- package/dist/{runtime-CNbsMJQ7.js → runtime-CSZ1hjlN.js} +8 -10
- package/dist/{runtime-CNbsMJQ7.js.map → runtime-CSZ1hjlN.js.map} +1 -1
- package/dist/{sandbox-RButGfIE.js → sandbox-B5W8lLL1.js} +4 -5
- package/dist/{sandbox-RButGfIE.js.map → sandbox-B5W8lLL1.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -5
- package/dist/sandbox.js +1 -1
- package/dist/{server-XgCYs_MF.js → server-BfN6q2lH.js} +17 -126
- package/dist/server-BfN6q2lH.js.map +1 -0
- package/dist/server-SBzrf35T.js +1 -0
- package/dist/{shared-B_lvGqWz.js → shared-ogqPJ0uo.js} +4 -6
- package/dist/shared-ogqPJ0uo.js.map +1 -0
- package/dist/{source-BOkbJCD2.js → source-BD6Bg0Tr.js} +3 -4
- package/dist/{source-BOkbJCD2.js.map → source-BD6Bg0Tr.js.map} +1 -1
- package/dist/source-Dn1uqPon.js +1 -0
- package/dist/{src-DSZyg0Uq.js → src-B-L8O1Ld.js} +38 -9
- package/dist/src-B-L8O1Ld.js.map +1 -0
- package/dist/{src-D4CZjMQ4.js → src-BC2WXULN.js} +2 -2
- package/dist/src-BKDZO3ra.js +2 -0
- package/dist/{src-A8HLtO0a.js → src-B_igzM1O.js} +3 -5
- package/dist/src-B_igzM1O.js.map +1 -0
- package/dist/{src-BkhEPVJZ.js → src-BvM0jWuZ.js} +83 -12
- package/dist/src-BvM0jWuZ.js.map +1 -0
- package/dist/src-BxrHipV0.js +3 -0
- package/dist/src-C9elednc.js +2 -0
- package/dist/src-CM7xdJFA.js +2 -0
- package/dist/src-CWEGknZo.js +2 -0
- package/dist/{src-BJCKWacl.js → src-Cpy9MNS3.js} +155 -83
- package/dist/src-Cpy9MNS3.js.map +1 -0
- package/dist/{src-GkJ-1Rdy.js → src-Cx0HJIJ-.js} +2 -2
- package/dist/{src-gn7YlnRg.js → src-D5t4OiRt.js} +5 -8
- package/dist/src-D5t4OiRt.js.map +1 -0
- package/dist/src-DVaAKCsH.js +1 -0
- package/dist/src-Dp-fWOfG.js +1448 -0
- package/dist/src-Dp-fWOfG.js.map +1 -0
- package/dist/src-DurP--_D.js +2 -0
- package/dist/{src-C2Japg_I.js → src-HTG2SoNz.js} +69 -29
- package/dist/src-HTG2SoNz.js.map +1 -0
- package/dist/{src-8yTF251P.js → src-KlDOGD0B.js} +66 -16
- package/dist/src-KlDOGD0B.js.map +1 -0
- package/dist/{src-BYDOXw_Y.js → src-uluei5nU.js} +40 -17
- package/dist/src-uluei5nU.js.map +1 -0
- package/dist/{state-D6-tvqtp.js → state-CdSjXIfE.js} +31 -40
- package/dist/state-CdSjXIfE.js.map +1 -0
- package/dist/state-Cicc9Oa5.js +1 -0
- package/dist/{stats-hfpuYsAM.js → stats-D-Gqwqt0.js} +2 -3
- package/dist/{stats-hfpuYsAM.js.map → stats-D-Gqwqt0.js.map} +1 -1
- package/dist/suite-BYJ2Q0S-.js +2 -0
- package/dist/{suite-DYI1k2Ie.js → suite-BtHPBFYL.js} +5 -7
- package/dist/suite-BtHPBFYL.js.map +1 -0
- package/dist/{summary-CN2aqAUz.js → summary-zP3cCXDN.js} +11 -13
- package/dist/{summary-CN2aqAUz.js.map → summary-zP3cCXDN.js.map} +1 -1
- package/dist/{tarball-CHcahyNK.js → tarball-CESxtK4X.js} +2 -3
- package/dist/{tarball-CHcahyNK.js.map → tarball-CESxtK4X.js.map} +1 -1
- package/dist/testing.d.ts +9 -11
- package/dist/testing.js +5 -6
- package/dist/{text-DTigcA64.js → text-D--Q5KsA.js} +2 -3
- package/dist/{text-DTigcA64.js.map → text-D--Q5KsA.js.map} +1 -1
- package/dist/{tool-target-CKWkd6QI.js → tool-target-DS5d72eH.js} +3 -4
- package/dist/{tool-target-CKWkd6QI.js.map → tool-target-DS5d72eH.js.map} +1 -1
- package/dist/{transcript-BJrKfgRW.js → transcript-C_IpYMRk.js} +8 -8
- package/dist/transcript-C_IpYMRk.js.map +1 -0
- package/dist/{types-DDeCnNmp.d.ts → types-BHR1lGPh.d.ts} +99 -24
- package/dist/{types-C5vI3g1C.js → types-C4OkYhZg.js} +2 -3
- package/dist/types-C4OkYhZg.js.map +1 -0
- package/dist/{usage-BNY1ABrU.js → usage-DJiDnh2D.js} +2 -3
- package/dist/{usage-BNY1ABrU.js.map → usage-DJiDnh2D.js.map} +1 -1
- package/dist/{vertex-DypYjx71.js → vertex-6uSpZ5L8.js} +8 -8
- package/dist/vertex-6uSpZ5L8.js.map +1 -0
- package/dist/vertex-BaAz44YN.js +1 -0
- package/dist/view/index.d.ts +13 -37
- package/dist/view/index.js +12 -12
- package/dist/{views-Bd-H71_t.js → views-CKoN5NAx.js} +7 -8
- package/dist/{views-Bd-H71_t.js.map → views-CKoN5NAx.js.map} +1 -1
- package/dist/views-CmyX255j.js +2 -0
- package/dist/{worker-entry-BIHJqE_z.js → worker-entry-D5dB8cgT.js} +12 -17
- package/dist/worker-entry-D5dB8cgT.js.map +1 -0
- package/dist/{workspace-Sy5g8zW9.js → workspace-Cho30qlM.js} +8 -9
- package/dist/{workspace-Sy5g8zW9.js.map → workspace-Cho30qlM.js.map} +1 -1
- package/dist/workspace-Dw9DEZox.js +1 -0
- package/docker/Dockerfile +4 -4
- package/package.json +12 -12
- package/skills/eval-rubric/SKILL.md +1 -1
- package/dist/adapter-CqUt4A0w.js.map +0 -1
- package/dist/agent-BtAbRTFq.js.map +0 -1
- package/dist/app-files-CDttHPOz.js +0 -11
- package/dist/app-files-CDttHPOz.js.map +0 -1
- package/dist/assemble-Bb7zqpRw.js +0 -1
- package/dist/assemble-J9rkE29m.js.map +0 -1
- package/dist/budget-DBUrH4jX.js.map +0 -1
- package/dist/capture-D68TiQQy.js +0 -1
- package/dist/capture-DheNqeYS.js.map +0 -1
- package/dist/cli-KTHjNFoo.js.map +0 -1
- package/dist/command-1FBECMs-.js.map +0 -1
- package/dist/command-Ce03LLtg.js.map +0 -1
- package/dist/command-CztRuwUM.js.map +0 -1
- package/dist/command-nNN0mx65.js.map +0 -1
- package/dist/config-BjjDryd5.js.map +0 -1
- package/dist/cost-C5RBTTJr.js +0 -1
- package/dist/db-B9Ityuit.js.map +0 -1
- package/dist/db-Bk6LWwza.js +0 -2
- package/dist/debug-CAx3qcaP.js +0 -1
- package/dist/default-effort-DK8gospa.js.map +0 -1
- package/dist/dist-CYrBwyHA.js.map +0 -1
- package/dist/docker-SKpYOlGi.js.map +0 -1
- package/dist/environment-CZdercoj.js.map +0 -1
- package/dist/evaluate-fhC1PCd_.js.map +0 -1
- package/dist/files-BIGxpL08.js.map +0 -1
- package/dist/files-C2RC7K4m.js +0 -1
- package/dist/files-C3tQtD26.js +0 -1
- package/dist/files-dJmrfmOM.js +0 -1
- package/dist/harness-ylJLvrbp.js.map +0 -1
- package/dist/init-BNTXrtMO.js +0 -1
- package/dist/judge-Dp7FPJ1P.js.map +0 -1
- package/dist/model-6dfKFkQd.js.map +0 -1
- package/dist/model-Be7fPvgi.js.map +0 -1
- package/dist/model-C_5TspZS.js +0 -1
- package/dist/node-D_Zkwhq2.js.map +0 -1
- package/dist/opencode-providers-DPgdqRfg.js.map +0 -1
- package/dist/plugins-Cx6ReJWE.js.map +0 -1
- package/dist/provider-DehR4CMN.js +0 -1
- package/dist/query-BkwQ0W-c.js +0 -1
- package/dist/record-Bu1Xi6MQ.js.map +0 -1
- package/dist/remarks-A3j-SkJ_.js.map +0 -1
- package/dist/remarks-Bp6AwPgb.js +0 -1
- package/dist/remote-EIFHdMd8.js.map +0 -1
- package/dist/resolve-Tm9pX7Hv.js +0 -1
- package/dist/resume-Blu4yIcq.js.map +0 -1
- package/dist/server-Dqrq-Sia.js +0 -1
- package/dist/server-XgCYs_MF.js.map +0 -1
- package/dist/shared-B_lvGqWz.js.map +0 -1
- package/dist/source-DooSTtyz.js +0 -1
- package/dist/src-1M0srjji.js +0 -1
- package/dist/src-8yTF251P.js.map +0 -1
- package/dist/src-A8HLtO0a.js.map +0 -1
- package/dist/src-BJCKWacl.js.map +0 -1
- package/dist/src-BYDOXw_Y.js.map +0 -1
- package/dist/src-BZlhaDhE.js +0 -2
- package/dist/src-BkhEPVJZ.js.map +0 -1
- package/dist/src-Bp6X4193.js +0 -2
- package/dist/src-C2Japg_I.js.map +0 -1
- package/dist/src-C5q3_fRV.js +0 -3
- package/dist/src-C9EHSYyP.js +0 -2
- package/dist/src-CJdpsrXi.js +0 -2
- package/dist/src-D1ET6oPL.js +0 -629
- package/dist/src-D1ET6oPL.js.map +0 -1
- package/dist/src-DSZyg0Uq.js.map +0 -1
- package/dist/src-OOKE-5GA.js +0 -2
- package/dist/src-gn7YlnRg.js.map +0 -1
- package/dist/state-BCyNaz13.js +0 -1
- package/dist/state-D6-tvqtp.js.map +0 -1
- package/dist/suite-BiWajSlA.js +0 -2
- package/dist/suite-DYI1k2Ie.js.map +0 -1
- package/dist/transcript-BJrKfgRW.js.map +0 -1
- package/dist/types-C5vI3g1C.js.map +0 -1
- package/dist/vertex-Bv7vhMgs.js +0 -1
- package/dist/vertex-DypYjx71.js.map +0 -1
- package/dist/views-7bNY17Z6.js +0 -2
- package/dist/worker-entry-BIHJqE_z.js.map +0 -1
- package/dist/workspace-CxwHv8mG.js +0 -1
- /package/dist/bridge/{pnpm-lock.yaml → pnpm-lockfile.yaml} +0 -0
|
@@ -1,41 +1,43 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-
|
|
2
|
-
import { r as withLease } from "./client-
|
|
3
|
-
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-
|
|
4
|
-
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-
|
|
5
|
-
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-
|
|
6
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-
|
|
7
|
-
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-
|
|
8
|
-
import { A as
|
|
9
|
-
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-
|
|
10
|
-
import { v as vertexSettings } from "./vertex-
|
|
11
|
-
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-
|
|
12
|
-
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-
|
|
13
|
-
import { t as inCI } from "./env-
|
|
14
|
-
import {
|
|
15
|
-
import { t as InfraError } from "./types-
|
|
16
|
-
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-
|
|
17
|
-
import { c as formatBytes } from "./debug-
|
|
18
|
-
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-
|
|
19
|
-
import { t as attemptDebugDir } from "./files-
|
|
20
|
-
import { r as openRouterFallbacks } from "./openrouter-models-
|
|
21
|
-
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-
|
|
22
|
-
import { t as contextTaker } from "./context-
|
|
23
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import { i as
|
|
31
|
-
import { i as
|
|
1
|
+
import { t as isObjectLike } from "./object-CeQ8dgUC.js";
|
|
2
|
+
import { r as withLease } from "./client-CF-DOjOU.js";
|
|
3
|
+
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-ttMTnIYX.js";
|
|
4
|
+
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-ogqPJ0uo.js";
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Cy3FVLR5.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-BEs2Am3L.js";
|
|
7
|
+
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-DuK6xl4w.js";
|
|
8
|
+
import { $ as allowedHosts, A as evalPlan, At as refusedMcp, Bt as judgeProblem, Ct as brokeringFor, D as cascade, E as brokeredIn, Et as expectedBlockedFor, F as matchesTags, Ft as NO_JUDGE_INPUT, G as sandboxSetBy, Gt as decideState, H as runsOf, I as plainHeaderError, J as unbrokeredError, K as secretVarsIn, Kt as filePartsOf, L as plainHeaders, Lt as inputsOn, M as harnessId, Mt as writePluginFiles, Ot as keyRejected, P as majorityOf, Pt as INPUT_KEYS, U as sandboxId, Ut as artifactRefsOf, V as refuseIllegalCompare, W as sandboxProvider, Wt as badJudgeArtifacts, X as resolveStateOptions, Y as mergeJudge, _ as fellBackToDirect, at as unenforceable, b as resolveModelGateway, it as isDirect, j as expand, jt as refusedPlugins, k as checkOptions, kt as layoutOf, nt as egressLabel, q as unbrokered, qt as resolveJudgeArtifacts, st as createFakeSandbox, v as gatewayScope, vt as AGENTS, w as resolveGateway, wt as claudeMcpServers, xt as agentMcp } from "./config-ImY6AlRE.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-ByEUlzOM.js";
|
|
10
|
+
import { v as vertexSettings } from "./vertex-6uSpZ5L8.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-BEEGc37e.js";
|
|
12
|
+
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-D--Q5KsA.js";
|
|
13
|
+
import { t as inCI } from "./env-T86lAqUD.js";
|
|
14
|
+
import { dt as OPAQUE_OUTPUT, mt as shellQuote, t as MCP_TIMER } from "./src-Cpy9MNS3.js";
|
|
15
|
+
import { t as InfraError } from "./types-C4OkYhZg.js";
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CSZ1hjlN.js";
|
|
17
|
+
import { c as formatBytes } from "./debug-B5jDImcW.js";
|
|
18
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CtKGvaQR.js";
|
|
19
|
+
import { t as attemptDebugDir } from "./files-p-uTdOjA.js";
|
|
20
|
+
import { r as openRouterFallbacks } from "./openrouter-models-C8nzPKG2.js";
|
|
21
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-Bdcx9Gap.js";
|
|
22
|
+
import { t as contextTaker } from "./context-BajfiH0S.js";
|
|
23
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-B_JlQ0La.js";
|
|
24
|
+
import { n as isDockerSandbox } from "./docker--2ytOHcu.js";
|
|
25
|
+
import { t as PrepareError } from "./provider-DZd8t_XW.js";
|
|
26
|
+
import { i as preparedWorkspace } from "./workspace-Cho30qlM.js";
|
|
27
|
+
import { t as patchPath } from "./patch-C7yBztZj.js";
|
|
28
|
+
import { n as judgeDiff } from "./patch-DvLr8wCZ.js";
|
|
29
|
+
import { t as transcriptForJudge } from "./transcript-C_IpYMRk.js";
|
|
30
|
+
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CESxtK4X.js";
|
|
31
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-BTsTRiBW.js";
|
|
32
|
+
import { i as withDefaultEffort } from "./default-effort-BZRa-SQP.js";
|
|
32
33
|
import { dirname, join } from "node:path";
|
|
33
34
|
import { parse, stringify } from "yaml";
|
|
34
35
|
import { z } from "zod";
|
|
36
|
+
import { randomBytes } from "node:crypto";
|
|
35
37
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
36
38
|
import { gunzipSync, gzipSync } from "node:zlib";
|
|
37
39
|
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
38
|
-
|
|
40
|
+
// src/attempt/assertions.ts
|
|
39
41
|
const MESSAGE_LIMIT = 1024;
|
|
40
42
|
const LABEL_LIMIT = 60;
|
|
41
43
|
/** Masked first, so a secret the limit would cut through is masked whole. */
|
|
@@ -103,8 +105,7 @@ function checkFailure(raw) {
|
|
|
103
105
|
caught.stack = error.stack?.replace(error.message, caught.message);
|
|
104
106
|
return caught;
|
|
105
107
|
}
|
|
106
|
-
|
|
107
|
-
//#region src/attempt/debug.ts
|
|
108
|
+
// src/attempt/debug.ts
|
|
108
109
|
/** Which config and profile the attempt runs under, and where each resolved option came from. */
|
|
109
110
|
function tracePlan(debug, spec, ctx) {
|
|
110
111
|
const config = ctx.configFile ?? "none (no config file found)";
|
|
@@ -189,8 +190,7 @@ function traceEnvironment(debug, spec, env) {
|
|
|
189
190
|
}))
|
|
190
191
|
}, "resolved environment");
|
|
191
192
|
}
|
|
192
|
-
|
|
193
|
-
//#region src/harness/fake.ts
|
|
193
|
+
// src/harness/fake.ts
|
|
194
194
|
const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
|
|
195
195
|
/**
|
|
196
196
|
* Writes canned files and returns a canned message on every turn, after running its `script`'s
|
|
@@ -207,6 +207,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
207
207
|
const hangs = descriptor.startHangs;
|
|
208
208
|
if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
|
|
209
209
|
const layout = layoutOf(descriptor);
|
|
210
|
+
const brokered = descriptor.brokers ? await brokerKey(descriptor, sandbox) : void 0;
|
|
210
211
|
const mcp = agentMcp(layout, all, plugins);
|
|
211
212
|
for (const path of await writePluginFiles(sandbox, layout, plugins)) debugLog("env", `plugin settings → ${path} by the fake agent`);
|
|
212
213
|
const dir = `~/${layout.harnessSkillDir}`;
|
|
@@ -288,7 +289,7 @@ const createFakeHarness = (descriptor) => {
|
|
|
288
289
|
tool: tool.tool,
|
|
289
290
|
input: tool.input,
|
|
290
291
|
output: tool.output,
|
|
291
|
-
ok: tool.ok
|
|
292
|
+
ok: tool.ok === void 0 ? true : tool.ok,
|
|
292
293
|
durationMs: 0
|
|
293
294
|
});
|
|
294
295
|
if (flaked) {
|
|
@@ -305,7 +306,10 @@ const createFakeHarness = (descriptor) => {
|
|
|
305
306
|
for (const step of steps) if ("exec" in step) {
|
|
306
307
|
const time = at();
|
|
307
308
|
debugLog("harness", `tool shell called: $ ${step.exec}`);
|
|
308
|
-
const r = await sandbox.exec(step.exec, {
|
|
309
|
+
const r = await sandbox.exec(step.exec, {
|
|
310
|
+
signal,
|
|
311
|
+
...brokered && { env: brokered }
|
|
312
|
+
});
|
|
309
313
|
debugLog("harness", `tool shell returned exit ${r.exitCode} in ${Date.now() - startedAt - time.atMs}ms`);
|
|
310
314
|
transcript.push({
|
|
311
315
|
...time,
|
|
@@ -342,6 +346,18 @@ const createFakeHarness = (descriptor) => {
|
|
|
342
346
|
message: step.error
|
|
343
347
|
});
|
|
344
348
|
return done("", "error");
|
|
349
|
+
} else if ("refuseKey" in step) {
|
|
350
|
+
const message = `${keyRejected(step.refuseKey)}. 401 Unauthorized (the fake agent's)`;
|
|
351
|
+
debugLog("harness", `turn failed: ${message}`);
|
|
352
|
+
transcript.push({
|
|
353
|
+
...at(),
|
|
354
|
+
type: "error",
|
|
355
|
+
message
|
|
356
|
+
});
|
|
357
|
+
return {
|
|
358
|
+
...done("", "error"),
|
|
359
|
+
rejectedKey: message
|
|
360
|
+
};
|
|
345
361
|
} else scripted = step.reply;
|
|
346
362
|
if (descriptor.delayMs) try {
|
|
347
363
|
await setTimeout$1(descriptor.delayMs, void 0, descriptor.ignoreAbort ? {} : { signal });
|
|
@@ -363,6 +379,30 @@ const createFakeHarness = (descriptor) => {
|
|
|
363
379
|
} };
|
|
364
380
|
};
|
|
365
381
|
/**
|
|
382
|
+
* Has the egress gateway swap the host's key for a placeholder on the hosts of the agent's
|
|
383
|
+
* endpoints, as a real adapter does, and returns the variable its `exec` steps get it in.
|
|
384
|
+
*/
|
|
385
|
+
async function brokerKey(descriptor, sandbox) {
|
|
386
|
+
const { credential, endpoints } = AGENTS[descriptor.layout ?? "cursor"];
|
|
387
|
+
const key = process.env[credential];
|
|
388
|
+
if (!key) throw new InfraError(`${credential} isn't set, so the fake agent has no key to broker. Set it in the environment or .env.local.`);
|
|
389
|
+
const network = isDockerSandbox(sandbox) ? sandbox.network : void 0;
|
|
390
|
+
if (!network?.setRequestTransformations) throw new InfraError("fakeHarness({ brokers: true }) needs docker(): this sandbox can't broker");
|
|
391
|
+
const placeholder = `reeval_fake_${randomBytes(24).toString("base64url")}`;
|
|
392
|
+
await network.setRequestTransformations([...new Set(endpoints.map((e) => e.host))].map((host) => ({
|
|
393
|
+
match: {
|
|
394
|
+
host,
|
|
395
|
+
headers: [{
|
|
396
|
+
key: { exact: "authorization" },
|
|
397
|
+
value: { exact: `Bearer ${placeholder}` }
|
|
398
|
+
}]
|
|
399
|
+
},
|
|
400
|
+
transform: { headers: { authorization: `Bearer ${key}` } }
|
|
401
|
+
})));
|
|
402
|
+
debugLog("harness", `${credential} is a placeholder for the fake agent, brokered on its endpoints`);
|
|
403
|
+
return { [credential]: placeholder };
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
366
406
|
* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
|
|
367
407
|
* arguments: initialize, then the call.
|
|
368
408
|
*/
|
|
@@ -437,17 +477,15 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
437
477
|
ok: !reply.result?.isError
|
|
438
478
|
};
|
|
439
479
|
}
|
|
440
|
-
|
|
441
|
-
//#region src/harness/index.ts
|
|
480
|
+
// src/harness/index.ts
|
|
442
481
|
async function createHarness(descriptor, vertex) {
|
|
443
482
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
444
|
-
const { createAgentHarness } = await import("./agent-
|
|
483
|
+
const { createAgentHarness } = await import("./agent-v036GEgG.js");
|
|
445
484
|
return createAgentHarness(descriptor, vertex);
|
|
446
485
|
}
|
|
447
486
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
448
487
|
const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
|
|
449
|
-
|
|
450
|
-
//#region src/attempt/environment.ts
|
|
488
|
+
// src/attempt/environment.ts
|
|
451
489
|
/**
|
|
452
490
|
* Resolves the attempt's environment (files, skills, secrets and their placeholders)
|
|
453
491
|
* and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
|
|
@@ -471,8 +509,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
471
509
|
if (debug) traceEnvironment(debug, spec, env);
|
|
472
510
|
record.image = null;
|
|
473
511
|
if (spec.sandbox.kind === "docker") {
|
|
474
|
-
const { preparedFor } = await import("./preflight-
|
|
475
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
512
|
+
const { preparedFor } = await import("./preflight-BAIRsTpl.js");
|
|
513
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-CP_bBtYm.js");
|
|
476
514
|
signal.throwIfAborted();
|
|
477
515
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
478
516
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -498,8 +536,7 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
498
536
|
debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
|
|
499
537
|
} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
|
|
500
538
|
}
|
|
501
|
-
|
|
502
|
-
//#region src/judge/size.ts
|
|
539
|
+
// src/judge/size.ts
|
|
503
540
|
/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
|
|
504
541
|
const CHARS_PER_TOKEN = 4;
|
|
505
542
|
const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
|
|
@@ -554,8 +591,7 @@ function stateHint(state, question) {
|
|
|
554
591
|
* no model's limit, so this follows every provider error.
|
|
555
592
|
*/
|
|
556
593
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
557
|
-
|
|
558
|
-
//#region src/judge/state.ts
|
|
594
|
+
// src/judge/state.ts
|
|
559
595
|
function turnState(t) {
|
|
560
596
|
return {
|
|
561
597
|
...t.task !== void 0 ? { task: t.task } : {},
|
|
@@ -576,8 +612,7 @@ function stateOf(d) {
|
|
|
576
612
|
...transcript !== void 0 ? { transcript } : {}
|
|
577
613
|
};
|
|
578
614
|
}
|
|
579
|
-
|
|
580
|
-
//#region src/attempt/judge.ts
|
|
615
|
+
// src/attempt/judge.ts
|
|
581
616
|
const noJudgeModel = (configFile) => `c.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.14.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
|
|
582
617
|
/** A turn this attempt sent becomes its index; any other value is graded as the output. */
|
|
583
618
|
function judgeOn(turns, on) {
|
|
@@ -807,14 +842,13 @@ function judgeCaller(a, readFile) {
|
|
|
807
842
|
return Object.defineProperty(await ask(order, question, call), SUBJECT, { value: question });
|
|
808
843
|
};
|
|
809
844
|
}
|
|
810
|
-
|
|
811
|
-
//#region src/sandbox/index.ts
|
|
845
|
+
// src/sandbox/index.ts
|
|
812
846
|
async function createSandbox(descriptor, options) {
|
|
813
847
|
try {
|
|
814
848
|
switch (descriptor.kind) {
|
|
815
849
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
816
850
|
case "docker": {
|
|
817
|
-
const { createDockerSandbox } = await import("./docker-
|
|
851
|
+
const { createDockerSandbox } = await import("./docker-CIzHzwlC.js");
|
|
818
852
|
return await createDockerSandbox(descriptor, options);
|
|
819
853
|
}
|
|
820
854
|
}
|
|
@@ -824,8 +858,7 @@ async function createSandbox(descriptor, options) {
|
|
|
824
858
|
throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
|
|
825
859
|
}
|
|
826
860
|
}
|
|
827
|
-
|
|
828
|
-
//#region src/attempt/record.ts
|
|
861
|
+
// src/attempt/record.ts
|
|
829
862
|
function classify(error, signal) {
|
|
830
863
|
const message = error instanceof Error ? error.message : String(error);
|
|
831
864
|
if (signal.aborted) return isTimeout(signal.reason) ? {
|
|
@@ -915,7 +948,7 @@ async function recordOom(a, sandbox) {
|
|
|
915
948
|
/**
|
|
916
949
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
917
950
|
* Refused hosts the harness always tries (in a session like this one) are `expectedBlocked`,
|
|
918
|
-
* summarized in one `-v` line.
|
|
951
|
+
* summarized in one `-v` line. Names that didn't resolve are `unresolved`, never `blocked`.
|
|
919
952
|
*/
|
|
920
953
|
function recordEgress(a, sandbox) {
|
|
921
954
|
const traffic = sandbox?.traffic?.();
|
|
@@ -930,7 +963,8 @@ function recordEgress(a, sandbox) {
|
|
|
930
963
|
...traffic && isDirect(a.spec.egress.policy) ? { partial: true } : {},
|
|
931
964
|
contacted: traffic?.contacted ?? [],
|
|
932
965
|
blocked: refused.filter((h) => !isExpected(h)),
|
|
933
|
-
expectedBlocked
|
|
966
|
+
expectedBlocked,
|
|
967
|
+
unresolved: traffic?.unresolved ?? []
|
|
934
968
|
};
|
|
935
969
|
if (expected && expectedBlocked.length) {
|
|
936
970
|
const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
|
|
@@ -1001,8 +1035,7 @@ function turnErrors(transcripts) {
|
|
|
1001
1035
|
return found?.type === "error" ? [`turn ${i + 1} error: ${clip(redactSecrets(found.message.trim()), 500)}`] : [];
|
|
1002
1036
|
});
|
|
1003
1037
|
}
|
|
1004
|
-
|
|
1005
|
-
//#region src/harness/unpack.ts
|
|
1038
|
+
// src/harness/unpack.ts
|
|
1006
1039
|
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
1007
1040
|
const FRONTMATTER = /^\uFEFF?---\r?\n/;
|
|
1008
1041
|
const FRONTMATTER_BLOCK = /^\uFEFF?---\r?\n(?:---|([\s\S]*?)\r?\n---)[ \t]*(?:\r?\n|$)/;
|
|
@@ -1113,8 +1146,7 @@ function unpackPlugins(plugins, unpack, hooksFile = null) {
|
|
|
1113
1146
|
});
|
|
1114
1147
|
return out;
|
|
1115
1148
|
}
|
|
1116
|
-
|
|
1117
|
-
//#region src/sandbox/phases.ts
|
|
1149
|
+
// src/sandbox/phases.ts
|
|
1118
1150
|
const strip = ({ phase: _, ...sub }) => sub;
|
|
1119
1151
|
/**
|
|
1120
1152
|
* Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
|
|
@@ -1142,8 +1174,7 @@ function brokerPhases(box, subs) {
|
|
|
1142
1174
|
}
|
|
1143
1175
|
};
|
|
1144
1176
|
}
|
|
1145
|
-
|
|
1146
|
-
//#region src/attempt/mcp-startup.ts
|
|
1177
|
+
// src/attempt/mcp-startup.ts
|
|
1147
1178
|
/**
|
|
1148
1179
|
* The timer and its log beside the workspace: `/reeval/bin/mcp-timer.cjs` in Docker. `.cjs`, since
|
|
1149
1180
|
* a sandbox's directories may be in a `"type": "module"` package.
|
|
@@ -1206,8 +1237,7 @@ const mcpStartupOf = (entries) => entries.map((e) => ({
|
|
|
1206
1237
|
}));
|
|
1207
1238
|
/** The first server in `entries` that never answered `initialize`. */
|
|
1208
1239
|
const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
1209
|
-
|
|
1210
|
-
//#region src/attempt/state.ts
|
|
1240
|
+
// src/attempt/state.ts
|
|
1211
1241
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1212
1242
|
const SETUP_BUDGET_MS = 12e5;
|
|
1213
1243
|
function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
@@ -1304,8 +1334,7 @@ function evalTimer(ms) {
|
|
|
1304
1334
|
clear: () => clearTimeout(timer)
|
|
1305
1335
|
};
|
|
1306
1336
|
}
|
|
1307
|
-
|
|
1308
|
-
//#region src/attempt/session.ts
|
|
1337
|
+
// src/attempt/session.ts
|
|
1309
1338
|
const tailOf = (text) => text.trim().split("\n").slice(-10).join("\n");
|
|
1310
1339
|
/** Setup steps in order, each with its own timeout; a failure is an infra error. */
|
|
1311
1340
|
async function runSetup(a, box, env) {
|
|
@@ -1592,7 +1621,11 @@ async function startSession(a, runId) {
|
|
|
1592
1621
|
brokering: brokeringFor(spec.harness, spec.vertex),
|
|
1593
1622
|
expectedBlocked: expectedBlockedFor(spec.harness, env.plugins.length > 0, spec.vertex)?.hosts,
|
|
1594
1623
|
substitutions: env.substitutions,
|
|
1595
|
-
egress: spec.egress
|
|
1624
|
+
egress: spec.egress,
|
|
1625
|
+
onWarning: (text) => a.report({
|
|
1626
|
+
kind: "warning",
|
|
1627
|
+
text
|
|
1628
|
+
})
|
|
1596
1629
|
});
|
|
1597
1630
|
a.sandbox = box;
|
|
1598
1631
|
if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
|
|
@@ -1660,8 +1693,7 @@ async function startSession(a, runId) {
|
|
|
1660
1693
|
});
|
|
1661
1694
|
return [box, session];
|
|
1662
1695
|
}
|
|
1663
|
-
|
|
1664
|
-
//#region src/attempt/snapshot.ts
|
|
1696
|
+
// src/attempt/snapshot.ts
|
|
1665
1697
|
/** The most a snapshot directory's files may add up to; a larger one isn't taken. */
|
|
1666
1698
|
const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
|
|
1667
1699
|
/** The most a compressed snapshot may be. */
|
|
@@ -1891,8 +1923,7 @@ async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SN
|
|
|
1891
1923
|
failed(error instanceof Error ? error.message : String(error));
|
|
1892
1924
|
}
|
|
1893
1925
|
}
|
|
1894
|
-
|
|
1895
|
-
//#region src/attempt/turns.ts
|
|
1926
|
+
// src/attempt/turns.ts
|
|
1896
1927
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1897
1928
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
1898
1929
|
/**
|
|
@@ -1919,6 +1950,7 @@ function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
|
|
|
1919
1950
|
usage: null,
|
|
1920
1951
|
model: null,
|
|
1921
1952
|
unreported: [],
|
|
1953
|
+
reasoningMs: null,
|
|
1922
1954
|
startedAt,
|
|
1923
1955
|
durationMs: at
|
|
1924
1956
|
});
|
|
@@ -2019,11 +2051,13 @@ function turnSender(a, takeContext, checks) {
|
|
|
2019
2051
|
usage: null,
|
|
2020
2052
|
model: null,
|
|
2021
2053
|
unreported: [],
|
|
2054
|
+
reasoningMs: null,
|
|
2022
2055
|
startedAt: entry.startedAt,
|
|
2023
2056
|
durationMs: Date.now() - sendStarted
|
|
2024
2057
|
};
|
|
2025
2058
|
});
|
|
2026
|
-
const { raw, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2059
|
+
const { raw, rejectedKey, reasoningMs, ...sent } = await a.timed("agentMs", () => Promise.race([sending, cutoff])).catch(async (error) => {
|
|
2060
|
+
entry.reasoningMs = null;
|
|
2027
2061
|
if (watch) entry.mcpStartup = mcpStartupOf((await watch.end()).entries);
|
|
2028
2062
|
throw error;
|
|
2029
2063
|
}).finally(() => {
|
|
@@ -2044,7 +2078,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2044
2078
|
atMs: Math.max(0, seenAt - Date.parse(sent.startedAt))
|
|
2045
2079
|
}].sort((x, y) => x.atMs - y.atMs)
|
|
2046
2080
|
} : sent;
|
|
2047
|
-
Object.assign(entry, result, raw ? { raw } : {}, {
|
|
2081
|
+
Object.assign(entry, result, raw ? { raw } : {}, reasoningMs !== void 0 ? { reasoningMs } : {}, {
|
|
2048
2082
|
durationMs: Math.max(result.durationMs, sendWallMs),
|
|
2049
2083
|
...startup ? { mcpStartup: mcpStartupOf(startup.entries) } : {}
|
|
2050
2084
|
});
|
|
@@ -2066,8 +2100,9 @@ function turnSender(a, takeContext, checks) {
|
|
|
2066
2100
|
});
|
|
2067
2101
|
debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(redactSecrets(String(failure.message)), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
|
|
2068
2102
|
for (const s of entry.mcpStartup ?? []) debugLog("harness", `turn ${turnNumber} mcp ${s.server} ${s.ms === null ? "didn't answer initialize" : `answered initialize in ${formatDuration(s.ms)}`}`);
|
|
2069
|
-
|
|
2070
|
-
|
|
2103
|
+
const infra = timedOut || result.status !== "aborted" && rejectedKey;
|
|
2104
|
+
if (infra) {
|
|
2105
|
+
const error = new InfraError(infra);
|
|
2071
2106
|
a.startFailure ??= { error };
|
|
2072
2107
|
throw error;
|
|
2073
2108
|
}
|
|
@@ -2088,8 +2123,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
2088
2123
|
return turn;
|
|
2089
2124
|
};
|
|
2090
2125
|
}
|
|
2091
|
-
|
|
2092
|
-
//#region src/attempt/index.ts
|
|
2126
|
+
// src/attempt/index.ts
|
|
2093
2127
|
const TRACE_BUFFER_DEFAULT = 8 * 2 ** 20;
|
|
2094
2128
|
async function runAttempt(spec, fn, handle) {
|
|
2095
2129
|
const ctx = getRunContext();
|
|
@@ -2208,8 +2242,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
2208
2242
|
if (oom) throw oom;
|
|
2209
2243
|
if (failure) throw failure.error;
|
|
2210
2244
|
}
|
|
2211
|
-
|
|
2212
|
-
//#region src/best-of.ts
|
|
2245
|
+
// src/best-of.ts
|
|
2213
2246
|
/** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
|
|
2214
2247
|
function sideOf({ state, meta }) {
|
|
2215
2248
|
const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
|
|
@@ -2232,8 +2265,7 @@ function bestOfCell(bestOf) {
|
|
|
2232
2265
|
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2233
2266
|
});
|
|
2234
2267
|
}
|
|
2235
|
-
|
|
2236
|
-
//#region src/contract/declaration.ts
|
|
2268
|
+
// src/contract/declaration.ts
|
|
2237
2269
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
2238
2270
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
2239
2271
|
const declaredModeSchema = z.enum([
|
|
@@ -2289,8 +2321,7 @@ const NODE_SCHEMAS = {
|
|
|
2289
2321
|
* tree's `children`. Options are left to the cascade.
|
|
2290
2322
|
*/
|
|
2291
2323
|
const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
|
|
2292
|
-
|
|
2293
|
-
//#region src/plan-eval.ts
|
|
2324
|
+
// src/plan-eval.ts
|
|
2294
2325
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
2295
2326
|
const GRACE_MS = 5e3;
|
|
2296
2327
|
function configLevel(config) {
|
|
@@ -2606,8 +2637,7 @@ function attemptLimits(planned, concurrency) {
|
|
|
2606
2637
|
if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
|
|
2607
2638
|
return limits;
|
|
2608
2639
|
}
|
|
2609
|
-
|
|
2610
|
-
//#region src/evaluate.ts
|
|
2640
|
+
// src/evaluate.ts
|
|
2611
2641
|
let groupStack = [];
|
|
2612
2642
|
/** Each eval file's declarations as `evaluate` and `group` were called, and each eval's body by id. */
|
|
2613
2643
|
const trees = /* @__PURE__ */ new Map();
|
|
@@ -2896,7 +2926,6 @@ function optionsFrom(written, where, base) {
|
|
|
2896
2926
|
}
|
|
2897
2927
|
};
|
|
2898
2928
|
}
|
|
2899
|
-
//#endregion
|
|
2900
2929
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2901
2930
|
|
|
2902
|
-
//# sourceMappingURL=evaluate-
|
|
2931
|
+
//# sourceMappingURL=evaluate-CFFdyYZX.js.map
|