reeval 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/dist/{agent-B72lZ6w5.js → agent-6FefIhX7.js} +5 -5
- package/dist/{agent-B72lZ6w5.js.map → agent-6FefIhX7.js.map} +1 -1
- package/dist/{agent-Cfj1wD4D.js → agent-WSmTaesf.js} +51 -16
- package/dist/agent-WSmTaesf.js.map +1 -0
- package/dist/{analysis-BTNnKh1w.js → analysis-DlGBDjTj.js} +3 -3
- package/dist/{analysis-BTNnKh1w.js.map → analysis-DlGBDjTj.js.map} +1 -1
- package/dist/{assemble-BbFVjHeS.js → assemble-BCZzTyR7.js} +243 -11
- package/dist/assemble-BCZzTyR7.js.map +1 -0
- package/dist/assemble-C5qq5yT2.js +2 -0
- package/dist/{capture-BXhBfmsM.js → capture-Bp74Z6Fr.js} +66 -15
- package/dist/capture-Bp74Z6Fr.js.map +1 -0
- package/dist/capture-DS-cPyuE.js +2 -0
- package/dist/{cli-BMjTKY-D.js → cli-CVrMh_qz.js} +201 -108
- package/dist/cli-CVrMh_qz.js.map +1 -0
- package/dist/{cli-spec-DSvKsOK4.js → cli-spec-CKEYa8V2.js} +3 -3
- package/dist/{cli-spec-DSvKsOK4.js.map → cli-spec-CKEYa8V2.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{client-CwzU9CiW.js → client-Bbo9SPlK.js} +31 -2
- package/dist/client-Bbo9SPlK.js.map +1 -0
- package/dist/{command-BdstSEHu.js → command-BFP0hKEb.js} +7 -7
- package/dist/{command-BdstSEHu.js.map → command-BFP0hKEb.js.map} +1 -1
- package/dist/command-BrmXUC-W.js +2 -0
- package/dist/{command-BK4kOgkH.js → command-C5xFPKHM.js} +19 -18
- package/dist/command-C5xFPKHM.js.map +1 -0
- package/dist/{command-JPGNzFsA.js → command-CzIFp1JQ.js} +10 -10
- package/dist/{command-JPGNzFsA.js.map → command-CzIFp1JQ.js.map} +1 -1
- package/dist/{command-BqJtFiPy.js → command-DUT5l13I.js} +82 -32
- package/dist/command-DUT5l13I.js.map +1 -0
- package/dist/{commands-B618jg3C.js → commands-CLmPBrTS.js} +6 -6
- package/dist/{commands-B618jg3C.js.map → commands-CLmPBrTS.js.map} +1 -1
- package/dist/{commands-Duo2ur3r.js → commands-fI3G2Wnh.js} +1 -1
- package/dist/{config-B-E0xRkf.js → config-CGvBeewH.js} +17 -7
- package/dist/config-CGvBeewH.js.map +1 -0
- package/dist/{context-CvUv7DUb.js → context-DbSMEVKs.js} +1 -5
- package/dist/{context-CvUv7DUb.js.map → context-DbSMEVKs.js.map} +1 -1
- package/dist/{db-CnoUy-FZ.js → db-CSSFtGcN.js} +1 -1
- package/dist/{db-DFbtGIYy.js → db-CfQFwJtq.js} +137 -27
- package/dist/db-CfQFwJtq.js.map +1 -0
- package/dist/{docker-2qns7hJ2.js → docker-CRQGQbIt.js} +17 -5
- package/dist/docker-CRQGQbIt.js.map +1 -0
- package/dist/{docker-IFRW8K5f.js → docker-DV2GNgle.js} +2 -2
- package/dist/{docker-IFRW8K5f.js.map → docker-DV2GNgle.js.map} +1 -1
- package/dist/{docker-D6t23fZl.js → docker-Df_YOxxB.js} +1 -1
- package/dist/{doctor-B6oYTPoR.js → doctor-DlKycx5H.js} +12 -12
- package/dist/{doctor-B6oYTPoR.js.map → doctor-DlKycx5H.js.map} +1 -1
- package/dist/{egress-IUKdeayc.js → egress-BhzNh7Yi.js} +10 -9
- package/dist/{egress-IUKdeayc.js.map → egress-BhzNh7Yi.js.map} +1 -1
- package/dist/{environment-DiCljXMD.js → environment-1jE-HhO-.js} +2 -2
- package/dist/{environment-DiCljXMD.js.map → environment-1jE-HhO-.js.map} +1 -1
- package/dist/family-D2b7WD10.js +67 -0
- package/dist/family-D2b7WD10.js.map +1 -0
- package/dist/files-AiEKuEZe.js +2 -0
- package/dist/files-BMCvfB4X.js +2 -0
- package/dist/{files-CMSUXAEg.js → files-CSzVADHn.js} +4 -4
- package/dist/{files-CMSUXAEg.js.map → files-CSzVADHn.js.map} +1 -1
- package/dist/{files-B6aVVGrk.js → files-DJUFg-4e.js} +2 -2
- package/dist/{files-B6aVVGrk.js.map → files-DJUFg-4e.js.map} +1 -1
- package/dist/{git-EmdFJ4DY.js → git-DfG-YJ1u.js} +2 -2
- package/dist/{git-EmdFJ4DY.js.map → git-DfG-YJ1u.js.map} +1 -1
- package/dist/{harness-iqWeJQMM.js → harness-CeCp2GSW.js} +19 -2
- package/dist/harness-CeCp2GSW.js.map +1 -0
- package/dist/harness.d.ts +5 -3
- package/dist/harness.js +1 -1
- package/dist/{header-_F0EWob3.js → header-CALhmafp.js} +10 -3
- package/dist/{header-_F0EWob3.js.map → header-CALhmafp.js.map} +1 -1
- package/dist/{images-CscddrJU.js → images-BMgdpMAX.js} +6 -6
- package/dist/{images-CscddrJU.js.map → images-BMgdpMAX.js.map} +1 -1
- package/dist/index.d.ts +17 -5
- package/dist/index.js +92 -62
- package/dist/index.js.map +1 -1
- package/dist/init-B6vegaP_.js +2 -0
- package/dist/{init-NQfIvi-0.js → init-BIdLF0F9.js} +4 -4
- package/dist/{init-NQfIvi-0.js.map → init-BIdLF0F9.js.map} +1 -1
- package/dist/{invocations-nPvrlZPj.js → invocations-CLaL8AuW.js} +2 -2
- package/dist/{invocations-nPvrlZPj.js.map → invocations-CLaL8AuW.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/{model-Cl507trk.js → model-B1BZoZ9b.js} +4 -4
- package/dist/{model-Cl507trk.js.map → model-B1BZoZ9b.js.map} +1 -1
- package/dist/{model-BndnmcLm.js → model-BOnlAPzs.js} +162 -17
- package/dist/model-BOnlAPzs.js.map +1 -0
- package/dist/model-DWeg5J7z.js +2 -0
- package/dist/{model-DMOraCcm.js → model-FXko2kl7.js} +1 -1
- package/dist/{options-DPUlm4FI.js → options-CKqyPGxX.js} +12 -8
- package/dist/options-CKqyPGxX.js.map +1 -0
- package/dist/{patch-CGCVipPu.js → patch-bDV8YFW0.js} +2 -2
- package/dist/{patch-CGCVipPu.js.map → patch-bDV8YFW0.js.map} +1 -1
- package/dist/{plan-PPc4-6xd.js → plan-DUydFc_1.js} +53 -25
- package/dist/plan-DUydFc_1.js.map +1 -0
- package/dist/{preflight-CddycK92.js → preflight-BkoSxf4C.js} +3 -3
- package/dist/{preflight-CddycK92.js.map → preflight-BkoSxf4C.js.map} +1 -1
- package/dist/{preflight-Zumy0adT.js → preflight-D5LW7Oqi.js} +1 -1
- package/dist/{prepare-CsP-057p.js → prepare-CUAWHebF.js} +5 -5
- package/dist/{prepare-CsP-057p.js.map → prepare-CUAWHebF.js.map} +1 -1
- package/dist/{prepare-CKdMoC7e.js → prepare-oji2oO8W.js} +2 -2
- package/dist/process-CSVO_jsc.js +119 -0
- package/dist/process-CSVO_jsc.js.map +1 -0
- package/dist/{provenance-CsX6PHDR.js → provenance-D7HK3vzt.js} +2 -2
- package/dist/{provenance-CsX6PHDR.js.map → provenance-D7HK3vzt.js.map} +1 -1
- package/dist/{providers-DkwiLKDS.js → providers-B1DzMvmH.js} +3 -3
- package/dist/{providers-DkwiLKDS.js.map → providers-B1DzMvmH.js.map} +1 -1
- package/dist/{providers-yXZZcPfx.js → providers-CpEfNU1F.js} +1 -1
- package/dist/{query-BUcwCYsI.js → query-BjBMLa-C.js} +2 -2
- package/dist/{query-BUcwCYsI.js.map → query-BjBMLa-C.js.map} +1 -1
- package/dist/query-oPO_XrTy.js +2 -0
- package/dist/{questions-BEPj98fZ.js → questions-B8by4Bbt.js} +3 -3
- package/dist/{questions-BEPj98fZ.js.map → questions-B8by4Bbt.js.map} +1 -1
- package/dist/{results-BcSHWbgB.js → results-Di7m8dQ3.js} +4 -4
- package/dist/{results-BcSHWbgB.js.map → results-Di7m8dQ3.js.map} +1 -1
- package/dist/{resume-DZ9AmfPv.js → resume-SUz1GskV.js} +3 -3
- package/dist/{resume-DZ9AmfPv.js.map → resume-SUz1GskV.js.map} +1 -1
- package/dist/{runtime-Dfb2r-cS.js → runtime-DdsfnRz_.js} +148 -6
- package/dist/runtime-DdsfnRz_.js.map +1 -0
- package/dist/{sandbox--ZXuscAZ.js → sandbox-D5v-of6d.js} +10 -5
- package/dist/sandbox-D5v-of6d.js.map +1 -0
- package/dist/sandbox.d.ts +4 -3
- package/dist/sandbox.js +1 -1
- package/dist/{server-Da60oe4b.js → server-BNnXtmS8.js} +7 -7
- package/dist/{server-Da60oe4b.js.map → server-BNnXtmS8.js.map} +1 -1
- package/dist/server-BNwoKzlm.js +2 -0
- package/dist/source-Bk_0ZaEY.js +2 -0
- package/dist/{source-BVOnL3sn.js → source-Dof0NQBm.js} +2 -2
- package/dist/{source-BVOnL3sn.js.map → source-Dof0NQBm.js.map} +1 -1
- package/dist/{src-TdJuDYqB.js → src-8rRXy4u9.js} +71 -124
- package/dist/src-8rRXy4u9.js.map +1 -0
- package/dist/{src-DdRlaiRc.js → src-BGWuWp6I.js} +1 -1
- package/dist/{src-D13vpMh0.js → src-BRt0O02i.js} +1 -1
- package/dist/{src-BrMbt1B2.js → src-Bx2x1ekD.js} +1 -1
- package/dist/{src-CxTC5kNI.js → src-COgE7991.js} +4 -6
- package/dist/{src-CxTC5kNI.js.map → src-COgE7991.js.map} +1 -1
- package/dist/{src-D_cXqmnI.js → src-C_4Umgh2.js} +9 -10
- package/dist/{src-D_cXqmnI.js.map → src-C_4Umgh2.js.map} +1 -1
- package/dist/src-DLa-ESW0.js +4 -0
- package/dist/src-cN0aB-WR.js +3 -0
- package/dist/{src-BT4dB5CO.js → src-cdIVjC5b.js} +1 -1
- package/dist/{src-BT5pwco0.js → src-sIsDHWYq.js} +1 -1
- package/dist/state-Bo5zg4da.js +2 -0
- package/dist/{state-B72h_jEw.js → state-CiXyLqg0.js} +5 -5
- package/dist/{state-B72h_jEw.js.map → state-CiXyLqg0.js.map} +1 -1
- package/dist/studio-app/assets/index-B9MCTfuu.css +2 -0
- package/dist/studio-app/assets/{index-Bgr-efZm.js → index-CROH2NB3.js} +12 -12
- package/dist/studio-app/index.html +2 -2
- package/dist/{summary-CJdxaRdM.js → summary-2OAWF2N7.js} +6 -6
- package/dist/{summary-CJdxaRdM.js.map → summary-2OAWF2N7.js.map} +1 -1
- package/dist/{summary-k3mhWWXv.js → summary-BTksrIxn.js} +58 -4
- package/dist/summary-BTksrIxn.js.map +1 -0
- package/dist/testing.d.ts +1 -1
- package/dist/{tools-D0gF2QoV.js → tools-B10Uj7OI.js} +4 -4
- package/dist/tools-B10Uj7OI.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-_LOFdPUf.d.ts → types-D8RLfkFP.d.ts} +195 -13
- package/dist/view/index.d.ts +44 -12
- package/dist/view/index.js +5 -4
- package/dist/views-Cs2Oqayd.js +3 -0
- package/dist/{views-DZ1YDqey.js → views-Dqa5ojfZ.js} +4 -4
- package/dist/{views-DZ1YDqey.js.map → views-Dqa5ojfZ.js.map} +1 -1
- package/dist/{worker-entry-DXeyWY-m.js → worker-entry-BAikHl9F.js} +9 -5
- package/dist/worker-entry-BAikHl9F.js.map +1 -0
- package/dist/{workspace-DpXdrslo.js → workspace-C-TrNIUc.js} +4 -4
- package/dist/{workspace-DpXdrslo.js.map → workspace-C-TrNIUc.js.map} +1 -1
- package/dist/{workspace-DzNL-3iO.js → workspace-Do_SiWBs.js} +1 -1
- package/docker/Dockerfile +7 -5
- package/package.json +1 -1
- package/dist/agent-Cfj1wD4D.js.map +0 -1
- package/dist/assemble-BbFVjHeS.js.map +0 -1
- package/dist/assemble-CXiTVyHD.js +0 -2
- package/dist/capture-BXhBfmsM.js.map +0 -1
- package/dist/capture-N7r3yeak.js +0 -2
- package/dist/cli-BMjTKY-D.js.map +0 -1
- package/dist/client-CwzU9CiW.js.map +0 -1
- package/dist/command-BK4kOgkH.js.map +0 -1
- package/dist/command-BqJtFiPy.js.map +0 -1
- package/dist/command-CTVGuH8y.js +0 -2
- package/dist/config-B-E0xRkf.js.map +0 -1
- package/dist/db-DFbtGIYy.js.map +0 -1
- package/dist/docker-2qns7hJ2.js.map +0 -1
- package/dist/files-BpYgF2z0.js +0 -2
- package/dist/files-YCCVaTQC.js +0 -2
- package/dist/harness-iqWeJQMM.js.map +0 -1
- package/dist/init-DBHyi92L.js +0 -2
- package/dist/model-BbLGRtOs.js +0 -2
- package/dist/model-BndnmcLm.js.map +0 -1
- package/dist/options-DPUlm4FI.js.map +0 -1
- package/dist/plan-PPc4-6xd.js.map +0 -1
- package/dist/query-noGbekGF.js +0 -2
- package/dist/runtime-Dfb2r-cS.js.map +0 -1
- package/dist/sandbox--ZXuscAZ.js.map +0 -1
- package/dist/server-DiGR4JxK.js +0 -2
- package/dist/source-Dqu0jKwx.js +0 -2
- package/dist/src-BHNiP-5A.js +0 -4
- package/dist/src-TdJuDYqB.js.map +0 -1
- package/dist/src-xczuQbE6.js +0 -3
- package/dist/state-Bf_Vq_pV.js +0 -2
- package/dist/studio-app/assets/index-CUx6GoL5.css +0 -2
- package/dist/summary-k3mhWWXv.js.map +0 -1
- package/dist/tools-D0gF2QoV.js.map +0 -1
- package/dist/views-Ds0GXWtX.js +0 -3
- package/dist/worker-entry-DXeyWY-m.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"images-
|
|
1
|
+
{"version":3,"file":"images-BMgdpMAX.js","names":[],"sources":["../src/sandbox/images.ts"],"mappings":";;;;;;;AAsCA,SAAgB,aACd,MACA;CACA,MAAM,0BAAU,IAAI,IAAiD;CACrE,KAAK,MAAM,EAAE,IAAI,SAAS,GAAG,SAAS,MAAM;EAC1C,IAAI,QAAQ,SAAS,UAAU;EAC/B,MAAM,UAAU,aAAa,IAAI,OAAO;EAExC,MAAM,EAAE,OAAO,MAAM,YAAY,YAAY;EAC7C,MAAM,MAAM,KAAK,UAAU;GAAE;GAAS,OAAO;IAAE;IAAO;IAAM;IAAY;GAAQ;EAAE,CAAC;EACnF,MAAM,SAAS,QAAQ,IAAI,GAAG,KAAK;GACjC,IAAI,YAAY,WAAW,UAAU,OAAO,IAAI,GAAG,UAAU,OAAO,EAAE,KAAK;GAC3E;GACA;GACA,OAAO;GACP,sBAAM,IAAI,IAAI;EAChB;EACA,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,GAAG;GACxB,OAAO,KAAK,IAAI,EAAE;GAClB,OAAO;EACT;EACA,QAAQ,IAAI,KAAK,MAAM;CACzB;CAEA,MAAM,yBAAS,IAAI,IAAoB;CACvC,OAAO,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,QAAQ;EACtD,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,QAAQ,SAAS,SAAS,eAAe;EACnE,MAAM,KAAK,OAAO,IAAI,KAAK,KAAK,KAAK;EACrC,OAAO,IAAI,OAAO,CAAC;EACnB,OAAO;GAAE,GAAG;GAAG,IAAI,MAAM,IAAI,QAAQ,GAAG,MAAM,IAAI,EAAE;EAAG;CACzD,CAAC;AACH;AAcA,MAAM,SAAS,WAAmB,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;;AAG5E,eAAe,WAAW,MAAc,KAAwD;CAC9F,MAAM,CAAC,QAAQ,OAAO,MAAM,QAAQ,IAAI,CAAC,UAAU,GAAG,GAAG,YAAY,IAAI,CAAC,CAAC;CAC3E,IAAI,CAAC,QAAQ;EACX,IAAI,EAAE,WAAW,MAAM,OAAO;GAAE,QAAQ;GAAS,QAAQ,MAAM,KAAK,IAAI,MAAM,IAAI,MAAM,EAAE;EAAG;EAE7F,OAAO,YAAY,IAAI,KAAK,IACxB;GAAE,QAAQ;GAAS,QAAQ,GAAG,KAAK,wBAAwB,IAAI;EAAQ,IACvE;GAAE,QAAQ;GAAW,QAAQ,oBAAoB,IAAI;EAAQ;CACnE;CACA,MAAM,MAAM,OAAO,OAAO,MAAM;CAChC,IAAI,WAAW,KAAK,OAAO;EAAE,QAAQ;EAAS,QAAQ,UAAU,KAAK;CAAsB;CAC3F,IAAI,OAAO,QAAQ,IAAI,QACrB,OAAO;EACL,QAAQ;EACR,QAAQ,GAAG,KAAK,mBAAmB,MAAM,GAAG,EAAE,KAAK,MAAM,IAAI,MAAM,EAAE;CACvE;CAEF,OAAO;EAAE,QAAQ;EAAS,QAAQ,YAAY,MAAM,IAAI,MAAM;CAAI;AACpE;;AAGA,eAAsB,YAAY,QAAqB,MAAgC;CACrF,MAAM,EAAE,SAAS,YAAY;CAC7B,MAAM,SAAkB,CAAC;CACzB,MAAM,SAAS,OAAO,QAAgB;EACpC,MAAM,OAAO,MAAM,UAAU,GAAG;EAChC,OAAO;GACL;GACA,QAAS,OAAO,UAAU;GAC1B,QAAQ,OAAO,UAAU,MAAM,KAAK,EAAE,EAAE,KAAK;EAC/C;CACF;CACA,MAAM,SAAS,YAAY;EACzB,MAAM,OAAO,MAAM,UAAU,SAAS,OAAO,CAAC;EAC9C,IAAI,CAAC,MAAM,OAAO;GAAE;GAAM,QAAQ;GAAwB,QAAQ;EAAgB;EAClF,OAAO,KAAK,OAAO,MAAM,aAAa,YAAY,IAC9C;GAAE;GAAM,QAAQ;GAAwB,QAAQ,UAAU,MAAM,KAAK,EAAE,EAAE;EAAG,IAC5E;GACE;GACA,QAAQ;GACR,QAAQ;EACV;CACN;CACA,IAAI,SAAwB;CAC5B,IAAI,QAAQ,OAAO;EACjB,MAAM,QAAQ,MAAM,UAAU,QAAQ,KAAK;EAC3C,SAAS,OAAO,MAAM;EACtB,OAAO,KAAK;GACV,MAAM;GACN,KAAK,QAAQ;GACb,QAAQ;GACR,GAAI,QACA;IAAE,QAAQ;IAAS,QAAQ,UAAU,MAAM,MAAM,EAAE,EAAE;GAAG,IACxD;IAAE,QAAQ;IAAW,QAAQ;GAAuC;EAC1E,CAAC;CACH,OAAO,IAAI,QAAQ,MAAM;EACvB,MAAM,MAAM,gBAAgB,SAAS,QAAQ,IAAI;EACjD,MAAM,SAAS,MAAM,WAAW,QAAQ,MAAM,GAAG;EACjD,SAAS,OAAO,WAAW,WAAY,MAAM,UAAU,GAAG,EAAA,EAAI,MAAM,OAAQ;EAC5E,OAAO,KAAK;GACV,MAAM,QAAQ,QAAQ;GACtB;GACA,QACE;GACF,GAAG;EACL,CAAC;CACH,OAAO,IAAI,QAAQ,YAAY;EAC7B,MAAM,OAAO,WAAW,QAAQ,UAAU;EAC1C,MAAM,KAAK,OAAO,MAAM,OAAO,IAAI;EACnC,MAAM,OAAO,cAAc,QAAQ,YAAY,SAAS,IAAI,MAAM,MAAM,IAAI,CAAC,CAAC;EAE9E,MAAM,MAAM,OAAO,KAAK,QAAQ,kBAAkB,8BAA8B,IAAI;EACpF,MAAM,QAAQ,MAAM,OAAO,IAAI;EAC/B,OAAO,KAAK;GACV,MAAM,cAAc,YAAY,MAAM,QAAQ,UAAU;GACxD,KAAK;GACL,QAAQ;GACR,QAAQ,MAAM;GACd,QAAQ,MAAM,QAAQ,CAAC,OAAO,MAAM,SAAS,GAAG,MAAM,OAAO,IAAI,KAAK;EACxE,CAAC;EACD,IAAI,MAAM;GACR,SAAS,MAAM,MAAM,MAAM;GAC3B,OAAO,KAAK;IACV,MAAM;IACN,KAAK;IACL,QAAQ,sBAAsB,gBAAgB,cAAc,SAAS,OAAO;IAC5E,QAAQ,IAAI,UAAU;IACtB,QAAQ,GAAG,SAAS,OAAO,EAAE,GAAG,IAAI,UAAU,KAAK,KAAK;GAC1D,CAAC;EACH,OAAO;GACL,MAAM,QAAQ,MAAM,OAAO,gBAAgB,SAAS,IAAI,CAAC;GACzD,SAAS,MAAM,MAAM,MAAM;GAC3B,OAAO,KAAK;IACV,MAAM;IACN,KAAK,gBAAgB,SAAS,GAAG;IACjC,QAAQ;IACR,QAAQ,MAAM;IACd,QAAQ,MAAM;GAChB,CAAC;EACH;CACF,OAAO;EACL,MAAM,KAAK,MAAM,OAAO;EACxB,SAAS,GAAG,WAAW,UAAW,GAAG,MAAM,MAAM,OAAQ;EACzD,OAAO,KAAK;GACV,MAAM;GACN,KAAK,SAAS,OAAO;GACrB,QAAQ;GACR,QAAQ,GAAG;GACX,QAAQ,GAAG;EACb,CAAC;CACH;CACA,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,QAAQ;EAChB,MAAM,QAAQ,SAAS,MAAM,OAAO,gBAAgB,EAAE,IAAI,OAAO,GAAG,KAAK,CAAC,IAAI;EAC9E,OAAO,KAAK;GACV,MAAM,WAAW,MAAM,OAAO,UAAU,MAAM,WAAW,IAAI,KAAK;GAClE,KAAK;GACL,QAAQ;GACR,QAAQ,OAAO,UAAU;GACzB,QAAQ,OAAO,UAAU;EAC3B,CAAC;CACH;CACA,OAAO;AACT;;AAGA,MAAa,YAAY,YAA2C,CAClE,GAAG,IAAI,IACL,QACG,QACE,EAAE,SAAS,QACT,CAAC,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,EAAE,cAAgB,EAAE,cAAc,WAAW,EAAE,UAAU,CACtF,CAAC,CACA,KAAK,MAAM,EAAE,OAAO,CACzB,CACF;AAEA,MAAM,QAAuB;CAAC;CAAW;CAAS;CAAS;AAAO;AAClE,MAAM,QAAqC;CAAE,OAAO;CAAK,OAAO;CAAK,OAAO;CAAK,SAAS;AAAI;AAC9F,MAAM,QAAqC;CACzC,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;AACX;;AAGA,MAAa,gBAAgB,WAC3B,MAAM,MAAM,MAAM,OAAO,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,KAAK;;AAG3D,eAAsB,eAAe,SAAwB,MAAiC;CAC5F,MAAM,QAAQ,CACZ,WAAW,QAAQ,OAAO,gBAAgB,QAAQ,WAAW,IAAI,KAAK,IAAI,4BAC5E;CACA,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,OAAO,OAAO,QAAQ,MAAM,OAAO,MAAM,OAAO,OAAO,UAAU,IAAI,KAAK,QAAQ;EACxF,MAAM,KAAK,GAAG,OAAO,KAAK,MAAM;EAChC,MAAM,SAAS,MAAM,YAAY,QAAQ,IAAI;EAC7C,MAAM,QAAQ,KAAK,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,KAAK,MAAM,CAAC;EAC1D,KAAK,MAAM,KAAK,QACd,MAAM,KACJ,KAAK,MAAM,EAAE,QAAQ,GAAG,EAAE,KAAK,OAAO,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,MAAM,EAAE,QAAQ,IAAI,EAAE,OAAO,EAC3F;CAEJ;CACA,OAAO;AACT;;AAGA,eAAsB,iBAAiB,SAAwB,MAAiC;CAC9F,IAAI,CAAC,QAAQ,QAAQ,OAAO,CAAC;CAC7B,MAAM,OAAO,MAAM,QAAQ,IACzB,QAAQ,IAAI,OAAO,MAAM;EACvB,MAAM,SAAS,MAAM,YAAY,GAAG,IAAI;EACxC,MAAM,SAAS,aAAa,MAAM;EAClC,MAAM,QAAQ,OAAO,MAAM,MAAM,EAAE,WAAW,MAAM;EACpD,OAAO;GAAE;GAAG;GAAQ,QAAQ,WAAW,UAAU,KAAK,KAAK,OAAO,KAAK,IAAI,OAAO;EAAS;CAC7F,CAAC,CACH;CACA,MAAM,SAAS,MAAmB,KAAK,QAAQ,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;CACrE,MAAM,UAAU,MAAM,QAAQ,MAAM,MAAM,CAAC,CAAC,CAAC,CAC1C,QAAQ,CAAC,CACT,KAAK,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,CACrC,KAAK,IAAI;CACZ,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,KAAK,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC;CACxD,OAAO,CACL,WAAW,QAAQ,OAAO,gBAAgB,QAAQ,WAAW,IAAI,KAAK,IAAI,IAAI,WAC9E,GAAG,KAAK,KACL,MAAM,KAAK,MAAM,EAAE,QAAQ,GAAG,EAAE,EAAE,GAAG,OAAO,KAAK,EAAE,IAAI,MAAM,EAAE,UAAU,EAAE,QAC9E,CACF;AACF;;;;;AAYA,eAAsB,YACpB,SACA,MACA,SACuB;CACvB,MAAM,YAAY,gBAAgB,EAAE,SAAS,QAAQ,QAAQ,UAAU,QAAQ,CAAC;CAChF,MAAM,WAAW,QAAQ,KAAK,OAAO;EACnC,MAAM;EACN,KAAK,UAAU,EAAE,SAAS,WAAW,EAAE,OAAO,CAAC;EAC/C,OAAO,EAAE;EACT,SAAS,EAAE;EACX,SAAS,WAAW,EAAE,OAAO;EAC7B,OAAO,CAAC;EACR,SAAS;CACX,EAAE;CACF,MAAM,WAAW,MAAM,WAAW,UAAU;EAC1C;EACA;EACA,KAAK,QAAQ;EACb,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;CACrD,CAAC;CACD,OAAO,QAAQ,KAAK,GAAG,MAAM;EAC3B,MAAM,IAAI,SAAS;EACnB,OAAO,EAAE,KACL;GAAE,GAAG;GAAG,IAAI;GAAM,OAAO,EAAE;GAAmC,IAAI,EAAE;EAAG,IACvE;GAAE,GAAG;GAAG,IAAI;GAAO,SAAS,EAAE;GAAS,IAAI,EAAE;EAAG;CACtD,CAAC;AACH;;AAGA,MAAM,cAAc,aAA8C;CAAE,MAAM;CAAQ,QAAQ;AAAQ"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $ as HarnessData, A as DirectoryOptions, At as Turn, B as EsConfig, Bt as EvalEntry, C as CompareCut, Ct as Secret, Dt as SkillSource, E as ConfigHelpers, Et as SetupStep, F as EnvVars, Ft as Wait, H as EvalOptions, I as Environment, It as WebEventType, L as EnvironmentHelpers, M as Duration, N as EgressPolicy, Nt as Verdict, P as EgressRecord, Pt as Viewport, Q as GroupOptions, R as EnvironmentInput, Rt as AttemptRecord, S as CompareAnswer, T as CompareQuestion, Tt as ServeHandle, U as ExecResult, Ut as FamilyRecord, V as EvalFn, Yt as RunRecord, _ as Cell, _t as PinnableKey, a as BackgroundHandle, at as JudgeArtifactRef, bt as ProfileLimit, c as Capture, d as CaptureOptions, dt as McpServer, f as CaptureUse, g as CaptureWindow, h as CaptureWait, ht as PairVote, i as AttemptContext, it as HostFile, jt as TurnData, k as Diff, kt as TranscriptEvent, l as CaptureCall, lt as JudgeOptions, m as CaptureUseScript, mt as PairFactor, nt as HarnessOption, ot as JudgeCallOptions, p as CaptureUseAction, rt as HistoryConfig, s as Budget, st as JudgeFailure, tt as HarnessEdit, u as CaptureInstruction, v as CellRole, vt as Profile, w as CompareOptions, wt as SecretTemplate, yt as ProfileDefaults, z as EnvironmentSpec, zt as CellRecord } from "./types-D8RLfkFP.js";
|
|
2
2
|
import { Duplex } from "node:stream";
|
|
3
3
|
import { ExpectStatic, Tester } from "@vitest/expect";
|
|
4
4
|
import "@ai-sdk/harness";
|
|
@@ -51,6 +51,7 @@ declare module '@vitest/expect' {
|
|
|
51
51
|
toHaveCalledTool(name: string | RegExp, input?: string | RegExp | Record<string, unknown>): void;
|
|
52
52
|
toHaveRun(command?: RegExp | string): void;
|
|
53
53
|
toHaveEdited(path?: RegExp | string): void;
|
|
54
|
+
toHaveChanged(paths?: string | RegExp | ReadonlyArray<string | RegExp>): void;
|
|
54
55
|
toPass(): void;
|
|
55
56
|
}
|
|
56
57
|
}
|
|
@@ -100,14 +101,25 @@ type EvaluateFn<F = EvalFn> = {
|
|
|
100
101
|
(name: string, fn: F, options?: EvalOptions): void;
|
|
101
102
|
(name: string, options: EvalOptions, fn: F): void;
|
|
102
103
|
};
|
|
103
|
-
/**
|
|
104
|
-
|
|
104
|
+
/** How an `evaluate.each` treats its cases. */
|
|
105
|
+
type EachOptions = {
|
|
106
|
+
/**
|
|
107
|
+
* Pool the cases into one suite: a pass rate over them and, with `compare`, one paired lift
|
|
108
|
+
* that `minLift` gates. On by default; `false` keeps them independent evals.
|
|
109
|
+
*/
|
|
110
|
+
pool?: boolean;
|
|
111
|
+
};
|
|
112
|
+
/**
|
|
113
|
+
* One eval per case, pooled into one suite:
|
|
114
|
+
* `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
|
|
115
|
+
*/
|
|
116
|
+
declare const evalEach: <T>(cases: readonly T[], options?: EachOptions) => EvaluateFn<(c: AttemptContext, item: T) => Promise<void>>;
|
|
105
117
|
export declare const evaluate: EvaluateFn & {
|
|
106
118
|
skip: EvaluateFn;
|
|
107
119
|
only: EvaluateFn;
|
|
108
120
|
todo: (name: string) => void;
|
|
109
|
-
|
|
121
|
+
each: typeof evalEach;
|
|
110
122
|
};
|
|
111
123
|
//#endregion
|
|
112
|
-
export type { AttemptContext, AttemptRecord, BackgroundHandle, Budget, Capture, CaptureCall, CaptureInstruction, CaptureOptions, CaptureUse, CaptureUseAction, CaptureUseScript, CaptureWait, CaptureWindow, Cell, CellRecord, CellRole, CompareAnswer, CompareCut, CompareOptions, CompareQuestion, ConfigHelpers, Diff, DirectoryOptions, Duration, EgressPolicy, EgressRecord, EnvVars, Environment, EnvironmentHelpers, EnvironmentInput, EnvironmentSpec, EsConfig, EvalEntry, EvalFn, EvalOptions, ExecResult, GroupOptions, HarnessData, HarnessEdit, HarnessOption, HistoryConfig, HostFile, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeOptions, McpServer, PairFactor, PairVote, PinnableKey, Profile, ProfileDefaults, ProfileLimit, RunRecord, Secret, SecretTemplate, ServeHandle, SetupStep, SkillSource, TranscriptEvent, Turn, TurnData, Verdict, Viewport, Wait, WebEventType };
|
|
124
|
+
export type { AttemptContext, AttemptRecord, BackgroundHandle, Budget, Capture, CaptureCall, CaptureInstruction, CaptureOptions, CaptureUse, CaptureUseAction, CaptureUseScript, CaptureWait, CaptureWindow, Cell, CellRecord, CellRole, CompareAnswer, CompareCut, CompareOptions, CompareQuestion, ConfigHelpers, Diff, DirectoryOptions, Duration, EachOptions, EgressPolicy, EgressRecord, EnvVars, Environment, EnvironmentHelpers, EnvironmentInput, EnvironmentSpec, EsConfig, EvalEntry, EvalFn, EvalOptions, ExecResult, FamilyRecord, GroupOptions, HarnessData, HarnessEdit, HarnessOption, HistoryConfig, HostFile, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeOptions, McpServer, PairFactor, PairVote, PinnableKey, Profile, ProfileDefaults, ProfileLimit, RunRecord, Secret, SecretTemplate, ServeHandle, SetupStep, SkillSource, TranscriptEvent, Turn, TurnData, Verdict, Viewport, Wait, WebEventType };
|
|
113
125
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.js
CHANGED
|
@@ -1,25 +1,26 @@
|
|
|
1
|
-
import { i as HarnessCapabilityError, r as withLease } from "./client-
|
|
1
|
+
import { i as HarnessCapabilityError, r as withLease } from "./client-Bbo9SPlK.js";
|
|
2
2
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-yDAEwmZo.js";
|
|
3
3
|
import { t as parseDuration } from "./duration-JsJrIWXq.js";
|
|
4
4
|
import { t as inCI } from "./env-BQqj93gu.js";
|
|
5
5
|
import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
|
|
6
|
-
import { C as stepCommand, E as workspaceOf, T as workspaceLabel, _ as isSecret, b as parseGitRepo, c as defineEnvironment, d as filesOf, h as isHostFile, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES, v as isSecretTemplate, y as mcpHosts } from "./environment-
|
|
6
|
+
import { C as stepCommand, E as workspaceOf, T as workspaceLabel, _ as isSecret, b as parseGitRepo, c as defineEnvironment, d as filesOf, h as isHostFile, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES, v as isSecretTemplate, y as mcpHosts } from "./environment-1jE-HhO-.js";
|
|
7
7
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
8
|
-
import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-
|
|
8
|
+
import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-BhzNh7Yi.js";
|
|
9
9
|
import { c as relativeFile, i as evalSlug, l as sha256, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
10
10
|
import { c as judgeModelName, d as isApiError, i as badJudgeList, u as runJudge } from "./judge-CeYPN8P_.js";
|
|
11
|
-
import { S as placeholder
|
|
12
|
-
import {
|
|
13
|
-
import { n as
|
|
14
|
-
import {
|
|
11
|
+
import { S as placeholder } from "./src-8rRXy4u9.js";
|
|
12
|
+
import { n as shellQuote, t as run } from "./process-CSVO_jsc.js";
|
|
13
|
+
import { a as evalPlan, d as sandboxId, f as sandboxSetBy, h as unbrokeredError, i as checkOptions, l as matchesTags, m as unbrokered, n as cascade, o as expand, p as secretVarsIn, s as harnessId, t as brokeredIn } from "./options-CKqyPGxX.js";
|
|
14
|
+
import { n as badCharacterAt, s as defineConfig } from "./config-CGvBeewH.js";
|
|
15
|
+
import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile, u as expect } from "./runtime-DdsfnRz_.js";
|
|
15
16
|
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CGPV3BZY.js";
|
|
16
17
|
import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
|
|
17
18
|
import { n as clip, r as formatDuration, t as capMiddle } from "./text-gEcKjAyp.js";
|
|
18
|
-
import { a as asCheck, i as writeCaptureSidecars, n as serveCommand, o as createChecks, t as capturePage } from "./capture-
|
|
19
|
-
import { t as contextTaker } from "./context-
|
|
19
|
+
import { a as asCheck, c as snapshot, i as writeCaptureSidecars, n as serveCommand, o as createChecks, s as diffSinceSnapshot, t as capturePage } from "./capture-Bp74Z6Fr.js";
|
|
20
|
+
import { t as contextTaker } from "./context-DbSMEVKs.js";
|
|
20
21
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
21
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
22
|
-
import { n as judgeDiff } from "./patch-
|
|
22
|
+
import { i as preparedWorkspace } from "./workspace-C-TrNIUc.js";
|
|
23
|
+
import { n as judgeDiff } from "./patch-bDV8YFW0.js";
|
|
23
24
|
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
24
25
|
import { basename, isAbsolute, join, relative, sep } from "node:path";
|
|
25
26
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
@@ -865,7 +866,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
865
866
|
//#region src/harness/index.ts
|
|
866
867
|
async function createHarness(descriptor) {
|
|
867
868
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
868
|
-
const { createAgentHarness } = await import("./agent-
|
|
869
|
+
const { createAgentHarness } = await import("./agent-WSmTaesf.js");
|
|
869
870
|
return createAgentHarness(descriptor);
|
|
870
871
|
}
|
|
871
872
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
@@ -892,8 +893,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
892
893
|
if (debug) traceEnvironment(debug, spec, env);
|
|
893
894
|
record.image = null;
|
|
894
895
|
if (spec.sandbox.kind === "docker") {
|
|
895
|
-
const { preparedFor } = await import("./preflight-
|
|
896
|
-
const { dockerProvider } = await import("./providers-
|
|
896
|
+
const { preparedFor } = await import("./preflight-D5LW7Oqi.js");
|
|
897
|
+
const { dockerProvider } = await import("./providers-CpEfNU1F.js");
|
|
897
898
|
signal.throwIfAborted();
|
|
898
899
|
const prepared = preparedFor(root, runId, dockerProvider(), spec.sandbox, spec.harness);
|
|
899
900
|
if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
|
|
@@ -1046,7 +1047,7 @@ async function judgeState(turn, output, { include, files }, readFile) {
|
|
|
1046
1047
|
* checks after it still run.
|
|
1047
1048
|
*/
|
|
1048
1049
|
function judgeCaller(a, readFile) {
|
|
1049
|
-
|
|
1050
|
+
const ask = async (question, { on, ...call } = {}) => {
|
|
1050
1051
|
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
1051
1052
|
const { threshold } = call;
|
|
1052
1053
|
if (threshold !== void 0 && !(typeof threshold === "number" && threshold >= 0 && threshold <= 1)) throw new Error(`c.judge() threshold must be between 0 and 1 (got ${String(threshold)})`);
|
|
@@ -1160,6 +1161,7 @@ function judgeCaller(a, readFile) {
|
|
|
1160
1161
|
return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
|
|
1161
1162
|
}
|
|
1162
1163
|
};
|
|
1164
|
+
return async (question, options) => Object.defineProperty(await ask(question, options), SUBJECT, { value: question });
|
|
1163
1165
|
}
|
|
1164
1166
|
//#endregion
|
|
1165
1167
|
//#region src/sandbox/index.ts
|
|
@@ -1168,7 +1170,7 @@ async function createSandbox(descriptor, options) {
|
|
|
1168
1170
|
switch (descriptor.kind) {
|
|
1169
1171
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
1170
1172
|
case "docker": {
|
|
1171
|
-
const { createDockerSandbox } = await import("./docker-
|
|
1173
|
+
const { createDockerSandbox } = await import("./docker-Df_YOxxB.js");
|
|
1172
1174
|
return await createDockerSandbox(descriptor, options);
|
|
1173
1175
|
}
|
|
1174
1176
|
}
|
|
@@ -1224,6 +1226,27 @@ function recordFailure(a, error, timedOut) {
|
|
|
1224
1226
|
record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
|
|
1225
1227
|
}
|
|
1226
1228
|
/**
|
|
1229
|
+
* A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
|
|
1230
|
+
* whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
|
|
1231
|
+
*/
|
|
1232
|
+
async function recordOom(a, sandbox) {
|
|
1233
|
+
const oom = await sandbox?.oomKills?.().catch(() => null);
|
|
1234
|
+
const at = oom && `the ${oom.limit} limit${oom.swap ? ` (+${oom.swap} swap)` : ""}`;
|
|
1235
|
+
if (oom) debugLog("teardown", `oom_kill ${oom.kills} at ${at}`);
|
|
1236
|
+
if (!oom?.kills) return void 0;
|
|
1237
|
+
const message = `out of memory: ${oom.kills === 1 ? "1 process" : `${oom.kills} processes`} killed at ${at}; raise docker({ memory })`;
|
|
1238
|
+
a.record.error = {
|
|
1239
|
+
kind: "infra",
|
|
1240
|
+
message,
|
|
1241
|
+
resource: {
|
|
1242
|
+
cause: "oom",
|
|
1243
|
+
kills: oom.kills
|
|
1244
|
+
}
|
|
1245
|
+
};
|
|
1246
|
+
a.record.outcome = "infra_error";
|
|
1247
|
+
return new InfraError(message);
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1227
1250
|
* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
|
|
1228
1251
|
* Refused hosts the harness always tries are `expectedBlocked`, summarised in one `-v` line.
|
|
1229
1252
|
*/
|
|
@@ -1280,36 +1303,6 @@ function turnErrors(transcripts) {
|
|
|
1280
1303
|
});
|
|
1281
1304
|
}
|
|
1282
1305
|
//#endregion
|
|
1283
|
-
//#region src/diff.ts
|
|
1284
|
-
const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null";
|
|
1285
|
-
/** Records the workspace state before the agent runs, in a git dir outside the workspace. */
|
|
1286
|
-
async function snapshot(sandbox, signal) {
|
|
1287
|
-
const r = await sandbox.exec(`export GIT_DIR="$PWD/../.reeval-git-$(basename "$PWD")" GIT_WORK_TREE="$PWD" && ${GIT} init -q && ${GIT} add -A && ${GIT} commit -q --allow-empty -m base`, { signal });
|
|
1288
|
-
if (r.exitCode !== 0) throw new InfraError(`Failed to snapshot the workspace: ${r.stderr.trim()}`);
|
|
1289
|
-
}
|
|
1290
|
-
async function diffSinceSnapshot(sandbox, signal) {
|
|
1291
|
-
const env = `export GIT_DIR="$PWD/../.reeval-git-$(basename "$PWD")" GIT_WORK_TREE="$PWD" && ${GIT} add -A && `;
|
|
1292
|
-
const patch = await sandbox.exec(`${env}${GIT} diff --cached --binary HEAD`, { signal });
|
|
1293
|
-
const names = await sandbox.exec(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
|
|
1294
|
-
if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
|
|
1295
|
-
const status = {
|
|
1296
|
-
A: "added",
|
|
1297
|
-
M: "modified",
|
|
1298
|
-
D: "deleted"
|
|
1299
|
-
};
|
|
1300
|
-
const files = names.stdout.split("\n").filter(Boolean).map((line) => {
|
|
1301
|
-
const [code = "M", ...rest] = line.split(" ");
|
|
1302
|
-
return {
|
|
1303
|
-
path: rest.join(" "),
|
|
1304
|
-
status: status[code[0]] ?? "modified"
|
|
1305
|
-
};
|
|
1306
|
-
});
|
|
1307
|
-
return {
|
|
1308
|
-
patch: patch.stdout,
|
|
1309
|
-
files
|
|
1310
|
-
};
|
|
1311
|
-
}
|
|
1312
|
-
//#endregion
|
|
1313
1306
|
//#region src/sandbox/phases.ts
|
|
1314
1307
|
const strip = ({ phase: _, ...sub }) => sub;
|
|
1315
1308
|
/**
|
|
@@ -1710,7 +1703,7 @@ async function runAttempt(spec, fn, handle) {
|
|
|
1710
1703
|
});
|
|
1711
1704
|
return withDebug(debug, () => attemptBody(spec, fn, handle, id, debug));
|
|
1712
1705
|
}
|
|
1713
|
-
async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited }, id, debug) {
|
|
1706
|
+
async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited, assertions }, id, debug) {
|
|
1714
1707
|
const setupStart = performance.now();
|
|
1715
1708
|
const ctx = getRunContext();
|
|
1716
1709
|
const { runId, stagingDir, config } = ctx;
|
|
@@ -1733,7 +1726,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1733
1726
|
return starting;
|
|
1734
1727
|
};
|
|
1735
1728
|
const takeContext = contextTaker(a);
|
|
1736
|
-
const { exec, readFile } = createChecks(a);
|
|
1729
|
+
const { exec, readFile, writeFile } = createChecks(a);
|
|
1737
1730
|
const c = {
|
|
1738
1731
|
get signal() {
|
|
1739
1732
|
return a.signal;
|
|
@@ -1741,6 +1734,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1741
1734
|
turns: a.turns,
|
|
1742
1735
|
exec,
|
|
1743
1736
|
readFile,
|
|
1737
|
+
writeFile,
|
|
1744
1738
|
serve: (command, port) => serveCommand(a, command, port),
|
|
1745
1739
|
capture: (target, options) => capturePage(a, target, options),
|
|
1746
1740
|
attempt: {
|
|
@@ -1762,6 +1756,8 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1762
1756
|
judge: judgeCaller(a, readFile)
|
|
1763
1757
|
};
|
|
1764
1758
|
const timedOut = () => !!timer?.aborted && !testSignal.aborted;
|
|
1759
|
+
let failure;
|
|
1760
|
+
let oom;
|
|
1765
1761
|
try {
|
|
1766
1762
|
a.report({ kind: "sandbox" });
|
|
1767
1763
|
const where = `${id} [${attemptId(spec.cell.key, spec.n)}]`;
|
|
@@ -1778,8 +1774,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1778
1774
|
if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
|
|
1779
1775
|
} catch (error) {
|
|
1780
1776
|
recordFailure(a, error, timedOut());
|
|
1781
|
-
|
|
1782
|
-
throw error;
|
|
1777
|
+
failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
|
|
1783
1778
|
} finally {
|
|
1784
1779
|
if (!a.signal.aborted) await unawaited().catch(() => {});
|
|
1785
1780
|
clock?.clear();
|
|
@@ -1792,9 +1787,16 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1792
1787
|
await a.session?.close().catch(() => {});
|
|
1793
1788
|
if (a.session) debug.log("teardown", "harness session closed");
|
|
1794
1789
|
recordEgress(a, sandbox);
|
|
1790
|
+
oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
|
|
1795
1791
|
await sandbox?.destroy();
|
|
1796
1792
|
if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
|
|
1797
1793
|
const { record } = a;
|
|
1794
|
+
const checked = assertions();
|
|
1795
|
+
if (checked.length) record.assertions = checked;
|
|
1796
|
+
for (const x of checked) {
|
|
1797
|
+
const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
|
|
1798
|
+
debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
|
|
1799
|
+
}
|
|
1798
1800
|
const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
|
|
1799
1801
|
debug.log("teardown", `outcome ${record.outcome}${error}`);
|
|
1800
1802
|
const rel = (path) => relativeFile(config.root, path) || ".";
|
|
@@ -1803,6 +1805,8 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1803
1805
|
setMeta(await writeRecord(a, stagingDir, id));
|
|
1804
1806
|
await debug.finish(debug.verbosity === 2 || record.outcome !== "pass");
|
|
1805
1807
|
}
|
|
1808
|
+
if (oom) throw oom;
|
|
1809
|
+
if (failure) throw failure.error;
|
|
1806
1810
|
}
|
|
1807
1811
|
//#endregion
|
|
1808
1812
|
//#region src/plan-eval.ts
|
|
@@ -1861,6 +1865,11 @@ function planEval(declaration, ctx) {
|
|
|
1861
1865
|
if (mcp) throw new UsageError(`${where} on ${harnessId(planned.harness)}: ${mcp}`);
|
|
1862
1866
|
}
|
|
1863
1867
|
const path = ctx.file ? relativeFile(ctx.config.root, ctx.file) : null;
|
|
1868
|
+
const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
|
|
1869
|
+
const family = declaration.family && {
|
|
1870
|
+
id: evalId(rankFile, [...groups, declaration.family.name]),
|
|
1871
|
+
...declaration.family
|
|
1872
|
+
};
|
|
1864
1873
|
const harnessBy = resolved.sources?.harness ?? ctx.config.file;
|
|
1865
1874
|
const plan = evalPlan(resolvedEval, plannedAttempts, {
|
|
1866
1875
|
configFile: ctx.config.file,
|
|
@@ -1887,7 +1896,8 @@ function planEval(declaration, ctx) {
|
|
|
1887
1896
|
brokered: [],
|
|
1888
1897
|
secretVars: [],
|
|
1889
1898
|
egress: first.egress,
|
|
1890
|
-
plan
|
|
1899
|
+
plan,
|
|
1900
|
+
...family ? { family } : {}
|
|
1891
1901
|
};
|
|
1892
1902
|
return [{
|
|
1893
1903
|
name: `${name} [${attemptId(first.cell.key, 1)}]`,
|
|
@@ -1901,7 +1911,6 @@ function planEval(declaration, ctx) {
|
|
|
1901
1911
|
const workspace = workspaceOf(planned.environment.layers);
|
|
1902
1912
|
const files = filesOf(planned.environment.layers);
|
|
1903
1913
|
const setBy = setByOf(planned);
|
|
1904
|
-
const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
|
|
1905
1914
|
const id = evalId(rankFile, [...groups, name]);
|
|
1906
1915
|
const key = attemptKey(id, planned.cell.key, planned.n);
|
|
1907
1916
|
const skip = ctx.completed?.includes(key) ? "already completed in this run" : void 0;
|
|
@@ -1926,7 +1935,8 @@ function planEval(declaration, ctx) {
|
|
|
1926
1935
|
...plan,
|
|
1927
1936
|
skip
|
|
1928
1937
|
} : plan,
|
|
1929
|
-
...skip ? { skip } : {}
|
|
1938
|
+
...skip ? { skip } : {},
|
|
1939
|
+
...family ? { family } : {}
|
|
1930
1940
|
};
|
|
1931
1941
|
return {
|
|
1932
1942
|
name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
|
|
@@ -2049,7 +2059,7 @@ function checkUnique(where, groups, name, hint) {
|
|
|
2049
2059
|
if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
|
|
2050
2060
|
declared.add(key);
|
|
2051
2061
|
}
|
|
2052
|
-
function register(mode, name, own, fn, hint = "") {
|
|
2062
|
+
function register(mode, name, own, fn, hint = "", family) {
|
|
2053
2063
|
const groups = groupStack.map((g) => g.name);
|
|
2054
2064
|
checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
|
|
2055
2065
|
const body = `b${bodies.size + 1}`;
|
|
@@ -2059,7 +2069,8 @@ function register(mode, name, own, fn, hint = "") {
|
|
|
2059
2069
|
name,
|
|
2060
2070
|
mode,
|
|
2061
2071
|
options: data(own),
|
|
2062
|
-
body
|
|
2072
|
+
body,
|
|
2073
|
+
...family ? { family } : {}
|
|
2063
2074
|
});
|
|
2064
2075
|
const tests = planEval({
|
|
2065
2076
|
name,
|
|
@@ -2068,7 +2079,8 @@ function register(mode, name, own, fn, hint = "") {
|
|
|
2068
2079
|
name,
|
|
2069
2080
|
options
|
|
2070
2081
|
})),
|
|
2071
|
-
options: own
|
|
2082
|
+
options: own,
|
|
2083
|
+
...family ? { family } : {}
|
|
2072
2084
|
}, {
|
|
2073
2085
|
...getRunContext(),
|
|
2074
2086
|
file: collectingFile(),
|
|
@@ -2089,19 +2101,37 @@ const defineEval = (mode) => (name, a, b) => {
|
|
|
2089
2101
|
function formatName(template, item, index) {
|
|
2090
2102
|
return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => typeof item === "object" && item !== null && key in item ? String(item[key]) : whole);
|
|
2091
2103
|
}
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
const
|
|
2095
|
-
const
|
|
2104
|
+
function checkEachOptions(options) {
|
|
2105
|
+
if (options === void 0) return {};
|
|
2106
|
+
const keys = typeof options === "object" && options !== null ? Object.keys(options) : null;
|
|
2107
|
+
const pool = options?.pool;
|
|
2108
|
+
if (keys?.every((k) => k === "pool") && (pool === void 0 || typeof pool === "boolean")) return options;
|
|
2109
|
+
throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
|
|
2110
|
+
}
|
|
2111
|
+
/**
|
|
2112
|
+
* One eval per case, pooled into one suite:
|
|
2113
|
+
* `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
|
|
2114
|
+
*/
|
|
2115
|
+
const evalEach = (cases, options) => (name, a, b) => {
|
|
2116
|
+
const { pool = true } = checkEachOptions(options);
|
|
2117
|
+
if (!Array.isArray(cases)) throw new UsageError(`evaluate.each takes an array of cases (got ${typeof cases}), e.g. evaluate.each([{ n: 1 }, { n: 2 }])("case $n", async (c, item) => { … }).`);
|
|
2118
|
+
const [own, fn] = splitArgs("evaluate.each", name, a, b);
|
|
2119
|
+
const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
|
|
2120
|
+
const pooled = pool && cases.length >= 2;
|
|
2096
2121
|
cases.forEach((item, i) => {
|
|
2097
|
-
|
|
2122
|
+
const family = pooled ? {
|
|
2123
|
+
name,
|
|
2124
|
+
case: i,
|
|
2125
|
+
cases: cases.length
|
|
2126
|
+
} : void 0;
|
|
2127
|
+
register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
|
|
2098
2128
|
});
|
|
2099
2129
|
};
|
|
2100
2130
|
const evaluate = Object.assign(defineEval("run"), {
|
|
2101
2131
|
skip: defineEval("skip"),
|
|
2102
2132
|
only: defineEval("only"),
|
|
2103
2133
|
todo: registerTodo,
|
|
2104
|
-
|
|
2134
|
+
each: evalEach
|
|
2105
2135
|
});
|
|
2106
2136
|
//#endregion
|
|
2107
2137
|
export { DEFAULT_DIRECTORY_IGNORES, DEFAULT_EGRESS_ALLOW, DEFAULT_SKILL_EXCLUDE, HarnessCapabilityError, defineConfig, defineEnvironment, evaluate, expect, group };
|