reeval 0.15.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
- package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
- package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
- package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
- package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
- package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
- package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
- package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
- package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
- package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +63 -16
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
- package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
- package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
- package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
- package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
- package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
- package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
- package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
- package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
- package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +52 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
- package/dist/view/index.d.ts +16 -4
- package/dist/view/index.js +6 -6
- package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -1,36 +1,39 @@
|
|
|
1
|
-
import { t as isObjectLike } from "./object-
|
|
2
|
-
import { r as withLease } from "./client-
|
|
1
|
+
import { t as isObjectLike } from "./object-BkXGPKWY.js";
|
|
2
|
+
import { r as withLease } from "./client-DMmOkbRA.js";
|
|
3
3
|
import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
|
|
4
4
|
import { h as parseDuration, n as dataProblem, p as MAX_TIMER_MS } from "./shared-B_lvGqWz.js";
|
|
5
|
-
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-
|
|
6
|
-
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-
|
|
5
|
+
import { A as sandboxPaths, F as workspaceOf, a as callerDir, d as environmentFrom, f as environmentPaths, h as globFilter, m as filesOf } from "./environment-Cf5HMjib.js";
|
|
6
|
+
import { O as pluginSourceLabel, S as pluginBuildId, b as localSkills, c as PluginProblem, d as builtPlugins, f as droppedPaths, h as layeredPlugins, l as buildRefusal, m as fetchesAtStart, n as PLUGIN_DIR, o as PLUGIN_ROOT, p as duplicateLocalPlugin, r as PLUGIN_MANIFEST, s as PLUGIN_ROOT_VAR, v as localPlugin, w as pluginCall } from "./plugins-ngQEJBjJ.js";
|
|
7
7
|
import { c as relativeFile, i as evalSlug, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
8
|
-
import { $ as
|
|
9
|
-
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-
|
|
8
|
+
import { $ as egressLabel, A as harnessId, At as NO_JUDGE_INPUT, B as sandboxId, Bt as decideState, D as checkOptions, Dt as writePluginFiles, Et as refusedPlugins, F as plainHeaders, Ft as judgeProblem, G as unbrokeredError, H as sandboxSetBy, Ht as resolveJudgeArtifacts, K as mergeJudge, M as majorityOf, Mt as inputsOn, N as matchesTags, O as evalPlan, P as plainHeaderError, R as refuseIllegalCompare, Rt as artifactRefsOf, S as resolveGateway, St as expectedBlockedFor, T as cascade, Tt as refusedMcp, U as secretVarsIn, V as sandboxProvider, Vt as filePartsOf, W as unbrokered, X as allowedHosts, _ as fellBackToDirect, _t as agentMcp, b as resolveModelGateway, bt as claudeMcpServers, it as createFakeSandbox, k as expand, kt as INPUT_KEYS, nt as unenforceable, q as resolveStateOptions, tt as isDirect, v as gatewayScope, w as brokeredIn, wt as layoutOf, yt as brokeringFor, z as runsOf, zt as badJudgeArtifacts } from "./config-CneBZB84.js";
|
|
9
|
+
import { C as vercelCredentialProblem, f as runJudge, h as isApiError, l as judgeModelName } from "./judge-Bg4v4e_K.js";
|
|
10
10
|
import { v as vertexSettings } from "./vertex-DypYjx71.js";
|
|
11
|
-
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
11
|
+
import { d as redactJson, g as redactValue, h as redactSplitSecrets, m as redactSecrets, o as findSecrets } from "./redact-DVIaahLv.js";
|
|
12
12
|
import { a as plural, i as formatDuration, n as clip, s as withoutBom, t as capMiddle } from "./text-DTigcA64.js";
|
|
13
13
|
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
|
-
import {
|
|
14
|
+
import { pt as shellQuote, t as MCP_TIMER, ut as OPAQUE_OUTPUT } from "./src-C0ySiej5.js";
|
|
15
15
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
16
|
-
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-
|
|
17
|
-
import {
|
|
18
|
-
import { t as
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
16
|
+
import { a as registerTest, i as registerGroup, n as declareHarnesses, r as getRunContext, s as SUBJECT, t as collectingFile } from "./runtime-CZVpkr04.js";
|
|
17
|
+
import { c as formatBytes } from "./debug-CgG3T071.js";
|
|
18
|
+
import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CpbZ9arV.js";
|
|
19
|
+
import { t as attemptDebugDir } from "./files-D1y8Pg4Z.js";
|
|
20
|
+
import { r as openRouterFallbacks } from "./openrouter-models-Bo8PZ4Ew.js";
|
|
21
|
+
import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-iQ6kYJV0.js";
|
|
21
22
|
import { t as contextTaker } from "./context-Dgn48BqT.js";
|
|
22
|
-
import { a as skillFiles, i as resolveEnvironment } from "./resolve-
|
|
23
|
+
import { a as skillFiles, i as resolveEnvironment } from "./resolve-D_cLa4tW.js";
|
|
23
24
|
import { t as PrepareError } from "./provider-CmAYTQQs.js";
|
|
24
|
-
import { i as preparedWorkspace } from "./workspace-
|
|
25
|
+
import { i as preparedWorkspace } from "./workspace-CBuI9Uds.js";
|
|
25
26
|
import { t as patchPath } from "./patch-CquGAGX5.js";
|
|
26
|
-
import { n as judgeDiff } from "./patch-
|
|
27
|
-
import { t as transcriptForJudge } from "./transcript-
|
|
28
|
-
import { i as
|
|
27
|
+
import { n as judgeDiff } from "./patch-6T9XAipm.js";
|
|
28
|
+
import { t as transcriptForJudge } from "./transcript-BJrKfgRW.js";
|
|
29
|
+
import { i as writeTar, n as readTar, t as extractTar } from "./tarball-CHcahyNK.js";
|
|
30
|
+
import { i as optionsHash, n as caseHash, t as bodyHash } from "./eval-source-CAE4y2Sr.js";
|
|
29
31
|
import { dirname, join } from "node:path";
|
|
30
32
|
import { parse, stringify } from "yaml";
|
|
31
33
|
import { z } from "zod";
|
|
32
34
|
import { setTimeout as setTimeout$1 } from "node:timers/promises";
|
|
33
|
-
import {
|
|
35
|
+
import { gunzipSync, gzipSync } from "node:zlib";
|
|
36
|
+
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
34
37
|
//#region src/attempt/assertions.ts
|
|
35
38
|
const MESSAGE_LIMIT = 1024;
|
|
36
39
|
const LABEL_LIMIT = 60;
|
|
@@ -76,6 +79,7 @@ function assertionRecords(raw) {
|
|
|
76
79
|
matcher: r.matcher,
|
|
77
80
|
status: r.status,
|
|
78
81
|
soft: r.soft,
|
|
82
|
+
...r.at ? { at: r.at } : {},
|
|
79
83
|
...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
|
|
80
84
|
...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
|
|
81
85
|
};
|
|
@@ -84,7 +88,7 @@ function assertionRecords(raw) {
|
|
|
84
88
|
const CAUGHT_HINT = "(the eval caught this failed check, which still fails the attempt; to wait or retry, use expect.poll)";
|
|
85
89
|
/**
|
|
86
90
|
* What fails the attempt at its end for the checks that failed without ending the body: the first
|
|
87
|
-
* that didn't pass, an `expect
|
|
91
|
+
* that didn't pass, an `expect` as what it failed with, an `expect.require` the eval caught with
|
|
88
92
|
* `CAUGHT_HINT` after its message.
|
|
89
93
|
*/
|
|
90
94
|
function checkFailure(raw) {
|
|
@@ -436,7 +440,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
|
|
|
436
440
|
//#region src/harness/index.ts
|
|
437
441
|
async function createHarness(descriptor, vertex) {
|
|
438
442
|
if (descriptor.kind === "fake") return createFakeHarness(descriptor);
|
|
439
|
-
const { createAgentHarness } = await import("./agent-
|
|
443
|
+
const { createAgentHarness } = await import("./agent-BgH4-kFc.js");
|
|
440
444
|
return createAgentHarness(descriptor, vertex);
|
|
441
445
|
}
|
|
442
446
|
/** Where each harness's agent looks for repo skills, relative to the workspace. */
|
|
@@ -466,8 +470,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
|
|
|
466
470
|
if (debug) traceEnvironment(debug, spec, env);
|
|
467
471
|
record.image = null;
|
|
468
472
|
if (spec.sandbox.kind === "docker") {
|
|
469
|
-
const { preparedFor } = await import("./preflight-
|
|
470
|
-
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-
|
|
473
|
+
const { preparedFor } = await import("./preflight-BYXkweNN.js");
|
|
474
|
+
const { attemptImage, dockerProvider, preparedImage } = await import("./providers-DDNMbv1T.js");
|
|
471
475
|
signal.throwIfAborted();
|
|
472
476
|
const provider = dockerProvider({ engine: spec.sandbox.engine ?? "docker" });
|
|
473
477
|
const builds = builtPlugins(spec.environment.layers);
|
|
@@ -536,16 +540,16 @@ function describeSize(state, question) {
|
|
|
536
540
|
return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
|
|
537
541
|
}).join("; ");
|
|
538
542
|
}
|
|
539
|
-
/** `Narrow
|
|
543
|
+
/** `Narrow diff, transcript or output.`: the inputs that were on, largest first. */
|
|
540
544
|
function stateHint(state, question) {
|
|
541
|
-
const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) =>
|
|
545
|
+
const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => p.name);
|
|
542
546
|
if (!keys.length) return "";
|
|
543
547
|
const last = keys.pop();
|
|
544
548
|
return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
|
|
545
549
|
}
|
|
546
550
|
/**
|
|
547
551
|
* A provider's error, as the provider worded it, then what the judge's input held: its estimated
|
|
548
|
-
* size and biggest parts and files, and the
|
|
552
|
+
* size and biggest parts and files, and the inputs that were on, largest first. reeval knows
|
|
549
553
|
* no model's limit, so this follows every provider error.
|
|
550
554
|
*/
|
|
551
555
|
const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
|
|
@@ -585,7 +589,7 @@ const workspacePath = (path) => path.replace(/^\.\//, "");
|
|
|
585
589
|
/** A `contents` entry that names one file rather than matching many. */
|
|
586
590
|
const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
|
|
587
591
|
/**
|
|
588
|
-
* The judge's state for
|
|
592
|
+
* The judge's state for the inputs the call turned on: the graded turn's task and output (or every turn's),
|
|
589
593
|
* its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
|
|
590
594
|
* transcript.
|
|
591
595
|
*/
|
|
@@ -633,7 +637,7 @@ function checksShown(which, ran) {
|
|
|
633
637
|
if (which === false) return void 0;
|
|
634
638
|
if (which !== true) {
|
|
635
639
|
const missing = which.find((command) => !ran.some((r) => r.command === command));
|
|
636
|
-
if (missing !== void 0) throw new Error(`c.judge()
|
|
640
|
+
if (missing !== void 0) throw new Error(`c.judge() checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
|
|
637
641
|
}
|
|
638
642
|
return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
|
|
639
643
|
command,
|
|
@@ -657,24 +661,24 @@ function placeFor(calls, order) {
|
|
|
657
661
|
* checks after it still run.
|
|
658
662
|
*/
|
|
659
663
|
function judgeCaller(a, readFile) {
|
|
660
|
-
const ask = async (order, question, { on, ...call }
|
|
661
|
-
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
664
|
+
const ask = async (order, question, { on, ...call }) => {
|
|
662
665
|
const problem = judgeProblem(call, { at: "call" });
|
|
663
666
|
if (problem) throw new Error(problem);
|
|
664
667
|
const badArts = badJudgeArtifacts(call.artifacts);
|
|
665
668
|
if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
|
|
666
669
|
const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
|
|
667
670
|
if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
|
|
668
|
-
const
|
|
671
|
+
const wanted = resolveStateOptions(call);
|
|
672
|
+
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives a value to grade as the output, but output is off. Pass output: true.");
|
|
673
|
+
const { task: _task, output: _output, files: _files, diff: _diff, contents: _contents, transcript: _transcript, turns: _turns, checks: _checks, artifacts, ...callJudge } = call;
|
|
674
|
+
const options = mergeJudge(a.spec.judge, callJudge);
|
|
669
675
|
const { model } = options;
|
|
670
676
|
if (model === void 0) {
|
|
671
677
|
const message = noJudgeModel(getRunContext().config.file);
|
|
672
678
|
a.judgeThrown.add(message);
|
|
673
679
|
throw new Error(message);
|
|
674
680
|
}
|
|
675
|
-
const
|
|
676
|
-
if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
|
|
677
|
-
const { state: _state, maxConcurrent, ...judgeOptions } = options;
|
|
681
|
+
const { maxConcurrent, ...judgeOptions } = options;
|
|
678
682
|
const output = on && "value" in on ? on.value : subject.text;
|
|
679
683
|
const { draft, dropped } = await draftState({
|
|
680
684
|
turns: a.turns,
|
|
@@ -687,7 +691,8 @@ function judgeCaller(a, readFile) {
|
|
|
687
691
|
});
|
|
688
692
|
const name = judgeModelName(model);
|
|
689
693
|
const state = redactValue(stateOf(draft));
|
|
690
|
-
const resolved = resolveJudgeArtifacts(
|
|
694
|
+
const resolved = resolveJudgeArtifacts(artifacts, a.artifactBlobs);
|
|
695
|
+
if (artifacts === true && !resolved.length && inputsOn(call, INPUT_KEYS).length === 1) throw new Error("c.judge() artifacts: true shows the judge nothing yet: this attempt has no captures. Call c.capture() first, or turn on another input.");
|
|
691
696
|
const refs = artifactRefsOf(resolved);
|
|
692
697
|
const decide = decideState(state, filePartsOf(resolved));
|
|
693
698
|
const input = {
|
|
@@ -789,7 +794,10 @@ function judgeCaller(a, readFile) {
|
|
|
789
794
|
return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
|
|
790
795
|
}
|
|
791
796
|
};
|
|
792
|
-
return async (question,
|
|
797
|
+
return async (question, given) => {
|
|
798
|
+
if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
|
|
799
|
+
if (given === null || typeof given !== "object" || Array.isArray(given)) throw new Error(judgeProblem(given, { at: "call" }) ?? NO_JUDGE_INPUT);
|
|
800
|
+
const { on, ...options } = given;
|
|
793
801
|
const order = a.judgeCalls++;
|
|
794
802
|
const call = on === void 0 ? options : {
|
|
795
803
|
...options,
|
|
@@ -805,7 +813,7 @@ async function createSandbox(descriptor, options) {
|
|
|
805
813
|
switch (descriptor.kind) {
|
|
806
814
|
case "fake": return await createFakeSandbox(descriptor, options.env);
|
|
807
815
|
case "docker": {
|
|
808
|
-
const { createDockerSandbox } = await import("./docker-
|
|
816
|
+
const { createDockerSandbox } = await import("./docker-DCAtFMo9.js");
|
|
809
817
|
return await createDockerSandbox(descriptor, options);
|
|
810
818
|
}
|
|
811
819
|
}
|
|
@@ -849,8 +857,8 @@ function isJudges(a, error, raw) {
|
|
|
849
857
|
}
|
|
850
858
|
/**
|
|
851
859
|
* The error that fails the attempt: a judge's gives way to the first other check that didn't hold
|
|
852
|
-
* (an `expect
|
|
853
|
-
* as that check.
|
|
860
|
+
* (an `expect`, or an `expect.require` the eval caught), whichever came first, so the attempt
|
|
861
|
+
* fails as that check.
|
|
854
862
|
*/
|
|
855
863
|
function attemptError(a, error, raw) {
|
|
856
864
|
if (!isJudges(a, error, raw)) return error;
|
|
@@ -968,12 +976,23 @@ async function writeRecord(a, stagingDir, id) {
|
|
|
968
976
|
question: clip(j.question, 200)
|
|
969
977
|
})),
|
|
970
978
|
turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
|
|
979
|
+
...failedChecks(record),
|
|
971
980
|
...record.error?.prepare ? { prepare: {
|
|
972
981
|
ref: record.error.prepare.ref,
|
|
973
982
|
stage: record.error.prepare.stage
|
|
974
983
|
} } : {}
|
|
975
984
|
};
|
|
976
985
|
}
|
|
986
|
+
/** The record's checks that didn't pass, for the terminal; unset when all did. */
|
|
987
|
+
function failedChecks(record) {
|
|
988
|
+
const failed = (record.assertions ?? []).filter((x) => x.status !== "pass");
|
|
989
|
+
if (!failed.length) return {};
|
|
990
|
+
return { failedChecks: failed.map((x) => ({
|
|
991
|
+
label: x.label,
|
|
992
|
+
...x.at ? { at: x.at } : {},
|
|
993
|
+
message: clip(x.message?.split("\n")[0] ?? "", 500)
|
|
994
|
+
})) };
|
|
995
|
+
}
|
|
977
996
|
/** `turn N error: <message>` for each turn with an error event. */
|
|
978
997
|
function turnErrors(transcripts) {
|
|
979
998
|
return transcripts.flatMap((events, i) => {
|
|
@@ -1190,7 +1209,7 @@ const timedOutServer = (entries) => entries.find((e) => e.timeout)?.server;
|
|
|
1190
1209
|
//#region src/attempt/state.ts
|
|
1191
1210
|
/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
|
|
1192
1211
|
const SETUP_BUDGET_MS = 12e5;
|
|
1193
|
-
function createAttempt(spec, signal, progress, start) {
|
|
1212
|
+
function createAttempt(spec, signal, progress, start, snapshotDir) {
|
|
1194
1213
|
const record = {
|
|
1195
1214
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1196
1215
|
outcome: "pass",
|
|
@@ -1245,7 +1264,8 @@ function createAttempt(spec, signal, progress, start) {
|
|
|
1245
1264
|
judgeInputCalls: [],
|
|
1246
1265
|
captureNames: /* @__PURE__ */ new Set(),
|
|
1247
1266
|
backgrounds: [],
|
|
1248
|
-
artifactBlobs: /* @__PURE__ */ new Map()
|
|
1267
|
+
artifactBlobs: /* @__PURE__ */ new Map(),
|
|
1268
|
+
snapshotDir
|
|
1249
1269
|
};
|
|
1250
1270
|
return a;
|
|
1251
1271
|
}
|
|
@@ -1638,6 +1658,237 @@ async function startSession(a, runId) {
|
|
|
1638
1658
|
return [box, session];
|
|
1639
1659
|
}
|
|
1640
1660
|
//#endregion
|
|
1661
|
+
//#region src/attempt/snapshot.ts
|
|
1662
|
+
/** The most a snapshot directory's files may add up to; a larger one isn't taken. */
|
|
1663
|
+
const SNAPSHOT_DIR_LIMIT = 50 * 2 ** 20;
|
|
1664
|
+
/** The most a compressed snapshot may be. */
|
|
1665
|
+
const SNAPSHOT_TAR_LIMIT = 10 * 2 ** 20;
|
|
1666
|
+
/** The most a compressed snapshot may be uncompressed, which the host holds in memory to mask. */
|
|
1667
|
+
const SNAPSHOT_RAW_LIMIT = 200 * 2 ** 20;
|
|
1668
|
+
/** How long one snapshot may take before the attempt goes on without it. */
|
|
1669
|
+
const SNAPSHOT_TIMEOUT_MS = 12e4;
|
|
1670
|
+
/** Installed dependencies, caches and git's objects: left out by directory name, at any depth. */
|
|
1671
|
+
const EXCLUDED_DIRS = [
|
|
1672
|
+
".git",
|
|
1673
|
+
"node_modules",
|
|
1674
|
+
".pnpm-store",
|
|
1675
|
+
".venv",
|
|
1676
|
+
"venv",
|
|
1677
|
+
".tox",
|
|
1678
|
+
"__pycache__",
|
|
1679
|
+
".pytest_cache",
|
|
1680
|
+
".mypy_cache",
|
|
1681
|
+
".ruff_cache",
|
|
1682
|
+
".next",
|
|
1683
|
+
".nuxt",
|
|
1684
|
+
".svelte-kit",
|
|
1685
|
+
".turbo",
|
|
1686
|
+
".parcel-cache",
|
|
1687
|
+
".cache",
|
|
1688
|
+
".dart_tool",
|
|
1689
|
+
".gradle"
|
|
1690
|
+
];
|
|
1691
|
+
const LISTED = 50;
|
|
1692
|
+
const SIZE_MARK = "--reeval-size--";
|
|
1693
|
+
const excluded = (path) => path.split("/").some((part) => EXCLUDED_DIRS.includes(part));
|
|
1694
|
+
const utf8 = new TextDecoder("utf-8", { fatal: true });
|
|
1695
|
+
function asText(data) {
|
|
1696
|
+
if (data.subarray(0, 8192).includes(0)) return null;
|
|
1697
|
+
try {
|
|
1698
|
+
return utf8.decode(data);
|
|
1699
|
+
} catch {
|
|
1700
|
+
return null;
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
/** The sandbox's archive, with the excluded directories gone and secret-looking values masked. */
|
|
1704
|
+
function cleanArchive(tar) {
|
|
1705
|
+
const entries = [];
|
|
1706
|
+
const masked = [];
|
|
1707
|
+
const dropped = [];
|
|
1708
|
+
const left = /* @__PURE__ */ new Set();
|
|
1709
|
+
for (const e of readTar(tar)) {
|
|
1710
|
+
if (excluded(e.path)) {
|
|
1711
|
+
const parts = e.path.split("/");
|
|
1712
|
+
const at = parts.findIndex((p) => EXCLUDED_DIRS.includes(p));
|
|
1713
|
+
left.add(redactSecrets(`${parts.slice(0, at + 1).join("/")}/`));
|
|
1714
|
+
continue;
|
|
1715
|
+
}
|
|
1716
|
+
if (findSecrets(e.path).length || e.type === "symlink" && findSecrets(e.target).length) {
|
|
1717
|
+
dropped.push(redactSecrets(e.path));
|
|
1718
|
+
continue;
|
|
1719
|
+
}
|
|
1720
|
+
if (e.type !== "file") {
|
|
1721
|
+
entries.push(e);
|
|
1722
|
+
continue;
|
|
1723
|
+
}
|
|
1724
|
+
const text = asText(e.data);
|
|
1725
|
+
if (text === null) {
|
|
1726
|
+
if (findSecrets(e.data.toString("latin1")).length) dropped.push(e.path);
|
|
1727
|
+
else entries.push(e);
|
|
1728
|
+
continue;
|
|
1729
|
+
}
|
|
1730
|
+
const clean = redactSecrets(text);
|
|
1731
|
+
if (clean === text) entries.push(e);
|
|
1732
|
+
else {
|
|
1733
|
+
masked.push(e.path);
|
|
1734
|
+
entries.push({
|
|
1735
|
+
...e,
|
|
1736
|
+
data: Buffer.from(clean)
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
return {
|
|
1741
|
+
entries,
|
|
1742
|
+
masked,
|
|
1743
|
+
dropped,
|
|
1744
|
+
excluded: [...left]
|
|
1745
|
+
};
|
|
1746
|
+
}
|
|
1747
|
+
const tarCommand = (workdir) => {
|
|
1748
|
+
const flags = EXCLUDED_DIRS.map((d) => `--exclude=${shellQuote(d)}`).join(" ");
|
|
1749
|
+
return `cd ${shellQuote(workdir)} && COPYFILE_DISABLE=1 tar -cf - ${flags} . 2>/dev/null`;
|
|
1750
|
+
};
|
|
1751
|
+
const GZIP = "if command -v gzip >/dev/null 2>&1; then gzip -c; else cat; fi";
|
|
1752
|
+
const NEEDS_TAR = "command -v tar >/dev/null 2>&1 || { echo 'the sandbox has no tar' >&2; exit 127; }";
|
|
1753
|
+
const listed = (paths) => paths.slice(0, LISTED);
|
|
1754
|
+
const LIMITS = {
|
|
1755
|
+
dir: SNAPSHOT_DIR_LIMIT,
|
|
1756
|
+
tar: SNAPSHOT_TAR_LIMIT,
|
|
1757
|
+
raw: SNAPSHOT_RAW_LIMIT
|
|
1758
|
+
};
|
|
1759
|
+
/**
|
|
1760
|
+
* Archives the sandbox's workspace and returns what to record and the masked entries to write,
|
|
1761
|
+
* or why none: over its form's cap, or an archive the sandbox couldn't make.
|
|
1762
|
+
*/
|
|
1763
|
+
async function archiveWorkspace(box, signal, compress, limits = LIMITS) {
|
|
1764
|
+
const none = (cause, message, dirs = []) => ({
|
|
1765
|
+
taken: {
|
|
1766
|
+
bytes: 0,
|
|
1767
|
+
files: 0,
|
|
1768
|
+
excluded: listed(dirs),
|
|
1769
|
+
masked: [],
|
|
1770
|
+
dropped: [],
|
|
1771
|
+
skipped: {
|
|
1772
|
+
cause,
|
|
1773
|
+
message
|
|
1774
|
+
}
|
|
1775
|
+
},
|
|
1776
|
+
entries: null
|
|
1777
|
+
});
|
|
1778
|
+
const names = EXCLUDED_DIRS.map((d) => `-name ${shellQuote(d)}`).join(" -o ");
|
|
1779
|
+
const tar = tarCommand(box.workdir);
|
|
1780
|
+
const sized = await box.execOwn(`${NEEDS_TAR}; cd ${shellQuote(box.workdir)} && { find . -type d \\( ${names} \\) -prune -print 2>/dev/null | head -n ${LISTED}; echo ${SIZE_MARK}; ${tar} | wc -c; echo ${SIZE_MARK}; ${tar} | { ${GZIP}; } | wc -c; }`, { signal });
|
|
1781
|
+
const [list = "", raw = "", size = ""] = sized.stdout.split(`${SIZE_MARK}\n`);
|
|
1782
|
+
const dirs = list.split("\n").filter(Boolean).map((p) => redactSecrets(`${p.replace(/^\.\//, "")}/`));
|
|
1783
|
+
const rawBytes = Number(raw.trim());
|
|
1784
|
+
const gzBytes = Number(size.trim());
|
|
1785
|
+
if (sized.exitCode !== 0 || !(rawBytes > 0) || !(gzBytes > 0)) return none("error", `couldn't archive the workspace: ${(sized.stderr || sized.stdout).trim()}`);
|
|
1786
|
+
const without = dirs.length ? dirs.join(", ") : "its dependency and cache directories";
|
|
1787
|
+
const rawLimit = compress ? limits.raw : limits.dir;
|
|
1788
|
+
if (rawBytes > rawLimit) return none("size", `the workspace is ${formatBytes(rawBytes)} without ${without}, over the ${formatBytes(rawLimit)} cap${compress ? "" : ` (compressed, the cap is ${formatBytes(limits.tar)})`}`, dirs);
|
|
1789
|
+
if (compress && gzBytes > limits.tar) return none("size", `the workspace is ${formatBytes(gzBytes)} compressed without ${without}, over the ${formatBytes(limits.tar)} cap`, dirs);
|
|
1790
|
+
const r = await box.execOwn(`${NEEDS_TAR}; echo ${OPAQUE_OUTPUT}; ${tar} | { ${GZIP}; } | base64`, { signal });
|
|
1791
|
+
const encoded = r.stdout.startsWith(`--opaque-output--\n`) ? r.stdout.slice(OPAQUE_OUTPUT.length + 1) : "";
|
|
1792
|
+
const fetched = Buffer.from(encoded, "base64");
|
|
1793
|
+
if (r.exitCode !== 0 || fetched.length === 0) return none("error", `couldn't archive the workspace: ${r.stderr.trim() || "no output"}`);
|
|
1794
|
+
const clean = cleanArchive(fetched[0] === 31 && fetched[1] === 139 ? gunzipSync(fetched, { maxOutputLength: rawLimit + 2 ** 20 }) : fetched);
|
|
1795
|
+
const all = [.../* @__PURE__ */ new Set([...dirs, ...clean.excluded])].sort();
|
|
1796
|
+
return {
|
|
1797
|
+
taken: {
|
|
1798
|
+
bytes: 0,
|
|
1799
|
+
files: clean.entries.filter((e) => e.type !== "dir").length,
|
|
1800
|
+
excluded: listed(all),
|
|
1801
|
+
masked: listed(clean.masked),
|
|
1802
|
+
dropped: listed(clean.dropped),
|
|
1803
|
+
skipped: null
|
|
1804
|
+
},
|
|
1805
|
+
entries: clean.entries
|
|
1806
|
+
};
|
|
1807
|
+
}
|
|
1808
|
+
/** Where an attempt's snapshots wait, beside its staging record, until the run is assembled. */
|
|
1809
|
+
const snapshotSidecar = (stagingDir, key) => join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".snapshots"));
|
|
1810
|
+
/**
|
|
1811
|
+
* Takes snapshot `name` of the workspace into the attempt's sidecar and adds it to the record;
|
|
1812
|
+
* never fails the attempt. Adds nothing when the attempt's signal stopped it.
|
|
1813
|
+
*/
|
|
1814
|
+
async function takeSnapshot(a, box, name, phase, limits = LIMITS, timeoutMs = SNAPSHOT_TIMEOUT_MS) {
|
|
1815
|
+
const { compress } = a.spec.snapshot;
|
|
1816
|
+
const form = compress ? "tar.gz" : "dir";
|
|
1817
|
+
const dir = a.snapshotDir;
|
|
1818
|
+
const target = join(dir, compress ? `${name}.tar.gz` : name);
|
|
1819
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
1820
|
+
const add = (taken) => {
|
|
1821
|
+
a.record.snapshots ??= [];
|
|
1822
|
+
a.record.snapshots.push({
|
|
1823
|
+
name,
|
|
1824
|
+
form,
|
|
1825
|
+
path: null,
|
|
1826
|
+
...taken
|
|
1827
|
+
});
|
|
1828
|
+
};
|
|
1829
|
+
const failed = (message) => {
|
|
1830
|
+
debugLog(phase, `no ${name} snapshot: ${message}`);
|
|
1831
|
+
add({
|
|
1832
|
+
bytes: 0,
|
|
1833
|
+
files: 0,
|
|
1834
|
+
excluded: [],
|
|
1835
|
+
masked: [],
|
|
1836
|
+
dropped: [],
|
|
1837
|
+
skipped: {
|
|
1838
|
+
cause: "error",
|
|
1839
|
+
message: `couldn't archive the workspace: ${message}`
|
|
1840
|
+
}
|
|
1841
|
+
});
|
|
1842
|
+
};
|
|
1843
|
+
try {
|
|
1844
|
+
const { taken, entries } = await archiveWorkspace(box, AbortSignal.any([a.signal, timeout]), compress, limits);
|
|
1845
|
+
if (!entries) {
|
|
1846
|
+
debugLog(phase, `no ${name} snapshot: ${taken.skipped?.message}`);
|
|
1847
|
+
add(taken);
|
|
1848
|
+
return;
|
|
1849
|
+
}
|
|
1850
|
+
await rm(target, {
|
|
1851
|
+
recursive: true,
|
|
1852
|
+
force: true
|
|
1853
|
+
});
|
|
1854
|
+
await mkdir(dir, { recursive: true });
|
|
1855
|
+
if (compress) {
|
|
1856
|
+
const archive = gzipSync(writeTar(entries));
|
|
1857
|
+
if (archive.length > limits.tar) {
|
|
1858
|
+
const message = `the workspace is ${formatBytes(archive.length)} compressed, over the ${formatBytes(limits.tar)} cap`;
|
|
1859
|
+
debugLog(phase, `no ${name} snapshot: ${message}`);
|
|
1860
|
+
add({
|
|
1861
|
+
...taken,
|
|
1862
|
+
files: 0,
|
|
1863
|
+
skipped: {
|
|
1864
|
+
cause: "size",
|
|
1865
|
+
message
|
|
1866
|
+
}
|
|
1867
|
+
});
|
|
1868
|
+
return;
|
|
1869
|
+
}
|
|
1870
|
+
await writeFile(target, archive);
|
|
1871
|
+
taken.bytes = archive.length;
|
|
1872
|
+
} else {
|
|
1873
|
+
const skipped = await extractTar(entries, target);
|
|
1874
|
+
taken.bytes = entries.reduce((n, e) => n + (e.type === "file" ? e.data.length : 0), 0);
|
|
1875
|
+
taken.dropped = listed([...taken.dropped, ...skipped]);
|
|
1876
|
+
}
|
|
1877
|
+
const masked = taken.masked.length ? ` · masked ${taken.masked.length}` : "";
|
|
1878
|
+
const left = taken.excluded.length ? ` · left out ${taken.excluded.join(", ")}` : "";
|
|
1879
|
+
debugLog(phase, `${name} snapshot: ${taken.files} files, ${formatBytes(taken.bytes)} (${form})${masked}${left}`);
|
|
1880
|
+
add(taken);
|
|
1881
|
+
} catch (error) {
|
|
1882
|
+
await rm(target, {
|
|
1883
|
+
recursive: true,
|
|
1884
|
+
force: true
|
|
1885
|
+
}).catch(() => {});
|
|
1886
|
+
if (a.signal.aborted) return;
|
|
1887
|
+
if (timeout.aborted) return failed(`it took over ${Math.round(timeoutMs / 1e3)}s`);
|
|
1888
|
+
failed(error instanceof Error ? error.message : String(error));
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
//#endregion
|
|
1641
1892
|
//#region src/attempt/turns.ts
|
|
1642
1893
|
/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
|
|
1643
1894
|
const TURN_STOP_GRACE_MS = 3e3;
|
|
@@ -1820,6 +2071,7 @@ function turnSender(a, takeContext, checks) {
|
|
|
1820
2071
|
a.signal.throwIfAborted();
|
|
1821
2072
|
const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
|
|
1822
2073
|
a.record.diff = diff;
|
|
2074
|
+
if (a.spec.snapshot.when === "turns") await takeSnapshot(a, box, `turn-${turnNumber}`, "harness");
|
|
1823
2075
|
const data = {
|
|
1824
2076
|
prompt,
|
|
1825
2077
|
...result,
|
|
@@ -1863,7 +2115,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1863
2115
|
} finally {
|
|
1864
2116
|
clock?.resume();
|
|
1865
2117
|
}
|
|
1866
|
-
});
|
|
2118
|
+
}, snapshotSidecar(stagingDir, attemptKey(id, spec.cell.key, spec.n)));
|
|
1867
2119
|
const destroy = () => void a.sandbox?.destroy();
|
|
1868
2120
|
testSignal.addEventListener("abort", destroy, { once: true });
|
|
1869
2121
|
const takeContext = contextTaker(a);
|
|
@@ -1932,6 +2184,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1932
2184
|
if (a.session) debug.log("teardown", "harness session closed");
|
|
1933
2185
|
recordEgress(a, sandbox);
|
|
1934
2186
|
oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
|
|
2187
|
+
if (sandbox && spec.snapshot.when && !a.signal.aborted) await takeSnapshot(a, sandbox, "final", "teardown");
|
|
1935
2188
|
await sandbox?.destroy();
|
|
1936
2189
|
if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
|
|
1937
2190
|
const { record } = a;
|
|
@@ -1953,6 +2206,30 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
|
|
|
1953
2206
|
if (failure) throw failure.error;
|
|
1954
2207
|
}
|
|
1955
2208
|
//#endregion
|
|
2209
|
+
//#region src/best-of.ts
|
|
2210
|
+
/** The side an attempt that settled counts on: its record's outcome, else the engine's state. */
|
|
2211
|
+
function sideOf({ state, meta }) {
|
|
2212
|
+
const outcome = meta?.outcome ?? (state === "passed" ? "pass" : state === "failed" ? "fail" : null);
|
|
2213
|
+
return outcome === "pass" || outcome === "fail" ? outcome : null;
|
|
2214
|
+
}
|
|
2215
|
+
/** Whether a best-of cell still needs another attempt after these. */
|
|
2216
|
+
function undecided(sides, bestOf) {
|
|
2217
|
+
const majority = majorityOf(bestOf);
|
|
2218
|
+
const pass = sides.filter((s) => s === "pass").length;
|
|
2219
|
+
const fail = sides.filter((s) => s === "fail").length;
|
|
2220
|
+
return pass < majority && fail < majority;
|
|
2221
|
+
}
|
|
2222
|
+
/** One best-of cell: the `ready` and `settled` hooks of its attempt `n`, from 1 to `bestOf`. */
|
|
2223
|
+
function bestOfCell(bestOf) {
|
|
2224
|
+
const sides = [];
|
|
2225
|
+
const settle = [];
|
|
2226
|
+
for (let n = 1; n <= bestOf; n++) sides.push(new Promise((resolve) => settle.push(resolve)));
|
|
2227
|
+
return (n) => ({
|
|
2228
|
+
...n > majorityOf(bestOf) ? { ready: async () => undecided(await Promise.all(sides.slice(0, n - 1)), bestOf) } : {},
|
|
2229
|
+
settled: (settled) => settle[n - 1]?.(sideOf(settled))
|
|
2230
|
+
});
|
|
2231
|
+
}
|
|
2232
|
+
//#endregion
|
|
1956
2233
|
//#region src/contract/declaration.ts
|
|
1957
2234
|
/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
|
|
1958
2235
|
const evalOptionsDataSchema = z.record(z.string(), z.unknown());
|
|
@@ -2014,12 +2291,14 @@ const declarationProblem = (children) => dataProblem(declarationTreeSchema, { ch
|
|
|
2014
2291
|
/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
|
|
2015
2292
|
const GRACE_MS = 5e3;
|
|
2016
2293
|
function configLevel(config) {
|
|
2294
|
+
const { runs, bestOf } = config.runs !== void 0 ? runsOf(config.runs) : { runs: void 0 };
|
|
2017
2295
|
return {
|
|
2018
2296
|
harness: config.harness,
|
|
2019
2297
|
sandbox: config.sandbox,
|
|
2020
2298
|
environment: config.environment,
|
|
2021
2299
|
compare: void 0,
|
|
2022
|
-
runs
|
|
2300
|
+
runs,
|
|
2301
|
+
bestOf,
|
|
2023
2302
|
gate: config.gate,
|
|
2024
2303
|
env: config.env,
|
|
2025
2304
|
egress: config.egress !== void 0 ? { inherited: config.egress } : {},
|
|
@@ -2028,6 +2307,7 @@ function configLevel(config) {
|
|
|
2028
2307
|
question: config.question,
|
|
2029
2308
|
questions: [],
|
|
2030
2309
|
timeout: void 0,
|
|
2310
|
+
snapshot: config.snapshot,
|
|
2031
2311
|
tags: [],
|
|
2032
2312
|
revision: void 0,
|
|
2033
2313
|
pin: [],
|
|
@@ -2210,6 +2490,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2210
2490
|
setBy,
|
|
2211
2491
|
gate: planned.gate,
|
|
2212
2492
|
...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
|
|
2493
|
+
...planned.bestOf !== void 0 ? { bestOf: planned.bestOf } : {},
|
|
2213
2494
|
brokered: brokeredIn(resolved.env, planned),
|
|
2214
2495
|
secretVars: secretVarsIn(resolved.env, planned),
|
|
2215
2496
|
egress: planned.egress,
|
|
@@ -2249,6 +2530,7 @@ function planTests(declaration, ctx, where, resolvedEval) {
|
|
|
2249
2530
|
env: resolved.env,
|
|
2250
2531
|
judge: resolved.judge,
|
|
2251
2532
|
capture: resolved.capture,
|
|
2533
|
+
snapshot: resolved.snapshot,
|
|
2252
2534
|
sources: {
|
|
2253
2535
|
...resolved.sources,
|
|
2254
2536
|
sandbox: setBy.sandbox
|
|
@@ -2487,7 +2769,20 @@ function register(mode, name, own, fn, source, hint = "", family) {
|
|
|
2487
2769
|
declareHarnesses(harnesses);
|
|
2488
2770
|
if (!tests) return;
|
|
2489
2771
|
registered++;
|
|
2490
|
-
|
|
2772
|
+
const bestOfCells = /* @__PURE__ */ new Map();
|
|
2773
|
+
for (const { name: testName, spec, ...options } of tests) {
|
|
2774
|
+
const { cell, n, bestOf } = options.attempt;
|
|
2775
|
+
let hooks = {};
|
|
2776
|
+
if (bestOf !== void 0) {
|
|
2777
|
+
const of = bestOfCells.get(cell.key) ?? bestOfCell(bestOf);
|
|
2778
|
+
bestOfCells.set(cell.key, of);
|
|
2779
|
+
hooks = of(n);
|
|
2780
|
+
}
|
|
2781
|
+
registerTest(testName, {
|
|
2782
|
+
...options,
|
|
2783
|
+
...hooks
|
|
2784
|
+
}, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
|
|
2785
|
+
}
|
|
2491
2786
|
}
|
|
2492
2787
|
/** Evals registered so far in this worker, for the scheduler's tie-break. */
|
|
2493
2788
|
let registered = 0;
|
|
@@ -2601,4 +2896,4 @@ function optionsFrom(written, where, base) {
|
|
|
2601
2896
|
//#endregion
|
|
2602
2897
|
export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
|
|
2603
2898
|
|
|
2604
|
-
//# sourceMappingURL=evaluate-
|
|
2899
|
+
//# sourceMappingURL=evaluate-CUtgyr43.js.map
|