reeval 0.15.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
- package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
- package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
- package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
- package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
- package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
- package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
- package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
- package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
- package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +63 -16
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
- package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
- package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
- package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
- package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
- package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
- package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
- package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
- package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
- package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +52 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
- package/dist/view/index.d.ts +16 -4
- package/dist/view/index.js +6 -6
- package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -1,29 +1,29 @@
|
|
|
1
|
-
import { n as isOneOf, t as isObjectLike } from "./object-
|
|
1
|
+
import { n as isOneOf, t as isObjectLike } from "./object-BkXGPKWY.js";
|
|
2
2
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
3
3
|
import { a as fieldPath, c as jsonValueSchema, d as refusal, f as valueAt, h as parseDuration, i as durationSchema, l as nonEmpty, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-B_lvGqWz.js";
|
|
4
|
-
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-
|
|
5
|
-
import { B as gatePolicySchema, D as pluginServerName, L as egressPolicySchema, V as gateProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-
|
|
4
|
+
import { A as sandboxPaths, E as plannedSecrets, F as workspaceOf, I as environmentDataSchema, L as sandboxDescriptorSchema, N as unbrokeredSecrets, O as sameWorkspaceOnly, P as workspaceLabel, T as plainCredentialHeaders, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor, i as applied, k as sandboxOf, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, y as isMovingRef, z as sandboxProblem } from "./environment-Cf5HMjib.js";
|
|
5
|
+
import { B as gatePolicySchema, D as pluginServerName, J as snapshotOptionSchema, L as egressPolicySchema, V as gateProblem, Y as snapshotProblem, a as PLUGIN_PARTS, n as PLUGIN_DIR, r as PLUGIN_MANIFEST, t as PART_PATHS, y as localPluginServers, z as envVarsSchema } from "./plugins-ngQEJBjJ.js";
|
|
6
6
|
import { _ as gatewayDescriptorSchema, f as FACTORY, g as directOffPin, h as GATEWAY_NAMES, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError, v as gatewayEntryDataSchema, y as gatewayNameSchema } from "./harness-3NVyTSGK.js";
|
|
7
7
|
import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
8
8
|
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-Bz0l7OZI.js";
|
|
9
|
-
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-
|
|
9
|
+
import { S as servedOnVercel, _ as FX_VERCEL_ENDPOINTS, a as RECOMMENDED_JUDGE_MODEL, b as VERCEL_KEY, c as judgeCredentials, d as modelKeyOptions, g as FX_VERCEL_CALL, n as DIRECT_PROVIDERS, o as checkModelString, r as OPENROUTER_JUDGE_MODEL, v as VERCEL_GATEWAY, w as OPENROUTER_GATEWAY, x as fxAskedModel, y as VERCEL_HOST } from "./judge-Bg4v4e_K.js";
|
|
10
10
|
import { a as VERTEX_ROUTES, g as vertexModelRefusalFor, h as vertexModelRefusal, i as VERTEX_REFUSALS, r as VERTEX_PROJECT_ENV, t as SANDBOX_ACCESS_TOKEN } from "./vertex-DypYjx71.js";
|
|
11
11
|
import { r as badCharacterAt } from "./redact-DVIaahLv.js";
|
|
12
12
|
import { s as withoutBom } from "./text-DTigcA64.js";
|
|
13
13
|
import { t as inCI } from "./env-CARj5Axq.js";
|
|
14
14
|
import { u as ownFileURL } from "./root-Dlff-zp8.js";
|
|
15
|
-
import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS,
|
|
15
|
+
import { F as PACKAGE_HOSTS, N as DOCKER_EGRESS, P as HARNESS_HOSTS, ft as run, pt as shellQuote } from "./src-C0ySiej5.js";
|
|
16
16
|
import { t as InfraError } from "./types-C5vI3g1C.js";
|
|
17
|
-
import {
|
|
17
|
+
import { n as gateLabel, u as pairFactors } from "./gate-BC6SdQZs.js";
|
|
18
18
|
import "./stats-hfpuYsAM.js";
|
|
19
|
-
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-
|
|
20
|
-
import { t as docker } from "./sandbox-
|
|
19
|
+
import { n as defaultIncludeGlobs, o as reevalJiti } from "./adapter-CRKYPkRt.js";
|
|
20
|
+
import { t as docker } from "./sandbox-Cmi6IQML.js";
|
|
21
21
|
import { existsSync, readFileSync } from "node:fs";
|
|
22
22
|
import { availableParallelism, tmpdir, totalmem } from "node:os";
|
|
23
23
|
import { dirname, join, resolve, win32 } from "node:path";
|
|
24
24
|
import { parseDocument } from "yaml";
|
|
25
25
|
import { z } from "zod";
|
|
26
|
-
import { isIP } from "node:net";
|
|
26
|
+
import { connect, isIP } from "node:net";
|
|
27
27
|
import { createHash, randomUUID } from "node:crypto";
|
|
28
28
|
import { spawn } from "node:child_process";
|
|
29
29
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
@@ -84,21 +84,43 @@ function decideState(json, files) {
|
|
|
84
84
|
}
|
|
85
85
|
//#endregion
|
|
86
86
|
//#region src/contract/judge.ts
|
|
87
|
-
|
|
87
|
+
/** The inputs every judge call and question shares; `c.judge()` adds `contents` and its own `artifacts`. */
|
|
88
|
+
const sharedInputFields = {
|
|
88
89
|
task: z.boolean().optional(),
|
|
89
90
|
output: z.boolean().optional(),
|
|
90
91
|
files: z.boolean().optional(),
|
|
91
92
|
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
92
|
-
contents: z.array(nonEmpty()).optional(),
|
|
93
93
|
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
94
|
-
turns: z.enum(["last", "all"]).optional(),
|
|
95
94
|
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
95
|
+
};
|
|
96
|
+
const turnsSchema = z.enum(["last", "all"]);
|
|
97
|
+
const { task, output, files, diff, transcript, checks } = sharedInputFields;
|
|
98
|
+
const judgeInputsSchema = z.strictObject({
|
|
99
|
+
task,
|
|
100
|
+
output,
|
|
101
|
+
files,
|
|
102
|
+
diff,
|
|
103
|
+
contents: z.array(nonEmpty()).optional(),
|
|
104
|
+
transcript,
|
|
105
|
+
checks,
|
|
106
|
+
artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
|
|
96
107
|
});
|
|
108
|
+
const judgeSubjectSchema = z.strictObject({
|
|
109
|
+
on: z.custom().optional(),
|
|
110
|
+
turns: turnsSchema.optional()
|
|
111
|
+
});
|
|
112
|
+
const INPUT_KEYS = Object.keys(judgeInputsSchema.shape);
|
|
113
|
+
/** True when an input's value turns it on: not `false`, unset or an empty list. */
|
|
114
|
+
const inputOn = (value) => value !== void 0 && value !== false && !(Array.isArray(value) && value.length === 0);
|
|
115
|
+
/** The input keys an options object turns on, in `keys`' order. */
|
|
116
|
+
const inputsOn = (value, keys) => keys.filter((k) => inputOn(value[k]));
|
|
117
|
+
/** `a`, `b` and `c` as prose: "a, b or c". */
|
|
118
|
+
const orList = (items) => items.length < 2 ? items[0] ?? "" : `${items.slice(0, -1).join(", ")} or ${items.at(-1)}`;
|
|
119
|
+
const NO_JUDGE_INPUT = `c.judge() shows the judge nothing: turn on at least one input, such as { diff: true } or { contents: ['src/**'] }. The inputs are ${orList(INPUT_KEYS)}.`;
|
|
97
120
|
const judgeFields = {
|
|
98
121
|
threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional(),
|
|
99
122
|
providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional(),
|
|
100
123
|
maxConcurrent: z.number().int().min(1).optional(),
|
|
101
|
-
state: judgeStateSchema.optional(),
|
|
102
124
|
gateway: z.union([gatewayNameSchema, gatewayDescriptorSchema]).optional()
|
|
103
125
|
};
|
|
104
126
|
const fakeJudgeDataSchema = z.strictObject({
|
|
@@ -114,18 +136,13 @@ const judgeOptionsSchema = z.strictObject({
|
|
|
114
136
|
model: z.custom().optional(),
|
|
115
137
|
...judgeFields
|
|
116
138
|
});
|
|
117
|
-
const judgeCallSchema = judgeOptionsSchema.extend(
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
shows: "state",
|
|
125
|
-
show: "state",
|
|
126
|
-
input: "state"
|
|
127
|
-
};
|
|
128
|
-
const VALUES = {
|
|
139
|
+
const judgeCallSchema = judgeOptionsSchema.extend(judgeInputsSchema.shape).extend(judgeSubjectSchema.shape);
|
|
140
|
+
const CALL_ONLY_KEYS = [
|
|
141
|
+
...INPUT_KEYS,
|
|
142
|
+
"on",
|
|
143
|
+
"turns"
|
|
144
|
+
];
|
|
145
|
+
const VALUES$1 = {
|
|
129
146
|
task: "true or false",
|
|
130
147
|
output: "true or false",
|
|
131
148
|
files: "true or false",
|
|
@@ -162,7 +179,10 @@ function judgeProblem(judge, site) {
|
|
|
162
179
|
return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
|
|
163
180
|
};
|
|
164
181
|
const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
|
|
165
|
-
if (typeof judge !== "object" || judge === null || Array.isArray(judge))
|
|
182
|
+
if (typeof judge !== "object" || judge === null || Array.isArray(judge)) {
|
|
183
|
+
const example = site.at === "call" ? "{ diff: true }" : "{ threshold: 0.8 }";
|
|
184
|
+
return `${field()} must be an object such as ${example} (got ${shown(judge)})`;
|
|
185
|
+
}
|
|
166
186
|
const value = judge;
|
|
167
187
|
const model = value.model;
|
|
168
188
|
if (site.at === "config" && typeof model === "string") {
|
|
@@ -171,7 +191,7 @@ function judgeProblem(judge, site) {
|
|
|
171
191
|
}
|
|
172
192
|
const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
|
|
173
193
|
const parsed = schema.safeParse(judge);
|
|
174
|
-
if (parsed.success) return void 0;
|
|
194
|
+
if (parsed.success) return site.at === "call" && !inputsOn(value, INPUT_KEYS).length ? NO_JUDGE_INPUT : void 0;
|
|
175
195
|
const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
|
|
176
196
|
const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
|
|
177
197
|
if (!issues.length) return void 0;
|
|
@@ -194,29 +214,22 @@ function judgeProblem(judge, site) {
|
|
|
194
214
|
const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
|
|
195
215
|
if (name === void 0) return void 0;
|
|
196
216
|
const allowed = Object.keys(schema.shape);
|
|
197
|
-
const hint =
|
|
198
|
-
const why = site.at !== "call" && (name
|
|
217
|
+
const hint = nearest(name, allowed);
|
|
218
|
+
const why = site.at !== "call" && CALL_ONLY_KEYS.includes(name) ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
|
|
199
219
|
return `Unknown judge option "${name}"${loc}.${why}`;
|
|
200
220
|
};
|
|
201
|
-
const
|
|
202
|
-
const issue = issues.find((i) => i.path[0] === "
|
|
221
|
+
const inputProblem = () => {
|
|
222
|
+
const issue = issues.find((i) => typeof i.path[0] === "string" && i.path[0] in VALUES$1);
|
|
203
223
|
if (!issue) return void 0;
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
|
|
207
|
-
const key = issue.keys[0] ?? "";
|
|
208
|
-
const hint = nearest(key, STATE_KEYS$1);
|
|
209
|
-
return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
210
|
-
}
|
|
211
|
-
const key = issue.path[1];
|
|
212
|
-
return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
|
|
224
|
+
const key = issue.path[0];
|
|
225
|
+
return `${field(key)} must be ${VALUES$1[key]} (got ${shown(value[key])})`;
|
|
213
226
|
};
|
|
214
227
|
const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
|
|
215
228
|
const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
|
|
216
229
|
const gatewayProblem = () => at("gateway") ? `${field("gateway")} names a gateway in the config's gateway list: ${GATEWAY_NAMES.map((g) => `'${g}'`).join(" or ")}, or is its helper (got ${shown(value.gateway)})` : void 0;
|
|
217
|
-
if (site.at === "config") order.push(modelProblem, keyProblem,
|
|
218
|
-
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem,
|
|
219
|
-
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem,
|
|
230
|
+
if (site.at === "config") order.push(modelProblem, keyProblem, providerOptionsProblem, thresholdProblem, gatewayProblem);
|
|
231
|
+
else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, inputProblem, providerOptionsProblem, artifactsProblem, gatewayProblem);
|
|
232
|
+
else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, providerOptionsProblem, gatewayProblem);
|
|
220
233
|
for (const check of order) {
|
|
221
234
|
const problem = check();
|
|
222
235
|
if (problem) return problem;
|
|
@@ -228,22 +241,15 @@ function judgeProblem(judge, site) {
|
|
|
228
241
|
const pairFactorSchema = z.enum(["environment", "harness"]);
|
|
229
242
|
/** Evaluation models score yes/no questions only, so they can't answer a question. */
|
|
230
243
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
231
|
-
const questionStateSchema = z.strictObject({
|
|
232
|
-
task: z.boolean().optional(),
|
|
233
|
-
output: z.boolean().optional(),
|
|
234
|
-
files: z.boolean().optional(),
|
|
235
|
-
diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional(),
|
|
236
|
-
transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional(),
|
|
237
|
-
turns: z.enum(["last", "all"]).optional(),
|
|
238
|
-
checks: z.union([z.boolean(), z.array(nonEmpty())]).optional()
|
|
239
|
-
});
|
|
240
244
|
const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
model: questionModelSchema.optional(),
|
|
244
|
-
state: questionStateSchema.optional(),
|
|
245
|
+
const questionInputFields = {
|
|
246
|
+
...sharedInputFields,
|
|
245
247
|
artifacts: questionArtifactsSchema.optional()
|
|
246
|
-
}
|
|
248
|
+
};
|
|
249
|
+
z.strictObject(questionInputFields);
|
|
250
|
+
const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
|
|
251
|
+
const questionOptionsSchema = z.strictObject({ model: questionModelSchema.optional() });
|
|
252
|
+
const QUESTION_INPUT_KEYS = Object.keys(questionInputFields);
|
|
247
253
|
/** The first item a list holds twice. */
|
|
248
254
|
const twice = (items) => items.find((k, i) => items.indexOf(k) !== i);
|
|
249
255
|
const unique = (list, ctx) => {
|
|
@@ -254,35 +260,38 @@ const unique = (list, ctx) => {
|
|
|
254
260
|
};
|
|
255
261
|
const acrossSchema = z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique)]);
|
|
256
262
|
const choicesSchema = z.array(nonEmpty()).min(2).superRefine(unique);
|
|
257
|
-
const
|
|
263
|
+
const questionSchema = z.strictObject({
|
|
258
264
|
question: nonEmpty(),
|
|
259
265
|
model: questionModelSchema.optional(),
|
|
260
266
|
across: acrossSchema.optional(),
|
|
261
267
|
choices: choicesSchema.optional(),
|
|
262
|
-
|
|
263
|
-
|
|
268
|
+
turns: turnsSchema.optional(),
|
|
269
|
+
...questionInputFields
|
|
264
270
|
}).superRefine((q, ctx) => {
|
|
265
271
|
if (q.across !== void 0 && q.choices !== void 0) ctx.addIssue({
|
|
266
272
|
code: "custom",
|
|
267
273
|
path: ["choices"],
|
|
268
274
|
message: "with across"
|
|
269
275
|
});
|
|
276
|
+
if (!inputsOn(q, QUESTION_INPUT_KEYS).length) ctx.addIssue({
|
|
277
|
+
code: "custom",
|
|
278
|
+
message: "no input"
|
|
279
|
+
});
|
|
270
280
|
});
|
|
271
|
-
const
|
|
272
|
-
const questionText = (q) => typeof q === "string" ? q : q.question;
|
|
281
|
+
const questionText = (q) => q.question;
|
|
273
282
|
const questionsSchema = z.array(questionSchema).superRefine((list, ctx) => {
|
|
274
283
|
if (twice(list.map(questionText)) !== void 0) ctx.addIssue({
|
|
275
284
|
code: "custom",
|
|
276
285
|
message: "twice"
|
|
277
286
|
});
|
|
278
287
|
});
|
|
279
|
-
const QUESTION_EXAMPLE = `questions: ['How did the agent find the failing test?']`;
|
|
288
|
+
const QUESTION_EXAMPLE = `questions: [{ question: 'How did the agent find the failing test?', transcript: true }]`;
|
|
280
289
|
const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
|
|
281
290
|
const ACROSS_MEAN = " Did you mean \"across\"? It is the factor a pairwise question varies: across: 'environment'.";
|
|
282
|
-
const QUESTION_KEYS = Object.keys(
|
|
283
|
-
const
|
|
284
|
-
const
|
|
285
|
-
const
|
|
291
|
+
const QUESTION_KEYS = Object.keys(questionSchema.shape);
|
|
292
|
+
const INPUTS = orList(QUESTION_INPUT_KEYS);
|
|
293
|
+
const QUESTION_SHAPE = `{ question, model?, across?, choices?, turns? } with at least one of ${INPUTS}`;
|
|
294
|
+
const VALUES = {
|
|
286
295
|
task: "true or false",
|
|
287
296
|
output: "true or false",
|
|
288
297
|
files: "true or false",
|
|
@@ -299,41 +308,31 @@ const describe$2 = (value) => {
|
|
|
299
308
|
}
|
|
300
309
|
return JSON.stringify(value) ?? String(value);
|
|
301
310
|
};
|
|
302
|
-
/**
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const { state, artifacts } = value;
|
|
309
|
-
if (state !== void 0) {
|
|
310
|
-
if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$2(state)})`;
|
|
311
|
-
for (const [key, v] of Object.entries(state)) {
|
|
312
|
-
if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
|
|
313
|
-
const schema = questionStateSchema.shape[key];
|
|
314
|
-
if (!schema) {
|
|
315
|
-
const hint = closest(key, STATE_KEYS);
|
|
316
|
-
return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
|
|
317
|
-
}
|
|
318
|
-
if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$2(v)})`;
|
|
311
|
+
/** What's wrong with an entry's inputs and `turns`, as one sentence, or undefined. */
|
|
312
|
+
function questionInputsProblem(value, at, where) {
|
|
313
|
+
for (const [key, v] of Object.entries(value)) {
|
|
314
|
+
if (key === "artifacts") {
|
|
315
|
+
if (!questionArtifactsSchema.safeParse(v).success) return `${at} artifacts in ${where} must be true or a non-empty list of capture names (got ${describe$2(v)})`;
|
|
316
|
+
continue;
|
|
319
317
|
}
|
|
318
|
+
if (!(key in VALUES)) continue;
|
|
319
|
+
if (!(key === "turns" ? turnsSchema : sharedInputFields[key]).safeParse(v).success) return `${at} ${key} in ${where} must be ${VALUES[key]} (got ${describe$2(v)})`;
|
|
320
320
|
}
|
|
321
|
-
if (
|
|
321
|
+
if (!inputsOn(value, QUESTION_INPUT_KEYS).length) return `${at} in ${where} shows the model nothing: turn on at least one of ${INPUTS}, e.g. { question: '…', diff: true }.`;
|
|
322
322
|
}
|
|
323
323
|
/** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
|
|
324
324
|
function questionOptionsProblem(value, where) {
|
|
325
|
-
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model
|
|
325
|
+
if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model? } (got ${describe$2(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
|
|
326
326
|
const keys = Object.keys(questionOptionsSchema.shape);
|
|
327
327
|
for (const key of Object.keys(value)) {
|
|
328
328
|
if (keys.includes(key)) continue;
|
|
329
|
-
if (key === "across" || key === "choices") return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }:
|
|
329
|
+
if (key === "across" || key === "choices" || key === "turns" || key in questionInputFields) return `Unknown question option "${key}" in ${where}. ${key} goes on each questions entry, { question, ${key} }: what a question is and what it's shown are its own.`;
|
|
330
330
|
const hint = closest(key, keys);
|
|
331
|
-
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model
|
|
331
|
+
return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model.`;
|
|
332
332
|
}
|
|
333
333
|
const { model } = value;
|
|
334
334
|
if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
335
335
|
if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
336
|
-
return questionPartsProblem(value, (key) => `question.${key}`, where);
|
|
337
336
|
}
|
|
338
337
|
/** What's wrong with an entry's `across`, or undefined. */
|
|
339
338
|
function acrossProblem(question, across, where) {
|
|
@@ -352,11 +351,11 @@ function choicesProblem(question, choices, where) {
|
|
|
352
351
|
if (blank !== void 0 && blank !== false) return `${at} has an answer that isn't a non-empty string (got ${describe$2(blank)}), e.g. choices: ['the test', 'the code']`;
|
|
353
352
|
return `${at} is a list of at least two answers, e.g. choices: ['the test', 'the code'] (got ${describe$2(choices)})`;
|
|
354
353
|
}
|
|
355
|
-
/** One question's sentence
|
|
354
|
+
/** One question's sentence. */
|
|
356
355
|
function questionProblem(item, where) {
|
|
357
|
-
if (typeof item
|
|
358
|
-
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
356
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) return `questions in ${where} takes ${QUESTION_SHAPE} (got ${describe$2(item)}), e.g. ${QUESTION_EXAMPLE}.`;
|
|
359
357
|
const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
|
|
358
|
+
if (unknown.includes("contents")) return `questions in ${where} can't set contents: questions are asked after the attempts, when their workspaces are gone. Narrow diff instead.`;
|
|
360
359
|
if (unknown.length) {
|
|
361
360
|
const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
|
|
362
361
|
return `questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
|
|
@@ -373,7 +372,7 @@ function questionProblem(item, where) {
|
|
|
373
372
|
if (problem) return problem;
|
|
374
373
|
}
|
|
375
374
|
if (across !== void 0 && choices !== void 0) return `questions "${text}" in ${where} sets both across and choices. choices are the answers a question asked of each attempt picks from, and across makes it pick between two attempts instead: drop choices to vote between pairs, or drop across to ask each attempt.`;
|
|
376
|
-
const parts =
|
|
375
|
+
const parts = questionInputsProblem(item, `questions "${text}"`, where);
|
|
377
376
|
if (parts) return parts;
|
|
378
377
|
if (typeof model === "string" && isEvaluationModel(model)) return `questions "${text}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't answer a question. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
|
|
379
378
|
return `questions "${text}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$2(model)})`;
|
|
@@ -389,6 +388,8 @@ function questionsProblem(value, where) {
|
|
|
389
388
|
//#endregion
|
|
390
389
|
//#region src/contract/config.ts
|
|
391
390
|
const count = () => z.number().int().min(1);
|
|
391
|
+
/** `runs`: a count, or `{ bestOf }` with an odd count of 3 or more. */
|
|
392
|
+
const runs = () => z.union([count(), z.strictObject({ bestOf: z.number().int().min(3).refine((n) => n % 2 === 1, "must be odd") })]);
|
|
392
393
|
const captureWindowSchema = z.strictObject({
|
|
393
394
|
width: count(),
|
|
394
395
|
height: count()
|
|
@@ -421,14 +422,15 @@ const profileDefaultsDataSchema = z.strictObject({
|
|
|
421
422
|
harness: harnessList.optional(),
|
|
422
423
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
423
424
|
environment: environmentDataSchema.optional(),
|
|
424
|
-
runs:
|
|
425
|
+
runs: runs().optional(),
|
|
425
426
|
gate: gatePolicySchema.optional(),
|
|
426
427
|
env: envVarsSchema.optional(),
|
|
427
428
|
egress: egressPolicySchema.optional(),
|
|
428
429
|
judge: configJudgeDataSchema.optional(),
|
|
429
430
|
timeout: durationSchema.optional(),
|
|
430
431
|
capture: captureOptionsSchema.optional(),
|
|
431
|
-
question: questionOptionsSchema.optional()
|
|
432
|
+
question: questionOptionsSchema.optional(),
|
|
433
|
+
snapshot: snapshotOptionSchema.optional()
|
|
432
434
|
});
|
|
433
435
|
const profileLimitSchema = z.strictObject({
|
|
434
436
|
runs: count().optional(),
|
|
@@ -481,7 +483,7 @@ const reevalConfigDataSchema = z.strictObject({
|
|
|
481
483
|
harness: harnessList.optional(),
|
|
482
484
|
sandbox: sandboxDescriptorSchema.optional(),
|
|
483
485
|
environment: environmentDataSchema.optional(),
|
|
484
|
-
runs:
|
|
486
|
+
runs: runs().optional(),
|
|
485
487
|
gate: gatePolicySchema.optional(),
|
|
486
488
|
env: envVarsSchema.optional(),
|
|
487
489
|
egress: egressPolicySchema.optional(),
|
|
@@ -489,6 +491,7 @@ const reevalConfigDataSchema = z.strictObject({
|
|
|
489
491
|
timeout: durationSchema.optional(),
|
|
490
492
|
capture: captureOptionsSchema.optional(),
|
|
491
493
|
question: questionOptionsSchema.optional(),
|
|
494
|
+
snapshot: snapshotOptionSchema.optional(),
|
|
492
495
|
concurrency: count().optional(),
|
|
493
496
|
record: recordMode.optional(),
|
|
494
497
|
history: historyConfigSchema.optional(),
|
|
@@ -1125,7 +1128,7 @@ const AGENTS = {
|
|
|
1125
1128
|
},
|
|
1126
1129
|
unreported: cursorUnreported,
|
|
1127
1130
|
mcpServers: cursorMcpServers,
|
|
1128
|
-
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-
|
|
1131
|
+
adapter: async ({ auth, mcpServers, descriptor }) => (await import("./src-DMo2eCOJ.js")).createCursorSdk({
|
|
1129
1132
|
auth,
|
|
1130
1133
|
...mcpServers ? { mcpServers } : {},
|
|
1131
1134
|
...descriptor.kind === "cursor" && descriptor.params ? { modelParams: descriptor.params } : {},
|
|
@@ -1192,7 +1195,7 @@ const AGENTS = {
|
|
|
1192
1195
|
},
|
|
1193
1196
|
servedModel: geminiServedModel,
|
|
1194
1197
|
mcpServers: antigravityMcpServers,
|
|
1195
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-
|
|
1198
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins, vertex }) => (await import("./src-OESRtvOi.js")).createAntigravity({
|
|
1196
1199
|
auth,
|
|
1197
1200
|
...vertex ? { vertex } : {},
|
|
1198
1201
|
...mcpServers ? { mcpServers } : {},
|
|
@@ -1219,7 +1222,7 @@ const AGENTS = {
|
|
|
1219
1222
|
servedModel: geminiServedModel,
|
|
1220
1223
|
forModel: opencodeForModel,
|
|
1221
1224
|
mcpServers: opencodeMcpServers,
|
|
1222
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1225
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-CgJrz5p0.js")).createOpenCode({
|
|
1223
1226
|
auth,
|
|
1224
1227
|
model,
|
|
1225
1228
|
...vertex ? { vertex } : { vars: opencodeVars(model) },
|
|
@@ -1241,14 +1244,14 @@ const AGENTS = {
|
|
|
1241
1244
|
},
|
|
1242
1245
|
servedModel: geminiServedModel,
|
|
1243
1246
|
mcpServers: gooseMcpServers,
|
|
1244
|
-
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-
|
|
1247
|
+
adapter: async ({ auth, mcpServers, descriptor, model, vertex }) => (await import("./src-BTqJbvj6.js")).createGoose({
|
|
1245
1248
|
...vertex ? { vertex } : {},
|
|
1246
1249
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1247
1250
|
auth,
|
|
1248
|
-
provider: (await import("./src-
|
|
1251
|
+
provider: (await import("./src-BTqJbvj6.js")).AI_GATEWAY_PROVIDER
|
|
1249
1252
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1250
1253
|
auth,
|
|
1251
|
-
provider: (await import("./src-
|
|
1254
|
+
provider: (await import("./src-BTqJbvj6.js")).OPENROUTER_PROVIDER
|
|
1252
1255
|
} : { auth: { GOOGLE_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" } },
|
|
1253
1256
|
model,
|
|
1254
1257
|
...descriptor.kind === "goose" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
@@ -1269,13 +1272,13 @@ const AGENTS = {
|
|
|
1269
1272
|
endpoints: GEMINI_ENDPOINTS,
|
|
1270
1273
|
servedModel: geminiServedModel,
|
|
1271
1274
|
mcpServers: piMcpServers,
|
|
1272
|
-
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-
|
|
1275
|
+
adapter: async ({ auth, mcpServers, descriptor, vertex }) => (await import("./src-BkiComGh.js")).createPi({
|
|
1273
1276
|
...auth["AI_GATEWAY_API_KEY"] ? {
|
|
1274
1277
|
auth,
|
|
1275
|
-
gateway: (await import("./src-
|
|
1278
|
+
gateway: (await import("./src-BkiComGh.js")).PI_AI_GATEWAY
|
|
1276
1279
|
} : auth["OPENROUTER_API_KEY"] ? {
|
|
1277
1280
|
auth,
|
|
1278
|
-
gateway: (await import("./src-
|
|
1281
|
+
gateway: (await import("./src-BkiComGh.js")).PI_OPENROUTER
|
|
1279
1282
|
} : {
|
|
1280
1283
|
auth: { GEMINI_API_KEY: auth.GOOGLE_GENERATIVE_AI_API_KEY ?? "" },
|
|
1281
1284
|
...vertex ? { vertex } : {}
|
|
@@ -1308,9 +1311,9 @@ const AGENTS = {
|
|
|
1308
1311
|
},
|
|
1309
1312
|
servedModel: openaiServedModel,
|
|
1310
1313
|
mcpServers: codexMcpServers,
|
|
1311
|
-
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-
|
|
1314
|
+
adapter: async ({ auth, mcpServers, descriptor, plugins }) => (await import("./src-BvgeZ4tS.js")).createCodex({
|
|
1312
1315
|
auth,
|
|
1313
|
-
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-
|
|
1316
|
+
...auth["AI_GATEWAY_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).AI_GATEWAY_PROVIDER } : auth["OPENROUTER_API_KEY"] ? { provider: (await import("./src-BvgeZ4tS.js")).OPENROUTER_PROVIDER } : {},
|
|
1314
1317
|
...descriptor.kind === "codex" && descriptor.effort ? { effort: descriptor.effort } : {},
|
|
1315
1318
|
...mcpServers ? { mcpServers } : {},
|
|
1316
1319
|
...plugins.length ? { plugins: plugins.map(({ name, dir }) => ({
|
|
@@ -1328,7 +1331,7 @@ const AGENTS = {
|
|
|
1328
1331
|
...fxForModel("provider/model"),
|
|
1329
1332
|
forModel: fxForModel,
|
|
1330
1333
|
mcpServers: acpMcpServers,
|
|
1331
|
-
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-
|
|
1334
|
+
adapter: async ({ auth, env, mcpServers, model, descriptor }) => (await import("./src-Dk2Na0b-.js")).createFx({
|
|
1332
1335
|
auth,
|
|
1333
1336
|
model,
|
|
1334
1337
|
...auth["OPENROUTER_API_KEY"] ? { provider: "openrouter" } : {},
|
|
@@ -1353,7 +1356,7 @@ const AGENTS = {
|
|
|
1353
1356
|
servedModel: xaiServedModel,
|
|
1354
1357
|
mcpServers: acpMcpServers,
|
|
1355
1358
|
adapter: async ({ auth, mcpServers, descriptor, plugins, model, vertexChat }) => {
|
|
1356
|
-
const grok = await import("./src-
|
|
1359
|
+
const grok = await import("./src-Cp-2Hdib.js");
|
|
1357
1360
|
return grok.createGrokBuild({
|
|
1358
1361
|
...vertexChat ? {
|
|
1359
1362
|
auth: { [grok.VERTEX_TOKEN_ENV]: SANDBOX_ACCESS_TOKEN },
|
|
@@ -1704,11 +1707,10 @@ function resolvedEffortOf(models) {
|
|
|
1704
1707
|
/** The error a per-attempt answer carries when the run stopped before a model answered it. */
|
|
1705
1708
|
const STOPPED_FIRST = "the run stopped before it was answered";
|
|
1706
1709
|
/**
|
|
1707
|
-
* What
|
|
1708
|
-
* turn's prompt.
|
|
1709
|
-
* win a side-by-side pick on its wording; the changed files repeat what the diff shows.
|
|
1710
|
+
* What a recorded pairwise answer without `stateOptions` showed each side: the task and the diff,
|
|
1711
|
+
* every turn's prompt.
|
|
1710
1712
|
*/
|
|
1711
|
-
const
|
|
1713
|
+
const UNRECORDED_QUESTION_STATE = {
|
|
1712
1714
|
task: true,
|
|
1713
1715
|
output: false,
|
|
1714
1716
|
files: false,
|
|
@@ -1717,62 +1719,39 @@ const DEFAULT_QUESTION_STATE = {
|
|
|
1717
1719
|
turns: "all",
|
|
1718
1720
|
checks: false
|
|
1719
1721
|
};
|
|
1722
|
+
const KEYS = Object.keys(UNRECORDED_QUESTION_STATE);
|
|
1720
1723
|
/**
|
|
1721
|
-
*
|
|
1722
|
-
*
|
|
1723
|
-
* agent went about the work is the point.
|
|
1724
|
+
* Every key set, in the record's key order, so equal selections serialize the same: an input the
|
|
1725
|
+
* question didn't turn on is off, and `turns` is `'all'` unless it says `'last'`.
|
|
1724
1726
|
*/
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
const defaults = defaultQuestionState(pairwise);
|
|
1736
|
-
const out = {};
|
|
1737
|
-
for (const key of KEYS) out[key] = state?.[key] ?? defaults[key];
|
|
1738
|
-
return out;
|
|
1727
|
+
function resolveQuestionState(state) {
|
|
1728
|
+
return {
|
|
1729
|
+
task: state?.task ?? false,
|
|
1730
|
+
output: state?.output ?? false,
|
|
1731
|
+
files: state?.files ?? false,
|
|
1732
|
+
diff: state?.diff ?? false,
|
|
1733
|
+
transcript: state?.transcript ?? false,
|
|
1734
|
+
turns: state?.turns ?? "all",
|
|
1735
|
+
checks: state?.checks ?? false
|
|
1736
|
+
};
|
|
1739
1737
|
}
|
|
1740
|
-
/** `inner` over `outer
|
|
1738
|
+
/** `inner` over `outer`, field by field. */
|
|
1741
1739
|
function mergeQuestion(outer, inner) {
|
|
1742
|
-
|
|
1743
|
-
const out = {
|
|
1740
|
+
return inner ? {
|
|
1744
1741
|
...outer,
|
|
1745
1742
|
...inner
|
|
1746
|
-
};
|
|
1747
|
-
if (outer.state || inner.state) out.state = {
|
|
1748
|
-
...outer.state,
|
|
1749
|
-
...inner.state
|
|
1750
|
-
};
|
|
1751
|
-
return out;
|
|
1743
|
+
} : outer;
|
|
1752
1744
|
}
|
|
1753
|
-
/** A planned question
|
|
1745
|
+
/** A planned question under the cascaded `question` defaults: its own `model`, else theirs. */
|
|
1754
1746
|
function withQuestionDefaults(q, defaults) {
|
|
1755
|
-
const { state, artifacts } = mergeQuestion(defaults, {
|
|
1756
|
-
...q.state ? { state: q.state } : {},
|
|
1757
|
-
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
1758
|
-
});
|
|
1759
|
-
const { state: _state, artifacts: _artifacts, ...rest } = q;
|
|
1760
1747
|
const model = q.model ?? defaults.model;
|
|
1761
|
-
return {
|
|
1762
|
-
...
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
...artifacts !== void 0 ? { artifacts } : {}
|
|
1766
|
-
};
|
|
1748
|
+
return model !== void 0 ? {
|
|
1749
|
+
...q,
|
|
1750
|
+
model
|
|
1751
|
+
} : q;
|
|
1767
1752
|
}
|
|
1768
1753
|
const isOff = (value) => value === false || Array.isArray(value) && !value.length;
|
|
1769
|
-
/**
|
|
1770
|
-
const showsNothing = (q) => {
|
|
1771
|
-
if (q.artifacts !== void 0) return false;
|
|
1772
|
-
const s = resolveQuestionState(q.state, q.across !== void 0);
|
|
1773
|
-
return KEYS.every((k) => k === "turns" || isOff(s[k]));
|
|
1774
|
-
};
|
|
1775
|
-
/** The state keys a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1754
|
+
/** The inputs a question shows, in order, plus `artifacts` when it sends captures. */
|
|
1776
1755
|
function questionShows(state, artifacts) {
|
|
1777
1756
|
const on = KEYS.filter((k) => k !== "turns" && !isOff(state[k]));
|
|
1778
1757
|
return artifacts === void 0 ? on : [...on, "artifacts"];
|
|
@@ -1838,6 +1817,189 @@ function compareProblem(value, where, environment) {
|
|
|
1838
1817
|
if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness? }.`;
|
|
1839
1818
|
return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
|
|
1840
1819
|
}
|
|
1820
|
+
const POLL_MS = 200;
|
|
1821
|
+
/**
|
|
1822
|
+
* Prints `alive` while `pid` runs, then the sandbox's TCP tables, or `noproc` without them
|
|
1823
|
+
* (or without `tables`: the fake sandbox shares the host's, where other processes bind too).
|
|
1824
|
+
*/
|
|
1825
|
+
function probeScript(pid, tables = true) {
|
|
1826
|
+
return `${pid ? `kill -0 ${pid} 2>/dev/null && echo alive; ` : ""}${tables ? "if [ -r /proc/net/tcp ]; then cat /proc/net/tcp /proc/net/tcp6 2>/dev/null; else echo noproc; fi" : "echo noproc"}; exit 0`;
|
|
1827
|
+
}
|
|
1828
|
+
/** `0100007F` → `127.0.0.1`; 32 hex digits → an IPv6 address. Each 32-bit word is host-endian. */
|
|
1829
|
+
function procAddress(hex) {
|
|
1830
|
+
const bytes = (hex.match(/.{8}/g) ?? []).flatMap((w) => (w.match(/../g) ?? []).reverse().map((b) => Number.parseInt(b, 16)));
|
|
1831
|
+
if (bytes.length === 4) return bytes.join(".");
|
|
1832
|
+
if (bytes.slice(0, 10).every((b) => b === 0) && bytes[10] === 255 && bytes[11] === 255) return bytes.slice(12).join(".");
|
|
1833
|
+
const groups = [];
|
|
1834
|
+
for (let i = 0; i < 16; i += 2) groups.push(((bytes[i] ?? 0) << 8 | (bytes[i + 1] ?? 0)).toString(16));
|
|
1835
|
+
const full = groups.join(":");
|
|
1836
|
+
const longest = (full.match(/(^|:)0(:0)+(:|$)/g) ?? []).reduce((a, b) => b.length > a.length ? b : a, "");
|
|
1837
|
+
return longest ? full.replace(longest, "::") : full;
|
|
1838
|
+
}
|
|
1839
|
+
/** A listener the tunnel reaches: on loopback or every address, IPv4 or IPv6. */
|
|
1840
|
+
const reachable = (address) => address === "0.0.0.0" || address === "::" || address === "::1" || address.startsWith("127.");
|
|
1841
|
+
function parseProbe(stdout) {
|
|
1842
|
+
const lines = stdout.split("\n");
|
|
1843
|
+
const alive = lines.includes("alive");
|
|
1844
|
+
if (lines.includes("noproc")) return {
|
|
1845
|
+
proc: false,
|
|
1846
|
+
alive
|
|
1847
|
+
};
|
|
1848
|
+
const listeners = [];
|
|
1849
|
+
for (const l of lines) {
|
|
1850
|
+
const m = /^\s*\d+:\s+([0-9A-F]+):([0-9A-F]{4})\s+\S+\s+0A\s/i.exec(l);
|
|
1851
|
+
const address = m ? procAddress(m[1]) : "";
|
|
1852
|
+
if (m && reachable(address)) listeners.push({
|
|
1853
|
+
port: Number.parseInt(m[2], 16),
|
|
1854
|
+
address
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
return {
|
|
1858
|
+
proc: true,
|
|
1859
|
+
alive,
|
|
1860
|
+
listeners
|
|
1861
|
+
};
|
|
1862
|
+
}
|
|
1863
|
+
/** The ports listening now that weren't before the command started, ascending. */
|
|
1864
|
+
const newPorts = (listeners, before) => [...new Set(listeners.map((l) => l.port).filter((p) => !before.has(p)))].sort((a, b) => a - b);
|
|
1865
|
+
/** The address to record for `port`: IPv4 first, since that's where the tunnel connects first. */
|
|
1866
|
+
const addressFor = (listeners, port) => {
|
|
1867
|
+
const all = listeners.filter((l) => l.port === port).map((l) => l.address);
|
|
1868
|
+
return all.find((a) => !a.includes(":")) ?? all.find((a) => a === "::") ?? all[0] ?? "";
|
|
1869
|
+
};
|
|
1870
|
+
/** `want` is taken whoever listens on it: reeval allocated it, or the eval named it. */
|
|
1871
|
+
function decide(s) {
|
|
1872
|
+
const { listeners, alive } = s.probe;
|
|
1873
|
+
const ports = newPorts(listeners, s.before);
|
|
1874
|
+
if (listeners.some((l) => l.port === s.want)) return {
|
|
1875
|
+
kind: "port",
|
|
1876
|
+
port: s.want,
|
|
1877
|
+
address: addressFor(listeners, s.want)
|
|
1878
|
+
};
|
|
1879
|
+
const late = s.elapsedMs >= s.timeoutMs;
|
|
1880
|
+
if (!s.given && ports.length && (s.stableMs >= 3e3 || late || !alive)) {
|
|
1881
|
+
if (ports.length === 1) {
|
|
1882
|
+
const port = ports[0];
|
|
1883
|
+
return {
|
|
1884
|
+
kind: "port",
|
|
1885
|
+
port,
|
|
1886
|
+
address: addressFor(listeners, port)
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
return {
|
|
1890
|
+
kind: "several",
|
|
1891
|
+
ports
|
|
1892
|
+
};
|
|
1893
|
+
}
|
|
1894
|
+
if (!alive) return { kind: "exited" };
|
|
1895
|
+
return late ? {
|
|
1896
|
+
kind: "timeout",
|
|
1897
|
+
ports
|
|
1898
|
+
} : { kind: "wait" };
|
|
1899
|
+
}
|
|
1900
|
+
const plain = (output) => output.replace(/\x1b\[[0-9;?]*[A-Za-z]/g, "");
|
|
1901
|
+
/** The URLs `output` prints on one of `ports`, as Vite's `Local: http://localhost:5173/`. */
|
|
1902
|
+
function printedUrls(output, ports) {
|
|
1903
|
+
const found = /* @__PURE__ */ new Map();
|
|
1904
|
+
for (const m of plain(output).matchAll(/https?:\/\/(?:\[[0-9a-f:]+\]|[^\s/:[\]]+):(\d+)[^\s'"]*/gi)) {
|
|
1905
|
+
const port = Number(m[1]);
|
|
1906
|
+
if (ports.includes(port) && !found.has(m[0])) found.set(m[0], port);
|
|
1907
|
+
}
|
|
1908
|
+
return [...found].map(([url, port]) => ({
|
|
1909
|
+
url,
|
|
1910
|
+
port
|
|
1911
|
+
}));
|
|
1912
|
+
}
|
|
1913
|
+
/** Whether something on the host accepts a TCP connection on `port`, IPv4 or IPv6 loopback. */
|
|
1914
|
+
async function hostListening(port) {
|
|
1915
|
+
for (const host of ["127.0.0.1", "::1"]) if (await new Promise((resolve) => {
|
|
1916
|
+
const socket = connect({
|
|
1917
|
+
port,
|
|
1918
|
+
host
|
|
1919
|
+
});
|
|
1920
|
+
socket.once("connect", () => {
|
|
1921
|
+
socket.destroy();
|
|
1922
|
+
resolve(true);
|
|
1923
|
+
});
|
|
1924
|
+
socket.once("error", () => resolve(false));
|
|
1925
|
+
})) return host;
|
|
1926
|
+
}
|
|
1927
|
+
const sleep = (ms, signal) => new Promise((resolve) => {
|
|
1928
|
+
const done = () => {
|
|
1929
|
+
clearTimeout(timer);
|
|
1930
|
+
signal?.removeEventListener("abort", done);
|
|
1931
|
+
resolve();
|
|
1932
|
+
};
|
|
1933
|
+
const timer = setTimeout(done, ms);
|
|
1934
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
1935
|
+
});
|
|
1936
|
+
const listOf = (items) => items.length > 1 ? `${items.slice(0, -1).join(", ")} and ${items.at(-1)}` : String(items[0]);
|
|
1937
|
+
/**
|
|
1938
|
+
* Polls until something listens on `want`, or, without a given port, until the ports that are
|
|
1939
|
+
* new since the command started have settled on one. Throws, with the command's last output,
|
|
1940
|
+
* when they settle on several, it exits, or nothing listens in time.
|
|
1941
|
+
*/
|
|
1942
|
+
async function awaitListening(command, w) {
|
|
1943
|
+
const timeoutMs = w.timeoutMs ?? 12e4;
|
|
1944
|
+
const started = Date.now();
|
|
1945
|
+
let last = "";
|
|
1946
|
+
let changed = started;
|
|
1947
|
+
const output = async () => plain(await w.output().catch(() => ""));
|
|
1948
|
+
const fail = async (why, out) => {
|
|
1949
|
+
const tail = (out ?? await output()).trimEnd().split("\n").slice(-20).join("\n").trim();
|
|
1950
|
+
return /* @__PURE__ */ new Error(`c.serve(${JSON.stringify(command)}) ${why}${tail ? `. Its last output:\n${tail}` : ""}`);
|
|
1951
|
+
};
|
|
1952
|
+
for (;;) {
|
|
1953
|
+
w.signal?.throwIfAborted();
|
|
1954
|
+
const probe = await w.probe();
|
|
1955
|
+
const now = Date.now();
|
|
1956
|
+
if (!probe.proc) {
|
|
1957
|
+
const address = await (w.hostListening ?? hostListening)(w.want);
|
|
1958
|
+
if (address !== void 0) return {
|
|
1959
|
+
port: w.want,
|
|
1960
|
+
address,
|
|
1961
|
+
detected: false,
|
|
1962
|
+
readyMs: now - started
|
|
1963
|
+
};
|
|
1964
|
+
if (!probe.alive) throw await fail(`exited before it listened on ${w.want}`);
|
|
1965
|
+
if (now - started >= timeoutMs) throw await fail(`didn't listen on ${w.want} within ${timeoutMs / 1e3}s. This sandbox can't list the ports it listens on, so it must listen on PORT or the port passed`);
|
|
1966
|
+
} else {
|
|
1967
|
+
const key = newPorts(probe.listeners, w.before).join(",");
|
|
1968
|
+
if (key !== last) {
|
|
1969
|
+
last = key;
|
|
1970
|
+
changed = now;
|
|
1971
|
+
}
|
|
1972
|
+
const d = decide({
|
|
1973
|
+
want: w.want,
|
|
1974
|
+
given: w.given,
|
|
1975
|
+
probe,
|
|
1976
|
+
before: w.before,
|
|
1977
|
+
stableMs: now - changed,
|
|
1978
|
+
elapsedMs: now - started,
|
|
1979
|
+
timeoutMs
|
|
1980
|
+
});
|
|
1981
|
+
if (d.kind === "port") return {
|
|
1982
|
+
port: d.port,
|
|
1983
|
+
address: d.address,
|
|
1984
|
+
detected: d.port !== w.want,
|
|
1985
|
+
readyMs: now - started
|
|
1986
|
+
};
|
|
1987
|
+
if (d.kind === "several") {
|
|
1988
|
+
const out = await output();
|
|
1989
|
+
const urls = printedUrls(out, d.ports);
|
|
1990
|
+
const printed = urls.length ? ` It printed ${listOf(urls.map((u) => u.url))}.` : "";
|
|
1991
|
+
const hint = urls[0]?.port ?? d.ports[0];
|
|
1992
|
+
throw await fail(`listens on ports ${listOf(d.ports)}, and not on PORT (${w.want}).${printed} Pass the one it serves on: c.serve(${JSON.stringify(command)}, ${hint})`, out);
|
|
1993
|
+
}
|
|
1994
|
+
if (d.kind === "exited") throw await fail("exited before it listened on a port");
|
|
1995
|
+
if (d.kind === "timeout") {
|
|
1996
|
+
const on = d.ports.length ? `; ${d.ports.length > 1 ? "ports" : "port"} ${listOf(d.ports)} started listening` : "";
|
|
1997
|
+
throw await fail(`didn't listen on ${w.given ? `the port passed (${w.want})` : "a port"} within ${timeoutMs / 1e3}s${on}`);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
await sleep(POLL_MS, w.signal);
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
1841
2003
|
//#endregion
|
|
1842
2004
|
//#region src/sandbox/fake.ts
|
|
1843
2005
|
/** Commands run on the host's own network, so the fake sandbox can neither restrict nor see it. */
|
|
@@ -1982,7 +2144,7 @@ async function createFakeSandbox(descriptor, env = {}) {
|
|
|
1982
2144
|
},
|
|
1983
2145
|
home: base,
|
|
1984
2146
|
readFile: (path) => readFile(resolve(path), "utf8").catch(() => null),
|
|
1985
|
-
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}
|
|
2147
|
+
getPortEndpoint: async ({ port, protocol = "http" }) => ({ url: `${protocol === "ws" ? "ws" : "http"}://${await hostListening(port) === "::1" ? "[::1]" : "127.0.0.1"}:${port}` }),
|
|
1986
2148
|
async writeFile(path, content) {
|
|
1987
2149
|
const target = resolve(path);
|
|
1988
2150
|
await mkdir(dirname(target), { recursive: true });
|
|
@@ -2200,34 +2362,28 @@ function egressLabel({ policy, auto }) {
|
|
|
2200
2362
|
}
|
|
2201
2363
|
//#endregion
|
|
2202
2364
|
//#region src/judge/state-options.ts
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2365
|
+
/**
|
|
2366
|
+
* What one call turned on, every key set (`false` when off), in the record's key order, so equal
|
|
2367
|
+
* selections serialize the same.
|
|
2368
|
+
*/
|
|
2369
|
+
function resolveStateOptions(call) {
|
|
2370
|
+
return {
|
|
2371
|
+
task: call.task ?? false,
|
|
2372
|
+
output: call.output ?? false,
|
|
2373
|
+
files: call.files ?? false,
|
|
2374
|
+
diff: call.diff ?? false,
|
|
2375
|
+
contents: call.contents ?? [],
|
|
2376
|
+
transcript: call.transcript ?? false,
|
|
2377
|
+
turns: call.turns ?? "last",
|
|
2378
|
+
checks: call.checks ?? false
|
|
2379
|
+
};
|
|
2218
2380
|
}
|
|
2219
|
-
/** `inner` over `outer`, field by field
|
|
2381
|
+
/** `inner` over `outer`, field by field. */
|
|
2220
2382
|
function mergeJudge(outer, inner) {
|
|
2221
|
-
|
|
2222
|
-
const out = {
|
|
2383
|
+
return inner ? {
|
|
2223
2384
|
...outer,
|
|
2224
2385
|
...inner
|
|
2225
|
-
};
|
|
2226
|
-
if (outer.state || inner.state) out.state = {
|
|
2227
|
-
...outer.state,
|
|
2228
|
-
...inner.state
|
|
2229
|
-
};
|
|
2230
|
-
return out;
|
|
2386
|
+
} : outer;
|
|
2231
2387
|
}
|
|
2232
2388
|
const KNOWN = [...[
|
|
2233
2389
|
"harness",
|
|
@@ -2243,6 +2399,7 @@ const KNOWN = [...[
|
|
|
2243
2399
|
"question",
|
|
2244
2400
|
"questions",
|
|
2245
2401
|
"timeout",
|
|
2402
|
+
"snapshot",
|
|
2246
2403
|
"tags",
|
|
2247
2404
|
"revision",
|
|
2248
2405
|
"pin"
|
|
@@ -2257,6 +2414,7 @@ const SUGGEST = {
|
|
|
2257
2414
|
repeats: "runs",
|
|
2258
2415
|
iterations: "runs",
|
|
2259
2416
|
trials: "runs",
|
|
2417
|
+
bestOf: "runs",
|
|
2260
2418
|
secrets: "env",
|
|
2261
2419
|
vars: "env",
|
|
2262
2420
|
network: "egress",
|
|
@@ -2265,9 +2423,26 @@ const SUGGEST = {
|
|
|
2265
2423
|
version: "revision",
|
|
2266
2424
|
rev: "revision",
|
|
2267
2425
|
pins: "pin",
|
|
2426
|
+
workspace: "snapshot",
|
|
2427
|
+
snapshots: "snapshot",
|
|
2268
2428
|
passRate: "gate",
|
|
2269
2429
|
threshold: "gate"
|
|
2270
2430
|
};
|
|
2431
|
+
/**
|
|
2432
|
+
* `snapshot` over the levels above: the shorthand sets `when`, the object sets the fields it
|
|
2433
|
+
* names, and `compress` carries through either.
|
|
2434
|
+
*/
|
|
2435
|
+
function mergeSnapshot(outer, inner) {
|
|
2436
|
+
if (inner === void 0) return outer;
|
|
2437
|
+
if (typeof inner !== "object") return {
|
|
2438
|
+
...outer,
|
|
2439
|
+
when: inner
|
|
2440
|
+
};
|
|
2441
|
+
return {
|
|
2442
|
+
when: inner.when ?? outer.when,
|
|
2443
|
+
compress: inner.compress ?? outer.compress
|
|
2444
|
+
};
|
|
2445
|
+
}
|
|
2271
2446
|
/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
|
|
2272
2447
|
function checkGate(value, where) {
|
|
2273
2448
|
const problem = gateProblem(value, where);
|
|
@@ -2335,19 +2510,44 @@ function refuseIllegalCompare(options, where) {
|
|
|
2335
2510
|
if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
|
|
2336
2511
|
}
|
|
2337
2512
|
const isRevision = (value) => typeof value === "number" && Number.isSafeInteger(value) || typeof value === "string" && value.trim() !== "";
|
|
2513
|
+
/** `runs: 3` or `runs: { bestOf: 5 }`, refused in the words of whichever it was meant to be. */
|
|
2514
|
+
function checkRuns(runs, where) {
|
|
2515
|
+
if (typeof runs === "number" && Number.isInteger(runs) && runs >= 1) return;
|
|
2516
|
+
if (typeof runs === "number") throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
|
|
2517
|
+
if (!isObjectLike(runs)) throw new UsageError(`runs in ${where} must be a whole number of at least 1, or { bestOf: n } (got ${JSON.stringify(runs)})`);
|
|
2518
|
+
const extra = Object.keys(runs).find((k) => k !== "bestOf");
|
|
2519
|
+
if (extra !== void 0) throw new UsageError(`runs in ${where} takes { bestOf: n } only (got the key "${extra}"), e.g. runs: { bestOf: 3 }`);
|
|
2520
|
+
const { bestOf } = runs;
|
|
2521
|
+
if (!(Number.isInteger(bestOf) && bestOf >= 3 && bestOf % 2 === 1)) throw new UsageError(`runs.bestOf in ${where} must be an odd whole number of 3 or more, so one side always has a majority, e.g. runs: { bestOf: 3 } (got ${JSON.stringify(bestOf)})`);
|
|
2522
|
+
}
|
|
2523
|
+
/** The most attempts `runs` plans a cell, and its `bestOf` when it has one. */
|
|
2524
|
+
const runsOf = (runs) => typeof runs === "number" ? { runs } : {
|
|
2525
|
+
runs: runs.bestOf,
|
|
2526
|
+
bestOf: runs.bestOf
|
|
2527
|
+
};
|
|
2528
|
+
/**
|
|
2529
|
+
* A best-of cell's gate: half or more of its counted attempts pass, which is its majority once it
|
|
2530
|
+
* has one (an infra error counts on neither side).
|
|
2531
|
+
*/
|
|
2532
|
+
const MAJORITY_GATE = { passRate: .5 };
|
|
2533
|
+
/** How many of a best-of cell's attempts always run: the majority. */
|
|
2534
|
+
const majorityOf = (bestOf) => (bestOf + 1) / 2;
|
|
2338
2535
|
/** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
|
|
2339
2536
|
function checkOptions(options, where, allowed = KNOWN) {
|
|
2340
2537
|
for (const key of Object.keys(options)) {
|
|
2341
2538
|
if (allowed.includes(key)) continue;
|
|
2342
2539
|
const near = SUGGEST[key] ?? closest(key, allowed);
|
|
2343
|
-
const
|
|
2540
|
+
const value = options[key];
|
|
2541
|
+
const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : key === "workspace" && typeof value !== "boolean" ? " workspace is an environment's starting tree: defineEnvironment(name, { workspace: git(…) })." : near ? ` Did you mean "${near}"?` : "";
|
|
2344
2542
|
throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
|
|
2345
2543
|
}
|
|
2346
2544
|
const o = options;
|
|
2347
|
-
if (o.runs !== void 0
|
|
2545
|
+
if (o.runs !== void 0) checkRuns(o.runs, where);
|
|
2348
2546
|
if (o.gate !== void 0) checkGate(o.gate, where);
|
|
2349
2547
|
if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
|
|
2350
2548
|
if (o.revision !== void 0 && !isRevision(o.revision)) throw new UsageError(`revision in ${where} must be a whole number or a non-empty string, e.g. revision: 2 (got ${JSON.stringify(o.revision)})`);
|
|
2549
|
+
const snapshotIssue = snapshotProblem(o.snapshot, where);
|
|
2550
|
+
if (snapshotIssue) throw new UsageError(snapshotIssue);
|
|
2351
2551
|
if (o.env !== void 0) checkEnv(o.env, where, UsageError);
|
|
2352
2552
|
if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
|
|
2353
2553
|
const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
|
|
@@ -2478,17 +2678,18 @@ function checkUnique(list, where) {
|
|
|
2478
2678
|
seen.add(key);
|
|
2479
2679
|
}
|
|
2480
2680
|
}
|
|
2481
|
-
const plannedQuestions = (value) => value.map((q, i) =>
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2681
|
+
const plannedQuestions = (value) => value.map((q, i) => {
|
|
2682
|
+
const { question, model, across, choices, artifacts, ...rest } = q;
|
|
2683
|
+
const state = Object.fromEntries(Object.entries(rest).filter(([, v]) => v !== void 0));
|
|
2684
|
+
return {
|
|
2685
|
+
n: i + 1,
|
|
2686
|
+
question,
|
|
2687
|
+
...model !== void 0 ? { model } : {},
|
|
2688
|
+
...across !== void 0 ? { across } : {},
|
|
2689
|
+
...choices ? { choices } : {},
|
|
2690
|
+
state,
|
|
2691
|
+
...artifacts !== void 0 ? { artifacts } : {}
|
|
2692
|
+
};
|
|
2492
2693
|
});
|
|
2493
2694
|
/** Whether the eval's matrix varies each factor a pairwise question can vary. */
|
|
2494
2695
|
const variesBy = (r) => ({
|
|
@@ -2529,9 +2730,14 @@ function applyLevel(out, options, where, egress = "own") {
|
|
|
2529
2730
|
if (options.sandbox !== void 0) out.sandbox = options.sandbox;
|
|
2530
2731
|
if (options.environment !== void 0) out.environment = options.environment;
|
|
2531
2732
|
if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
|
|
2532
|
-
if (options.runs !== void 0)
|
|
2733
|
+
if (options.runs !== void 0) {
|
|
2734
|
+
const { runs, bestOf } = runsOf(options.runs);
|
|
2735
|
+
out.runs = runs;
|
|
2736
|
+
out.bestOf = bestOf;
|
|
2737
|
+
}
|
|
2533
2738
|
if (options.gate !== void 0) out.gate = options.gate;
|
|
2534
2739
|
if (options.timeout !== void 0) out.timeout = options.timeout;
|
|
2740
|
+
out.snapshot = mergeSnapshot(out.snapshot, options.snapshot);
|
|
2535
2741
|
if (options.revision !== void 0) out.revision = options.revision;
|
|
2536
2742
|
Object.assign(out.env, options.env);
|
|
2537
2743
|
out.judge = mergeJudge(out.judge, options.judge);
|
|
@@ -2578,11 +2784,7 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2578
2784
|
baseline: out.environment,
|
|
2579
2785
|
from: "environment"
|
|
2580
2786
|
};
|
|
2581
|
-
if (out.questions.length)
|
|
2582
|
-
out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2583
|
-
const blank = out.questions.find(showsNothing);
|
|
2584
|
-
if (blank) throw new UsageError(`questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
|
|
2585
|
-
}
|
|
2787
|
+
if (out.questions.length) out.questions = out.questions.map((q) => withQuestionDefaults(q, out.question));
|
|
2586
2788
|
checkUnique(out.harness, where);
|
|
2587
2789
|
const harnessFromProfile = profile?.defaults.harness !== void 0 && !levels.some((l) => l.options.harness);
|
|
2588
2790
|
const authored = variesBy(out);
|
|
@@ -2648,8 +2850,10 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2648
2850
|
}
|
|
2649
2851
|
defaultRuns(out, notes, options);
|
|
2650
2852
|
if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && (out.runs ?? 1) > limit.runs) {
|
|
2651
|
-
|
|
2853
|
+
const was = out.bestOf !== void 0 ? `best of ${out.bestOf}` : String(out.runs);
|
|
2854
|
+
notes.push(`runs ${was} → ${limit.runs} (${tag})`);
|
|
2652
2855
|
out.runs = limit.runs;
|
|
2856
|
+
out.bestOf = void 0;
|
|
2653
2857
|
out.sources = {
|
|
2654
2858
|
...out.sources,
|
|
2655
2859
|
runs: `profiles.${profile.name}.limit`
|
|
@@ -2678,6 +2882,16 @@ function cascade(base, levels, where, options = {}) {
|
|
|
2678
2882
|
if (narrowed) notes.push(`${plural(narrowed)} narrowed (profile ${profile.name})`);
|
|
2679
2883
|
out.questions = kept;
|
|
2680
2884
|
}
|
|
2885
|
+
if (options.runs !== void 0) out.bestOf = void 0;
|
|
2886
|
+
if (out.bestOf !== void 0 && out.compare?.variants.length) {
|
|
2887
|
+
const from = out.sources?.runs ?? "the config";
|
|
2888
|
+
const set = from === where ? "" : ` from ${from}`;
|
|
2889
|
+
throw new UsageError(`${where} compares environments, so it can't take runs: { bestOf: ${out.bestOf} }${set}: a lift needs the same number of attempts on each side, and best-of stops each cell on its own. Set a fixed count on it, e.g. runs: 4.`);
|
|
2890
|
+
}
|
|
2891
|
+
if (out.bestOf !== void 0) {
|
|
2892
|
+
if (out.gate !== void 0 && gateLabel(out.gate) !== gateLabel(MAJORITY_GATE)) notes.push(`gate ${gateLabel(out.gate)} → majority (runs.bestOf)`);
|
|
2893
|
+
out.gate = MAJORITY_GATE;
|
|
2894
|
+
}
|
|
2681
2895
|
defaultRuns(out, notes, options);
|
|
2682
2896
|
return {
|
|
2683
2897
|
resolved: {
|
|
@@ -2813,6 +3027,7 @@ function expand(resolved, vertex) {
|
|
|
2813
3027
|
egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers, onVertex),
|
|
2814
3028
|
gate,
|
|
2815
3029
|
...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
|
|
3030
|
+
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
2816
3031
|
n,
|
|
2817
3032
|
label,
|
|
2818
3033
|
...onVertex ? { vertex: onVertex } : {}
|
|
@@ -2941,6 +3156,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
2941
3156
|
} } : {},
|
|
2942
3157
|
...!skip && resolved.questions.length ? { questions: resolved.questions } : {},
|
|
2943
3158
|
runs: resolved.runs,
|
|
3159
|
+
...resolved.bestOf !== void 0 ? { bestOf: resolved.bestOf } : {},
|
|
2944
3160
|
...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
|
|
2945
3161
|
attempts: skip ? 0 : attempts.length,
|
|
2946
3162
|
notes,
|
|
@@ -2964,7 +3180,7 @@ function evalPlan(r, attempts, origin) {
|
|
|
2964
3180
|
/** A pairwise question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
|
|
2965
3181
|
function questionWarnings(resolved) {
|
|
2966
3182
|
if (resolved.harness.length < 2) return [];
|
|
2967
|
-
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows
|
|
3183
|
+
return resolved.questions.filter((q) => pairFactors(q.across).includes("harness") && q.state?.transcript).map((q) => `question "${q.question}" shows the transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
|
|
2968
3184
|
}
|
|
2969
3185
|
function captureNote(c) {
|
|
2970
3186
|
const parts = [];
|
|
@@ -3157,6 +3373,7 @@ const CONFIG_KEYS = [
|
|
|
3157
3373
|
"capture",
|
|
3158
3374
|
"question",
|
|
3159
3375
|
"timeout",
|
|
3376
|
+
"snapshot",
|
|
3160
3377
|
"concurrency",
|
|
3161
3378
|
"record",
|
|
3162
3379
|
"history",
|
|
@@ -3187,7 +3404,8 @@ const DEFAULTS_KEYS = [
|
|
|
3187
3404
|
"judge",
|
|
3188
3405
|
"capture",
|
|
3189
3406
|
"question",
|
|
3190
|
-
"timeout"
|
|
3407
|
+
"timeout",
|
|
3408
|
+
"snapshot"
|
|
3191
3409
|
];
|
|
3192
3410
|
const LIMIT_KEYS = [
|
|
3193
3411
|
"runs",
|
|
@@ -3543,7 +3761,7 @@ function selectProfile(config, flag, env = process.env) {
|
|
|
3543
3761
|
function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
3544
3762
|
const user = withHostPaths(input, root, file);
|
|
3545
3763
|
if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
|
|
3546
|
-
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model
|
|
3764
|
+
if ("questions" in user) throw new UsageError(`questions can't be set in ${file}: each is about one eval's task. Put them on the group or eval they ask about; question: { model } in ${file} sets every question's model.`);
|
|
3547
3765
|
checkOptions(user, file, CONFIG_KEYS);
|
|
3548
3766
|
const include = user.include ?? defaultIncludeGlobs();
|
|
3549
3767
|
if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
|
|
@@ -3588,6 +3806,10 @@ function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
|
|
|
3588
3806
|
capture: user.capture ?? {},
|
|
3589
3807
|
question: user.question ?? {},
|
|
3590
3808
|
timeoutMs: parseDuration(user.timeout ?? "15m"),
|
|
3809
|
+
snapshot: mergeSnapshot({
|
|
3810
|
+
when: true,
|
|
3811
|
+
compress: false
|
|
3812
|
+
}, user.snapshot),
|
|
3591
3813
|
concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
|
|
3592
3814
|
record,
|
|
3593
3815
|
history: {
|
|
@@ -3772,6 +3994,6 @@ function malformedCredentials(config, env = process.env, options = {}) {
|
|
|
3772
3994
|
});
|
|
3773
3995
|
}
|
|
3774
3996
|
//#endregion
|
|
3775
|
-
export {
|
|
3997
|
+
export { egressLabel as $, harnessId as A, NO_JUDGE_INPUT as At, sandboxId as B, decideState as Bt, routedHarness as C, isAgentKind as Ct, checkOptions as D, writePluginFiles as Dt, checkLimitHarness as E, refusedPlugins as Et, plainHeaders as F, judgeProblem as Ft, unbrokeredError as G, sandboxSetBy as H, resolveJudgeArtifacts as Ht, plannedSandboxes as I, sharedInputFields as It, DEFAULT_EGRESS as J, mergeJudge as K, profileJudge as L, turnsSchema as Lt, majorityOf as M, inputsOn as Mt, matchesTags as N, judgeFields as Nt, evalPlan as O, captureWindowSchema as Ot, plainHeaderError as P, judgeModelNameSchema as Pt, directOnCloud as Q, refuseIllegalCompare as R, artifactRefsOf as Rt, resolveGateway as S, expectedBlockedFor as St, cascade as T, refusedMcp as Tt, secretVarsIn as U, sandboxProvider as V, filePartsOf as Vt, unbrokered as W, allowedHosts as X, DEFAULT_EGRESS_ALLOW as Y, ciRunner as Z, fellBackToDirect as _, agentMcp as _t, configPathFor as a, awaitListening as at, resolveModelGateway as b, claudeMcpServers as bt, findConfig as c, STOPPED_FIRST as ct, malformedCredentials as d, resolveQuestionState as dt, gatewayPolicy as et, missingCredentials as f, harnessEffort as ft, selectProfile as g, agentFor as gt, runJudgeModel as h, agentAuth as ht, configLabel as i, createFakeSandbox as it, levelOf as j, fakeJudgeDataSchema as jt, expand as k, INPUT_KEYS as kt, hostCapacity as l, UNRECORDED_QUESTION_STATE as lt, resolveConcurrency as m, servedEffortOf as mt, analysisModelOf as n, unenforceable as nt, defaultConcurrency as o, parseProbe as ot, requireConfig as p, resolvedEffortOf as pt, resolveStateOptions as q, configFilesPhrase as r, resolveOutput as rt, defineConfig as s, probeScript as st, CONFIG_FILES as t, isDirect as tt, loadConfig as u, questionShows as ut, gatewayScope as v, authHint as vt, brokeredIn as w, layoutOf as wt, fallbackHint as x, credentialOf as xt, modelGatewayOrNone as y, brokeringFor as yt, runsOf as z, badJudgeArtifacts as zt };
|
|
3776
3998
|
|
|
3777
|
-
//# sourceMappingURL=config-
|
|
3999
|
+
//# sourceMappingURL=config-CneBZB84.js.map
|