reeval 0.15.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/{adapter-CbqbMR7y.js → adapter-CRKYPkRt.js} +3 -3
- package/dist/{adapter-CbqbMR7y.js.map → adapter-CRKYPkRt.js.map} +1 -1
- package/dist/{agent-Zgn-kiqq.js → agent-BgH4-kFc.js} +6 -6
- package/dist/{agent-Zgn-kiqq.js.map → agent-BgH4-kFc.js.map} +1 -1
- package/dist/{app-files-ChbvRx3P.js → app-files-BMVBr079.js} +3 -3
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-SKQGW_QF.js → assemble-BVbI6NW-.js} +95 -29
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-Ba2QerOK.js → attempt-CpbZ9arV.js} +2 -2
- package/dist/{attempt-Ba2QerOK.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{attempt-host-BOp8JklD.js → attempt-host-CIxjs2Be.js} +13 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-BM_3100g.js → capture-iQ6kYJV0.js} +987 -161
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-PdYrcoKM.js → cli-1YQU-66F.js} +98 -61
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-D0qyrPZi.js → cli-spec-CPWT1mAG.js} +3 -2
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-Ct6s6OnF.js → client-DMmOkbRA.js} +13 -6
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-4pJYVCT4.js → command-8_Cg5v6b.js} +12 -12
- package/dist/{command-4pJYVCT4.js.map → command-8_Cg5v6b.js.map} +1 -1
- package/dist/{command-DbSMcxZL.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-DbSMcxZL.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{command-9xm5ImqI.js → command-C-HeGlap.js} +16 -16
- package/dist/{command-9xm5ImqI.js.map → command-C-HeGlap.js.map} +1 -1
- package/dist/{command-BqBxLgTc.js → command-Cr1zjRvl.js} +13 -13
- package/dist/{command-BqBxLgTc.js.map → command-Cr1zjRvl.js.map} +1 -1
- package/dist/{commands-Bi2SEQtr.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-BU6YJCz9.js → commands-BvTNZ2-b.js} +14 -9
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-k6m7XbAz.js → config-CneBZB84.js} +430 -208
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DLZF4dGF.js → contrasts-lhFsekco.js} +2 -2
- package/dist/{contrasts-DLZF4dGF.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/{db-CpsifVln.js → db-Cnh3jIv_.js} +8 -8
- package/dist/{db-CpsifVln.js.map → db-Cnh3jIv_.js.map} +1 -1
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-BhizgAsU.js → debug-CgG3T071.js} +4 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-51WvhN_X.js → docker-BihzJwbV.js} +9 -5
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-B4-rx4dA.js → docker-Cqq03H72.js} +5 -5
- package/dist/{docker-B4-rx4dA.js.map → docker-Cqq03H72.js.map} +1 -1
- package/dist/{docker-YFj4H1Cr.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-BuQnfz4e.js → doctor-Dl4vMeqt.js} +15 -15
- package/dist/{doctor-BuQnfz4e.js.map → doctor-Dl4vMeqt.js.map} +1 -1
- package/dist/{environment-BOFgzFDg.js → environment-Cf5HMjib.js} +3 -3
- package/dist/{environment-BOFgzFDg.js.map → environment-Cf5HMjib.js.map} +1 -1
- package/dist/{errors-UjcLWelF.js → errors-CHebohjc.js} +9 -3
- package/dist/{errors-UjcLWelF.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/{eval-source-Dje3rERU.js → eval-source-CAE4y2Sr.js} +9 -8
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BPP9s3s6.js → evaluate-CUtgyr43.js} +340 -45
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/{evaluate-dqXccn4k.d.ts → evaluate-D35T51AC.d.ts} +7 -6
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-Bi461Be2.js → files-BRViE7m-.js} +20 -13
- package/dist/{files-Bi461Be2.js.map → files-BRViE7m-.js.map} +1 -1
- package/dist/{files-H24pz0hL.js → files-BSqA2sZE.js} +3 -3
- package/dist/{files-H24pz0hL.js.map → files-BSqA2sZE.js.map} +1 -1
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/{files-Clipx4OW.js → files-CNiJ_fZf.js} +3 -3
- package/dist/{files-Clipx4OW.js.map → files-CNiJ_fZf.js.map} +1 -1
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-puYi-zj2.js → gate-BC6SdQZs.js} +49 -2
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-CsVZkNNL.js → git-UNxT_3z0.js} +2 -2
- package/dist/{git-CsVZkNNL.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/{images-4mwhNkvh.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-4mwhNkvh.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +63 -16
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-cOk3D44b.js → init-sFBgODhu.js} +4 -4
- package/dist/{init-cOk3D44b.js.map → init-sFBgODhu.js.map} +1 -1
- package/dist/{invocations-DW-lTk2f.js → invocations-C9ejlOYl.js} +2 -2
- package/dist/{invocations-DW-lTk2f.js.map → invocations-C9ejlOYl.js.map} +1 -1
- package/dist/{judge-CKdgEXN7.js → judge-Bg4v4e_K.js} +2 -2
- package/dist/{judge-CKdgEXN7.js.map → judge-Bg4v4e_K.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-XJQ6nqK4.js → live-DeMvnDD5.js} +2 -2
- package/dist/{live-XJQ6nqK4.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-DT2TaGix.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-Bq3G0OG1.js → model-2rBZrdZA.js} +4 -4
- package/dist/{model-Bq3G0OG1.js.map → model-2rBZrdZA.js.map} +1 -1
- package/dist/{model-vRiUycWh.js → model-3MJtpho8.js} +4 -4
- package/dist/{model-vRiUycWh.js.map → model-3MJtpho8.js.map} +1 -1
- package/dist/{model-CTnyvh7H.js → model-Bz7-j71_.js} +3 -50
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/{model-g3XnzUv0.js → model-DkPwypN4.js} +1 -1
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BVbbnFtv.js → node-fP-H2uxn.js} +2 -2
- package/dist/{node-BVbbnFtv.js.map → node-fP-H2uxn.js.map} +1 -1
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/{openrouter-models-CWRwLX-G.js → openrouter-models-Bo8PZ4Ew.js} +2 -2
- package/dist/{openrouter-models-CWRwLX-G.js.map → openrouter-models-Bo8PZ4Ew.js.map} +1 -1
- package/dist/{patch-DKJiC5Sq.js → patch-6T9XAipm.js} +4 -4
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/{plan-DdWkM6SW.js → plan-Dw2BjafJ.js} +20 -11
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-D-CP6mMd.js → plugins-ngQEJBjJ.js} +24 -3
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-DTiVv0gY.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-9ixuXzsx.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-9ixuXzsx.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-tmBBf4W8.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-BfyikQit.js → prepare-PkARUGoF.js} +7 -7
- package/dist/{prepare-BfyikQit.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{provenance-BizwUn3h.js → provenance-7Hzg88f4.js} +2 -2
- package/dist/{provenance-BizwUn3h.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-DkOupEBl.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-B6b9Tnyi.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-B6b9Tnyi.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-DyM-VExS.js → query-nBXrgEwl.js} +2 -2
- package/dist/{query-DyM-VExS.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-BRhU8MzB.js → questions-CkAwePPR.js} +16 -17
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{remarks-C7i2LMRb.js → remarks-Bt0zwiQB.js} +42 -8
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{remarks-D_3qhBWU.js → remarks-IFFe_oty.js} +4 -3
- package/dist/{remarks-D_3qhBWU.js.map → remarks-IFFe_oty.js.map} +1 -1
- package/dist/{remarks-dA8LyoPc.js → remarks-eBcrIZRu.js} +3 -2
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-DFIzcx_6.js → remote-DfJYiP-R.js} +5 -5
- package/dist/{remote-DFIzcx_6.js.map → remote-DfJYiP-R.js.map} +1 -1
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-DWSJpKHX.js → resolve-D_cLa4tW.js} +5 -5
- package/dist/{resolve-DWSJpKHX.js.map → resolve-D_cLa4tW.js.map} +1 -1
- package/dist/{results-BC3pcG_Z.js → results-DrKJuIQF.js} +9 -9
- package/dist/{results-BC3pcG_Z.js.map → results-DrKJuIQF.js.map} +1 -1
- package/dist/{resume-j-rcIryi.js → resume-BmIM52MJ.js} +4 -4
- package/dist/{resume-j-rcIryi.js.map → resume-BmIM52MJ.js.map} +1 -1
- package/dist/{runtime-d56Gg3J5.js → runtime-CZVpkr04.js} +111 -37
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-Cmln11gU.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-Cmln11gU.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-BOoH10J8.js → server-DS8BGhGU.js} +34 -10
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/{source-wD04-rRG.js → source-DcXv4de-.js} +17 -3
- package/dist/{source-wD04-rRG.js.map → source-DcXv4de-.js.map} +1 -1
- package/dist/{src-DgHuStEi.js → src-A8HLtO0a.js} +2 -2
- package/dist/{src-DgHuStEi.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-kYeaJJrl.js → src-BTqJbvj6.js} +1 -1
- package/dist/{src-Bp4aN3rA.js → src-BkiComGh.js} +1 -1
- package/dist/{src-dlX-Lmxg.js → src-BvgeZ4tS.js} +1 -1
- package/dist/{src-D91Hd6hT.js → src-C0ySiej5.js} +36 -17
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/{src-HSxBazcg.js → src-CgJrz5p0.js} +1 -1
- package/dist/{src-BYPMM1x7.js → src-Cp-2Hdib.js} +1 -1
- package/dist/{src-6_zz7oJs.js → src-DMo2eCOJ.js} +1 -1
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/{src-C7upnZSd.js → src-OESRtvOi.js} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-DrJ687ut.js → state-C_kdhp5v.js} +5 -5
- package/dist/{state-DrJ687ut.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-Co8cwelb.js → suite-DieLXvCT.js} +15 -4
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DLFHVoux.js → summary-CSzyBVsF.js} +40 -14
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +52 -44
- package/dist/testing.js +4 -4
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-DxPaWHTZ.js → transcript-BJrKfgRW.js} +2 -2
- package/dist/{transcript-DxPaWHTZ.js.map → transcript-BJrKfgRW.js.map} +1 -1
- package/dist/{types-coMWHaU2.d.ts → types-DVvW66ci.d.ts} +325 -171
- package/dist/view/index.d.ts +16 -4
- package/dist/view/index.js +6 -6
- package/dist/{views-BFO1BLKu.js → views-B9STsLB4.js} +5 -5
- package/dist/{views-BFO1BLKu.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-CJtdsKBP.js → worker-entry-C5bLCv6s.js} +34 -13
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-BRxjfa8N.js → workspace-CBuI9Uds.js} +4 -4
- package/dist/{workspace-BRxjfa8N.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/package.json +1 -1
- package/skills/eval-rubric/SKILL.md +11 -0
- package/dist/app-files-ChbvRx3P.js.map +0 -1
- package/dist/assemble-SKQGW_QF.js.map +0 -1
- package/dist/assemble-XBWuK3Ed.js +0 -1
- package/dist/attempt-host-BOp8JklD.js.map +0 -1
- package/dist/capture-BM_3100g.js.map +0 -1
- package/dist/capture-DSK8O5bA.js +0 -1
- package/dist/cli-PdYrcoKM.js.map +0 -1
- package/dist/cli-spec-D0qyrPZi.js.map +0 -1
- package/dist/client-Ct6s6OnF.js.map +0 -1
- package/dist/commands-BU6YJCz9.js.map +0 -1
- package/dist/config-k6m7XbAz.js.map +0 -1
- package/dist/db-DkD-F-X-.js +0 -2
- package/dist/debug-BhizgAsU.js.map +0 -1
- package/dist/debug-DeM5pswK.js +0 -1
- package/dist/docker-51WvhN_X.js.map +0 -1
- package/dist/eval-source-Dje3rERU.js.map +0 -1
- package/dist/evaluate-BPP9s3s6.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-C2h2Cw4F.js +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-DmUrRmmL.js +0 -1
- package/dist/init-BDEOopK7.js +0 -1
- package/dist/matchers-DT2TaGix.js.map +0 -1
- package/dist/model-CTnyvh7H.js.map +0 -1
- package/dist/model-D-SWrLZG.js +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/pairs-puYi-zj2.js.map +0 -1
- package/dist/patch-DKJiC5Sq.js.map +0 -1
- package/dist/plan-DdWkM6SW.js.map +0 -1
- package/dist/plugins-D-CP6mMd.js.map +0 -1
- package/dist/query-Clm2mNSm.js +0 -1
- package/dist/questions-BRhU8MzB.js.map +0 -1
- package/dist/remarks-BwJ8pcTQ.js +0 -1
- package/dist/remarks-C7i2LMRb.js.map +0 -1
- package/dist/remarks-dA8LyoPc.js.map +0 -1
- package/dist/resolve--rQZ8u6Z.js +0 -1
- package/dist/runtime-d56Gg3J5.js.map +0 -1
- package/dist/server-BOoH10J8.js.map +0 -1
- package/dist/server-DpuVAbrO.js +0 -1
- package/dist/source-BFd6f0-R.js +0 -1
- package/dist/src-D91Hd6hT.js.map +0 -1
- package/dist/src-DH-9IO7f.js +0 -1
- package/dist/src-LD1BAJFa.js +0 -2
- package/dist/state-B5PLm5qY.js +0 -1
- package/dist/suite-22Kt9Grx.js +0 -2
- package/dist/suite-Co8cwelb.js.map +0 -1
- package/dist/summary-DLFHVoux.js.map +0 -1
- package/dist/views-BOf8qEL8.js +0 -2
- package/dist/worker-entry-CJtdsKBP.js.map +0 -1
- package/dist/workspace-DDHzkMjm.js +0 -1
|
@@ -187,7 +187,7 @@ type FileRef = {
|
|
|
187
187
|
sha256: string;
|
|
188
188
|
bytes: number;
|
|
189
189
|
};
|
|
190
|
-
/** A generic attempt file from `c.capture()
|
|
190
|
+
/** A generic attempt file from `c.capture()`. */
|
|
191
191
|
type ArtifactRef = FileRef & {
|
|
192
192
|
name: string;
|
|
193
193
|
mediaType: 'image/png';
|
|
@@ -217,15 +217,18 @@ type GitInfo = {
|
|
|
217
217
|
*/
|
|
218
218
|
sha: string;
|
|
219
219
|
branch: string | null;
|
|
220
|
-
/** Uncommitted changes, outside `.reeval/` and the history folder, when the run started. */
|
|
220
|
+
/** Uncommitted changes, outside the `.reeval/` directory and the history folder, when the run started. */
|
|
221
221
|
dirty: boolean;
|
|
222
222
|
/** sha256 of the uncommitted changes (tracked and untracked, as a binary patch), when dirty. */
|
|
223
223
|
diffHash: string | null;
|
|
224
|
-
/**
|
|
224
|
+
/**
|
|
225
|
+
* Those changes are stored as the `worktree.patch` file in the artifact directory. Applying it
|
|
226
|
+
* to `sha` reproduces the workspace.
|
|
227
|
+
*/
|
|
225
228
|
worktreePatch: FileRef | null;
|
|
226
229
|
/**
|
|
227
|
-
*
|
|
228
|
-
* a secret (then where, and which rule, never the value).
|
|
230
|
+
* This explains why a dirty run has no `worktree.patch` file: it was too large, `artifacts` was
|
|
231
|
+
* `'none'`, or it looked like it held a secret (then where, and which rule, never the value).
|
|
229
232
|
*/
|
|
230
233
|
worktreePatchNote: string | null;
|
|
231
234
|
commitTime: string;
|
|
@@ -267,7 +270,7 @@ type JudgmentRecord = {
|
|
|
267
270
|
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
268
271
|
costUsd?: number;
|
|
269
272
|
/**
|
|
270
|
-
* `<index>:<sha256(question) prefix>`: the same question at the same place in the
|
|
273
|
+
* `<index>:<sha256(question) prefix>`: the same question at the same place in the evaluation, so a
|
|
271
274
|
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
272
275
|
* is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
|
|
273
276
|
* whichever finishes first.
|
|
@@ -279,14 +282,14 @@ type JudgmentRecord = {
|
|
|
279
282
|
artifacts?: JudgeArtifactRef[];
|
|
280
283
|
};
|
|
281
284
|
type AttemptFiles = {
|
|
282
|
-
/** Each environment setup step's command, exit code, duration and output. */
|
|
285
|
+
/** Each environment setup step's command, exit code, duration, and output. */
|
|
283
286
|
setupLog?: FileRef;
|
|
284
287
|
transcript?: FileRef;
|
|
285
288
|
/** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
|
|
286
289
|
rawTranscript?: FileRef;
|
|
287
290
|
diff?: FileRef;
|
|
288
291
|
judgeInput?: FileRef;
|
|
289
|
-
/** PNGs from `c.capture()`, under `artifacts/<name>.png
|
|
292
|
+
/** PNGs from `c.capture()`, under the `artifacts/<name>.png` path. */
|
|
290
293
|
artifacts?: ArtifactRef[];
|
|
291
294
|
};
|
|
292
295
|
type ContextFile = {
|
|
@@ -301,16 +304,22 @@ type UpstreamCount = {
|
|
|
301
304
|
type AttemptRecord = {
|
|
302
305
|
/** `<eval key> # <cell key>#<n>`: unique in the run. */
|
|
303
306
|
key: string;
|
|
304
|
-
/** `<cell key>#<n>`: unique in its
|
|
307
|
+
/** `<cell key>#<n>`: unique in its evaluation. */
|
|
305
308
|
id: string;
|
|
306
309
|
/** The cell's key; see `EvalEntry.cells`. */
|
|
307
310
|
cell: string;
|
|
308
311
|
/** Run number within the cell, from 1. */
|
|
309
312
|
n: number;
|
|
313
|
+
/**
|
|
314
|
+
* In a `runs: { bestOf }` cell: `added` when it ran past the majority's first attempts (the
|
|
315
|
+
* earlier ones split, or an infra error left them short), `skipped` when it didn't run because
|
|
316
|
+
* the earlier ones already had a majority. Unset on the attempts that always run.
|
|
317
|
+
*/
|
|
318
|
+
bestOf?: 'added' | 'skipped';
|
|
310
319
|
/** Start order across the run, from 1; null for attempts that never started. */
|
|
311
320
|
seq: number | null;
|
|
312
321
|
startedAt: string | null;
|
|
313
|
-
/** The image the container started from, in `stack.images`; null
|
|
322
|
+
/** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
|
|
314
323
|
imageId: string | null;
|
|
315
324
|
/** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
|
|
316
325
|
env: {
|
|
@@ -318,16 +327,17 @@ type AttemptRecord = {
|
|
|
318
327
|
secrets: string[];
|
|
319
328
|
brokered: string[];
|
|
320
329
|
};
|
|
321
|
-
/** Skills, rules and MCP config visible to the agent in its sandbox, by content hash. */
|
|
330
|
+
/** Skills, rules, and MCP config visible to the agent in its sandbox, by content hash. */
|
|
322
331
|
context: ContextFile[];
|
|
323
332
|
/** The egress policy and, when the sandbox records it, the hosts contacted and refused; null if it never ran. */
|
|
324
333
|
egress: EgressRecord | null;
|
|
325
334
|
/**
|
|
326
|
-
* The models the harness's
|
|
327
|
-
*
|
|
328
|
-
*
|
|
329
|
-
*
|
|
330
|
-
*
|
|
335
|
+
* The models served by the harness's API calls, as observed by the egress gateway. Values may
|
|
336
|
+
* come from requests or responses and include reported effort; null when no model was named or
|
|
337
|
+
* recorded. The gateway reads the request URL for Antigravity, Goose, pi, and OpenCode on a
|
|
338
|
+
* `google/` model, so a request for `<model>-low` records `<model>`, and the API response for
|
|
339
|
+
* Codex, Grok Build, and OpenCode on another provider. Null for Cursor and Claude Code, whose
|
|
340
|
+
* API requests name no model.
|
|
331
341
|
*/
|
|
332
342
|
servedModels: string[] | null;
|
|
333
343
|
/**
|
|
@@ -353,7 +363,7 @@ type AttemptRecord = {
|
|
|
353
363
|
*/
|
|
354
364
|
effort: string | null;
|
|
355
365
|
/**
|
|
356
|
-
* The reasoning effort the harness's API said it ran at, from `servedModels`
|
|
366
|
+
* The reasoning effort the harness's API said it ran at, from `servedModels` or
|
|
357
367
|
* each turn's `model` (`medium` from `gpt-5.6-luna[effort=medium]`); null when those
|
|
358
368
|
* labels name none. Fast (`[fast=true]`) is not this.
|
|
359
369
|
*/
|
|
@@ -362,7 +372,8 @@ type AttemptRecord = {
|
|
|
362
372
|
error: AttemptError | null;
|
|
363
373
|
durationMs: number;
|
|
364
374
|
/**
|
|
365
|
-
* `text` is the agent's final message, cut at 8 KB; the whole of it is in `transcript.jsonl
|
|
375
|
+
* `text` is the agent's final message, cut at 8 KB; the whole of it is in the `transcript.jsonl`
|
|
376
|
+
* file.
|
|
366
377
|
* `mcpStartup` times each stdio MCP server's start in the turn (`McpStartup`).
|
|
367
378
|
*/
|
|
368
379
|
turns: Array<{
|
|
@@ -385,8 +396,8 @@ type AttemptRecord = {
|
|
|
385
396
|
/** Tokens across the attempt's turns; null when the harness reported none. */
|
|
386
397
|
usage: TokenUsage | null;
|
|
387
398
|
/**
|
|
388
|
-
* Tokens the capture vision loop used. Infrastructure only: never
|
|
389
|
-
*
|
|
399
|
+
* Tokens the capture vision loop used. Infrastructure only: never included
|
|
400
|
+
* in `usage`, effort, compare, or lift.
|
|
390
401
|
*/
|
|
391
402
|
captureUse?: {
|
|
392
403
|
model: string;
|
|
@@ -394,7 +405,7 @@ type AttemptRecord = {
|
|
|
394
405
|
usage: TokenUsage | null;
|
|
395
406
|
};
|
|
396
407
|
timings: PhaseTimings;
|
|
397
|
-
/** Counts from the transcript, so
|
|
408
|
+
/** Counts from the transcript, so the attempt's activity is available without reading the transcript. `webSearches` includes refused searches. */
|
|
398
409
|
activity: {
|
|
399
410
|
tools: number;
|
|
400
411
|
fileEdits: number;
|
|
@@ -419,6 +430,22 @@ type AttemptRecord = {
|
|
|
419
430
|
* names. Absent when there were none.
|
|
420
431
|
*/
|
|
421
432
|
setupProcesses?: SetupProcess[];
|
|
433
|
+
/**
|
|
434
|
+
* Copies of the attempt's workspace in its directory in `.reeval/runs` on the machine that ran
|
|
435
|
+
* it (never in the history folder), in the order taken: `turn-<n>` after each `c.send` with
|
|
436
|
+
* `snapshot: 'turns'`, then `final` after the checks. Each has its form, path and size, its
|
|
437
|
+
* file count, the directories left out by name, the files whose secret-looking values were
|
|
438
|
+
* masked and the binary ones left out for holding one, or why it wasn't taken. Absent with
|
|
439
|
+
* `snapshot: false`, and when the sandbox was gone first (a timeout or a cancel).
|
|
440
|
+
*/
|
|
441
|
+
snapshots?: Snapshot[];
|
|
442
|
+
/**
|
|
443
|
+
* Each `c.serve` call that listened, in call order: `{ command, port, portEnv, detected,
|
|
444
|
+
* address, readyMs }`. `port` is where it listens; `detected` is true when that isn't the
|
|
445
|
+
* `PORT` it was given (`portEnv`), so reeval took the one port that started listening after
|
|
446
|
+
* the command did. Absent when the attempt served nothing.
|
|
447
|
+
*/
|
|
448
|
+
serves?: ServeRecord[];
|
|
422
449
|
/**
|
|
423
450
|
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
424
451
|
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
@@ -433,9 +460,9 @@ type AttemptRecord = {
|
|
|
433
460
|
/** Every try, in order, when it was given `retries`; its log has each try's output. */
|
|
434
461
|
tries?: ExecTry[];
|
|
435
462
|
}>;
|
|
436
|
-
/** Every `expect` and `expect.
|
|
463
|
+
/** Every `expect` and `expect.require` matcher call that settled, in call order; unset with none. */
|
|
437
464
|
assertions?: AssertionRecord[];
|
|
438
|
-
/** Files under a dependency directory
|
|
465
|
+
/** Files under a dependency directory, such as the `node_modules/` directory or `.venv/` directory, collapse to one entry with `count`. */
|
|
439
466
|
changedFiles: Array<FileChange & {
|
|
440
467
|
count?: number;
|
|
441
468
|
}>;
|
|
@@ -451,7 +478,7 @@ type AttemptRecord = {
|
|
|
451
478
|
type EvalEntry = {
|
|
452
479
|
/** `file › group › … › name`, as shown. */
|
|
453
480
|
id: string;
|
|
454
|
-
/**
|
|
481
|
+
/** Used to match an evaluation across runs; equal to `id`. */
|
|
455
482
|
key: string;
|
|
456
483
|
/** Directory name inside the artifact directory, from `key`. */
|
|
457
484
|
slug: string;
|
|
@@ -461,8 +488,8 @@ type EvalEntry = {
|
|
|
461
488
|
/** Results are comparable only when fingerprints match. */
|
|
462
489
|
fingerprint: string;
|
|
463
490
|
/**
|
|
464
|
-
* The
|
|
465
|
-
* the local files it imported, and were fingerprinted over them.
|
|
491
|
+
* The evaluation file, by content hash: where the evaluation lives. Records from before `source`
|
|
492
|
+
* also list the local files it imported, and were fingerprinted over them.
|
|
466
493
|
*/
|
|
467
494
|
sources: Array<{
|
|
468
495
|
path: string;
|
|
@@ -476,12 +503,12 @@ type EvalEntry = {
|
|
|
476
503
|
/** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
|
|
477
504
|
revision?: Revision;
|
|
478
505
|
tags: string[];
|
|
479
|
-
/** The baseline environment, when the
|
|
506
|
+
/** The baseline environment, when the evaluation uses `compare`. */
|
|
480
507
|
compare?: EvalCompareMeta;
|
|
481
508
|
/**
|
|
482
|
-
* Over gated cells
|
|
483
|
-
* only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
484
|
-
* failures don't fail the
|
|
509
|
+
* Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
|
|
510
|
+
* `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
511
|
+
* cell's failures don't fail the evaluation.
|
|
485
512
|
*/
|
|
486
513
|
outcome: Outcome;
|
|
487
514
|
/** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
|
|
@@ -494,9 +521,9 @@ type EvalEntry = {
|
|
|
494
521
|
* The eval's pairwise questions (those with `across`), answered after its attempts by votes,
|
|
495
522
|
* one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
496
523
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
497
|
-
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
498
|
-
* `<eval-slug>/questions/<n>.json
|
|
499
|
-
* masked and screenshots as refs; unset where the
|
|
524
|
+
* (rates by cut, votes, ties, a failed call, and a sample of reasons). Each answer's `input` is
|
|
525
|
+
* the `<eval-slug>/questions/<n>.json` file: every call its model was sent, per pair and order,
|
|
526
|
+
* with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
|
|
500
527
|
*/
|
|
501
528
|
answers?: CompareAnswer[];
|
|
502
529
|
/** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
|
|
@@ -513,7 +540,7 @@ type FamilyInterval = {
|
|
|
513
540
|
hi: number;
|
|
514
541
|
level: 0.95;
|
|
515
542
|
/**
|
|
516
|
-
* `wilson`
|
|
543
|
+
* `wilson` or `newcombe-paired`: one counted attempt a case and side, so each case is one
|
|
517
544
|
* Bernoulli trial (or pair). `t-cases`: several, so a t interval over the cases' pass@1, with
|
|
518
545
|
* attempts within a case never counted as independent.
|
|
519
546
|
*/
|
|
@@ -588,17 +615,17 @@ type FamilyLift = {
|
|
|
588
615
|
outcome: 'pass' | 'fail';
|
|
589
616
|
effort?: FamilyEffort;
|
|
590
617
|
};
|
|
591
|
-
/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an
|
|
618
|
+
/** A pooled `evaluate.each`: its cases as samples of one suite. Each case is also an evaluation. */
|
|
592
619
|
type FamilyRecord = {
|
|
593
620
|
/** `file › group › … › template`. */
|
|
594
621
|
id: string;
|
|
595
622
|
file: string;
|
|
596
623
|
group: string[];
|
|
597
|
-
/** The name template,
|
|
624
|
+
/** The name template, for example `fixes $id`. */
|
|
598
625
|
name: string;
|
|
599
626
|
/** Cases in the list. */
|
|
600
627
|
cases: number;
|
|
601
|
-
/** Cases with an
|
|
628
|
+
/** Cases with an evaluation in this run; fewer when a filter or tag left some out. */
|
|
602
629
|
ran: number;
|
|
603
630
|
/** One per cell. */
|
|
604
631
|
rates: FamilyRate[];
|
|
@@ -643,7 +670,7 @@ type FingerprintParts = {
|
|
|
643
670
|
environment: string | null;
|
|
644
671
|
image: string | null;
|
|
645
672
|
agentCli: string | null;
|
|
646
|
-
/** Over the judge's model, threshold and version
|
|
673
|
+
/** Over the judge's model, threshold and version. */
|
|
647
674
|
judge: string;
|
|
648
675
|
/** Over the attempt's context files. */
|
|
649
676
|
context: string;
|
|
@@ -655,7 +682,7 @@ type FingerprintParts = {
|
|
|
655
682
|
*/
|
|
656
683
|
gate?: string;
|
|
657
684
|
};
|
|
658
|
-
/** One harness and environment of an
|
|
685
|
+
/** One harness and environment of an evaluation, and how its attempts went. */
|
|
659
686
|
type CellRecord = Cell & {
|
|
660
687
|
/** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
|
|
661
688
|
fingerprint: string;
|
|
@@ -678,10 +705,15 @@ type CellRecord = Cell & {
|
|
|
678
705
|
* Fast is not this.
|
|
679
706
|
*/
|
|
680
707
|
effort: string | null;
|
|
681
|
-
/** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the
|
|
708
|
+
/** How the attempts decide the cell's outcome; `'none'` when it doesn't fail the evaluation. */
|
|
682
709
|
gate: CellGate;
|
|
683
710
|
/** Environment keys the variant replaced on the control's (or a child on its parent's). */
|
|
684
711
|
overrides: string[];
|
|
712
|
+
/**
|
|
713
|
+
* Set by `runs: { bestOf }`: the cell stopped once a majority of this many attempts agreed, and
|
|
714
|
+
* gates on that majority. `runs` still counts every planned attempt, the skipped ones included.
|
|
715
|
+
*/
|
|
716
|
+
bestOf?: number;
|
|
685
717
|
runs: number;
|
|
686
718
|
pass: number;
|
|
687
719
|
fail: number;
|
|
@@ -697,7 +729,7 @@ type CellRecord = Cell & {
|
|
|
697
729
|
outcome: Outcome;
|
|
698
730
|
lift?: Lift;
|
|
699
731
|
/**
|
|
700
|
-
* Each
|
|
732
|
+
* Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
|
|
701
733
|
* attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
|
|
702
734
|
*/
|
|
703
735
|
checks?: CellCheck[];
|
|
@@ -710,10 +742,10 @@ type CheckTally = {
|
|
|
710
742
|
notRun: number;
|
|
711
743
|
};
|
|
712
744
|
/**
|
|
713
|
-
* One check over a cell's counted attempts. `verdict` (variant cells)
|
|
714
|
-
*
|
|
715
|
-
*
|
|
716
|
-
*
|
|
745
|
+
* One check over a cell's counted attempts. `verdict` (variant cells) is `discriminates` when
|
|
746
|
+
* every answered instance passes on one side and fails on the other, `both-pass` when every
|
|
747
|
+
* answered instance passes on both sides, `both-fail` when every answered instance fails on both
|
|
748
|
+
* sides, or `mixed` otherwise. It is absent when either side has no pass or fail.
|
|
717
749
|
*/
|
|
718
750
|
type CellCheck = CheckTally & {
|
|
719
751
|
label: string;
|
|
@@ -773,7 +805,10 @@ type RunRecord = {
|
|
|
773
805
|
harness?: string[];
|
|
774
806
|
environment?: string;
|
|
775
807
|
sandbox?: string;
|
|
808
|
+
/** With `bestOf`, the most attempts a cell runs. */
|
|
776
809
|
runs?: number;
|
|
810
|
+
/** Set by `defaults.runs: { bestOf }`. */
|
|
811
|
+
bestOf?: number;
|
|
777
812
|
timeout?: string | number;
|
|
778
813
|
env?: string[];
|
|
779
814
|
};
|
|
@@ -807,8 +842,13 @@ type RunRecord = {
|
|
|
807
842
|
config: {
|
|
808
843
|
sandbox: string;
|
|
809
844
|
environment: string | null;
|
|
810
|
-
/**
|
|
845
|
+
/**
|
|
846
|
+
* The config's `runs` (with `bestOf`, the most a cell runs); 1 when it sets none, though a
|
|
847
|
+
* compare's cells then run 4 each.
|
|
848
|
+
*/
|
|
811
849
|
runs: number;
|
|
850
|
+
/** Set by the config's `runs: { bestOf }`. */
|
|
851
|
+
bestOf?: number;
|
|
812
852
|
judgeModel: string | null;
|
|
813
853
|
threshold: number;
|
|
814
854
|
timeoutMs: number;
|
|
@@ -827,28 +867,26 @@ type RunRecord = {
|
|
|
827
867
|
/** Each pooled `evaluate.each` that ran; unset when none did. */
|
|
828
868
|
families?: FamilyRecord[];
|
|
829
869
|
/**
|
|
830
|
-
*
|
|
831
|
-
* attempt awaited, a worker that died. Any one fails the run. Unset when there
|
|
870
|
+
* These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
|
|
871
|
+
* or throw no attempt awaited, or a worker that died. Any one fails the run. Unset when there
|
|
872
|
+
* were none.
|
|
832
873
|
*/
|
|
833
874
|
errors?: string[];
|
|
834
875
|
};
|
|
835
|
-
/**
|
|
876
|
+
/** These fields describe what ran and graded the agents: the machine, images, harnesses, and judge. */
|
|
836
877
|
type Stack = {
|
|
837
878
|
/** The reeval version that wrote the run. */
|
|
838
879
|
reeval: string;
|
|
839
|
-
/** `process.versions.node
|
|
880
|
+
/** `process.versions.node`, or the compatible Node version reported by another runtime. */
|
|
840
881
|
node: string;
|
|
841
|
-
/**
|
|
842
|
-
* The runtime reeval ran on, `node`, `bun`, `deno` or another, and its own version. Absent in
|
|
843
|
-
* runs recorded before it, which ran on Node.
|
|
844
|
-
*/
|
|
882
|
+
/** The runtime reeval ran on, `node`, `bun`, `deno`, or another, and its version. */
|
|
845
883
|
runtime?: {
|
|
846
884
|
name: string;
|
|
847
885
|
version: string;
|
|
848
886
|
};
|
|
849
887
|
/**
|
|
850
|
-
* The language
|
|
851
|
-
*
|
|
888
|
+
* The evaluation language, `typescript` for `*.eval.ts` files, and the version of its toolchain
|
|
889
|
+
* when the runner reports one.
|
|
852
890
|
*/
|
|
853
891
|
language?: {
|
|
854
892
|
name: string;
|
|
@@ -856,16 +894,17 @@ type Stack = {
|
|
|
856
894
|
};
|
|
857
895
|
os: string;
|
|
858
896
|
arch: string;
|
|
859
|
-
/** Every image an attempt started from. `id` is content-addressed; `repoDigests` only
|
|
897
|
+
/** Every image an attempt started from. `id` is content-addressed; `repoDigests` exist only after the image is pushed. */
|
|
860
898
|
images: Array<AttemptImage & {
|
|
861
899
|
repoDigests: string[];
|
|
862
900
|
}>;
|
|
863
901
|
/** Exact versions of the packages that run and grade the agent. */
|
|
864
902
|
packages: Record<string, string>;
|
|
865
903
|
/**
|
|
866
|
-
*
|
|
867
|
-
* reported serving, with the variant it resolved), and
|
|
868
|
-
* suffixes those labels named). Fast (`[fast=true]`) is not
|
|
904
|
+
* For each harness, the record includes its configured `model` and requested `effort`,
|
|
905
|
+
* `resolvedModels` (what it reported serving, with the variant it resolved), and
|
|
906
|
+
* `resolvedEffort` (reasoning-effort suffixes those labels named). Fast (`[fast=true]`) is not
|
|
907
|
+
* `resolvedEffort`.
|
|
869
908
|
*/
|
|
870
909
|
harnesses: Array<{
|
|
871
910
|
id: string;
|
|
@@ -901,12 +940,12 @@ type Stack = {
|
|
|
901
940
|
threshold: number;
|
|
902
941
|
/** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
|
|
903
942
|
version: string;
|
|
904
|
-
/**
|
|
943
|
+
/** The config's default judge inputs, every key resolved. Unset when the config names none, which reeval requires: each `c.judge()` call names its own. */
|
|
905
944
|
state?: ResolvedJudgeStateOptions;
|
|
906
945
|
};
|
|
907
946
|
/** Union of every attempt's context files. */
|
|
908
947
|
context: ContextFile[];
|
|
909
|
-
/** What
|
|
948
|
+
/** What evaluation fingerprints hash; fingerprints from different scopes aren't comparable. */
|
|
910
949
|
fingerprintScope: string[];
|
|
911
950
|
};
|
|
912
951
|
/** The parts of a run's stack that `reeval compare` checks for drift. */
|
|
@@ -1108,16 +1147,21 @@ type Wait = CaptureWait | {
|
|
|
1108
1147
|
type CaptureInstruction = string | {
|
|
1109
1148
|
wait: Wait;
|
|
1110
1149
|
};
|
|
1150
|
+
/** The keys a capture's driver presses. In a focused `<select>`, the arrows change the choice. */
|
|
1151
|
+
type CaptureKey = 'Tab' | 'Enter' | 'Escape' | 'Backspace' | 'Space' | 'ArrowUp' | 'ArrowDown' | 'ArrowLeft' | 'ArrowRight';
|
|
1111
1152
|
type CaptureUseAction = {
|
|
1112
1153
|
type: 'click';
|
|
1113
1154
|
x: number;
|
|
1114
1155
|
y: number;
|
|
1115
|
-
} |
|
|
1156
|
+
} |
|
|
1157
|
+
/** `replace` selects what the focused field holds first, so the text replaces it. */
|
|
1158
|
+
{
|
|
1116
1159
|
type: 'type';
|
|
1117
1160
|
text: string;
|
|
1161
|
+
replace?: boolean;
|
|
1118
1162
|
} | {
|
|
1119
1163
|
type: 'key';
|
|
1120
|
-
key:
|
|
1164
|
+
key: CaptureKey;
|
|
1121
1165
|
} | {
|
|
1122
1166
|
type: 'scroll';
|
|
1123
1167
|
x: number;
|
|
@@ -1152,7 +1196,7 @@ type CaptureUse = {
|
|
|
1152
1196
|
type CaptureOptions = {
|
|
1153
1197
|
/** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
|
|
1154
1198
|
viewport?: Viewport;
|
|
1155
|
-
/** How long to poll before failing. Default `'60s'`. */
|
|
1199
|
+
/** How long to poll before failing; a 502, 503 or 504 is polled past until then. Default `'60s'`. */
|
|
1156
1200
|
timeout?: Duration;
|
|
1157
1201
|
/** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
|
|
1158
1202
|
browser?: boolean;
|
|
@@ -1166,7 +1210,7 @@ type CaptureOptions = {
|
|
|
1166
1210
|
type BackgroundHandle = {
|
|
1167
1211
|
kill(): Promise<void>;
|
|
1168
1212
|
};
|
|
1169
|
-
/** `c.serve`: a background process plus the port it
|
|
1213
|
+
/** `c.serve`: a background process plus the port it listens on. */
|
|
1170
1214
|
type ServeHandle = BackgroundHandle & {
|
|
1171
1215
|
port: number;
|
|
1172
1216
|
};
|
|
@@ -1279,10 +1323,10 @@ type JudgeState = JudgeJsonState | Array<{
|
|
|
1279
1323
|
type: 'json';
|
|
1280
1324
|
value: JudgeJsonState;
|
|
1281
1325
|
} | JudgeFilePart>;
|
|
1282
|
-
/** A capture this judge call sent as a file part. Bytes live in `artifacts/<name>.png
|
|
1326
|
+
/** A capture this judge call sent as a file part. Bytes live in the `artifacts/<name>.png` file. */
|
|
1283
1327
|
type JudgeArtifactRef = {
|
|
1284
1328
|
name: string;
|
|
1285
|
-
/** `artifacts/<name>.png` under the attempt directory. */
|
|
1329
|
+
/** The `artifacts/<name>.png` path under the attempt directory. */
|
|
1286
1330
|
path: string;
|
|
1287
1331
|
sha256: string;
|
|
1288
1332
|
bytes: number;
|
|
@@ -1314,11 +1358,6 @@ type JudgeOptions = {
|
|
|
1314
1358
|
providerOptions?: Record<string, Record<string, JSONValue>>;
|
|
1315
1359
|
/** The most judge calls to this model's provider at once, across the run. */
|
|
1316
1360
|
maxConcurrent?: number;
|
|
1317
|
-
/**
|
|
1318
|
-
* The judge's state: what the judge is given beside the question, not the workspace's or the
|
|
1319
|
-
* attempt's state. Merged key by key down the cascade.
|
|
1320
|
-
*/
|
|
1321
|
-
state?: JudgeStateOptions;
|
|
1322
1361
|
/**
|
|
1323
1362
|
* The gateway the judge's calls go through, by name or by its helper. It skips the config's
|
|
1324
1363
|
* `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
|
|
@@ -1326,43 +1365,61 @@ type JudgeOptions = {
|
|
|
1326
1365
|
gateway?: GatewayName | GatewayDescriptor;
|
|
1327
1366
|
};
|
|
1328
1367
|
/**
|
|
1329
|
-
* What
|
|
1330
|
-
*
|
|
1368
|
+
* What one `c.judge()` call shows the judge beside the question. Every input is off unless the
|
|
1369
|
+
* call turns it on, and a call turns on at least one; each is a key of the state in
|
|
1370
|
+
* `judge-input.json`, or the captures sent as file parts.
|
|
1331
1371
|
*/
|
|
1332
|
-
type
|
|
1333
|
-
/** The graded turn's prompt.
|
|
1372
|
+
type JudgeInputs = {
|
|
1373
|
+
/** The graded turn's prompt. */
|
|
1334
1374
|
task?: boolean;
|
|
1335
|
-
/** The graded turn's final message, or `on`'s value.
|
|
1375
|
+
/** The graded turn's final message, or `on`'s value. */
|
|
1336
1376
|
output?: boolean;
|
|
1337
|
-
/** `status path` lines for the changed files, under `diff`'s patterns.
|
|
1377
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. */
|
|
1338
1378
|
files?: boolean;
|
|
1339
1379
|
/**
|
|
1340
1380
|
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
1341
|
-
* the last matching pattern wins and `!` excludes.
|
|
1381
|
+
* the last matching pattern wins and `!` excludes.
|
|
1342
1382
|
*/
|
|
1343
1383
|
diff?: boolean | string[];
|
|
1344
1384
|
/**
|
|
1345
1385
|
* Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
|
|
1346
1386
|
* patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
|
|
1347
1387
|
* the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
|
|
1348
|
-
* These files leave the diff.
|
|
1388
|
+
* These files leave the diff.
|
|
1349
1389
|
*/
|
|
1350
1390
|
contents?: string[];
|
|
1351
1391
|
/**
|
|
1352
1392
|
* The graded turn's transcript as text lines: `true` is messages and tool calls with tool
|
|
1353
1393
|
* output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
|
|
1354
|
-
* Default false.
|
|
1355
1394
|
*/
|
|
1356
1395
|
transcript?: boolean | 'tools' | 'full';
|
|
1396
|
+
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. */
|
|
1397
|
+
checks?: boolean | string[];
|
|
1398
|
+
/**
|
|
1399
|
+
* Captures sent as file parts: `c.capture()` handles or their names, or `true` for every
|
|
1400
|
+
* capture so far. Jev takes PNG captures; a provider that cannot take files fails the call.
|
|
1401
|
+
*/
|
|
1402
|
+
artifacts?: true | Array<Capture | string>;
|
|
1403
|
+
};
|
|
1404
|
+
/** What a judge call's inputs read: which turn, and whether the turns before it too. */
|
|
1405
|
+
type JudgeSubject = {
|
|
1406
|
+
/** The turn `task`, `output`, `transcript` and `diff` read instead of the latest, or a value used as the output. */
|
|
1407
|
+
on?: Turn | JSONValue;
|
|
1357
1408
|
/**
|
|
1358
1409
|
* `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
|
|
1359
1410
|
* `transcript`) transcript. Default `'last'`.
|
|
1360
1411
|
*/
|
|
1361
1412
|
turns?: 'last' | 'all';
|
|
1362
|
-
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
1363
|
-
checks?: boolean | string[];
|
|
1364
1413
|
};
|
|
1365
|
-
/** `
|
|
1414
|
+
/** One input turned on: `true`, a mode or a non-empty selection, not `false`. */
|
|
1415
|
+
type OneInput<I, K extends keyof I> = { [P in K]-?: Exclude<I[P], false | undefined>; };
|
|
1416
|
+
/** At least one of `I`'s keys turned on. */
|
|
1417
|
+
type AtLeastOneInput<I> = { [K in keyof I]-?: OneInput<I, K>; }[keyof I];
|
|
1418
|
+
/** `c.judge()`'s options, before the at-least-one-input rule that `JudgeCallOptions` adds. */
|
|
1419
|
+
type JudgeCallFields = JudgeOptions & JudgeInputs & JudgeSubject;
|
|
1420
|
+
/** `c.judge()`'s options: the judge options, the inputs (at least one) and what they read. */
|
|
1421
|
+
type JudgeCallOptions = JudgeCallFields & AtLeastOneInput<JudgeInputs>;
|
|
1422
|
+
/** The inputs one judge call saw, every key set (`false` when off); `artifacts` is recorded apart. */
|
|
1366
1423
|
type ResolvedJudgeStateOptions = {
|
|
1367
1424
|
task: boolean;
|
|
1368
1425
|
output: boolean;
|
|
@@ -1373,16 +1430,6 @@ type ResolvedJudgeStateOptions = {
|
|
|
1373
1430
|
turns: 'last' | 'all';
|
|
1374
1431
|
checks: boolean | string[];
|
|
1375
1432
|
};
|
|
1376
|
-
type JudgeCallOptions = JudgeOptions & {
|
|
1377
|
-
/** What to grade instead of the latest turn: another turn, or a value used as the output. */
|
|
1378
|
-
on?: Turn | JSONValue;
|
|
1379
|
-
/**
|
|
1380
|
-
* Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
|
|
1381
|
-
* `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
|
|
1382
|
-
* captures; a provider that cannot take files fails the call.
|
|
1383
|
-
*/
|
|
1384
|
-
artifacts?: true | Array<Capture | string>;
|
|
1385
|
-
};
|
|
1386
1433
|
type Verdict = {
|
|
1387
1434
|
verdict: 'pass' | 'fail';
|
|
1388
1435
|
/** The judge's probability that the answer is yes, from 0 to 1. */
|
|
@@ -1418,10 +1465,11 @@ type AttemptContext = {
|
|
|
1418
1465
|
/** Sends the next turn of this eval's agent session. */
|
|
1419
1466
|
send(prompt: string): Promise<Turn>;
|
|
1420
1467
|
/**
|
|
1421
|
-
* Asks the judge a yes/no question about
|
|
1422
|
-
*
|
|
1468
|
+
* Asks the judge a yes/no question about what `options` turns on, and nothing else: at least
|
|
1469
|
+
* one of `task`, `output`, `files`, `diff`, `contents`, `transcript`, `checks` or `artifacts`.
|
|
1470
|
+
* They read the latest turn, or `options.on`.
|
|
1423
1471
|
*/
|
|
1424
|
-
judge(question: string, options
|
|
1472
|
+
judge(question: string, options: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1425
1473
|
/**
|
|
1426
1474
|
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1427
1475
|
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
@@ -1436,8 +1484,10 @@ type AttemptContext = {
|
|
|
1436
1484
|
background?: boolean;
|
|
1437
1485
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1438
1486
|
/**
|
|
1439
|
-
* Starts `command` in the background
|
|
1440
|
-
* and
|
|
1487
|
+
* Starts `command` in the background with `PORT` set to `port`, or to a port reeval allocates,
|
|
1488
|
+
* and returns once it listens. Without `port`, a command that listens on one new port instead
|
|
1489
|
+
* (Vite's 5173) gets that one; several new ports, none of them `PORT`, or an exit first, throw
|
|
1490
|
+
* with its last output.
|
|
1441
1491
|
*/
|
|
1442
1492
|
serve(command: string, port?: number): Promise<ServeHandle>;
|
|
1443
1493
|
/**
|
|
@@ -2378,9 +2428,11 @@ type CompareOptions<E = Environment> = {
|
|
|
2378
2428
|
* With `across`, it picks between two attempts that differ only in that factor, in both orders;
|
|
2379
2429
|
* without it, it's asked of each attempt on its own. Without a model, the cascaded
|
|
2380
2430
|
* `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
|
|
2381
|
-
*
|
|
2431
|
+
* It's shown only the inputs it turns on, at least one.
|
|
2382
2432
|
*/
|
|
2383
|
-
type Question =
|
|
2433
|
+
type Question = QuestionFields & AtLeastOneInput<QuestionInputs>;
|
|
2434
|
+
/** A question's options, before the at-least-one-input rule that `Question` adds. */
|
|
2435
|
+
type QuestionFields = QuestionInputs & {
|
|
2384
2436
|
question: string;
|
|
2385
2437
|
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
2386
2438
|
model?: string;
|
|
@@ -2391,43 +2443,39 @@ type Question = string | {
|
|
|
2391
2443
|
across?: PairFactor | PairFactor[];
|
|
2392
2444
|
/** The answers a question asked of each attempt picks from, with a short reason. */
|
|
2393
2445
|
choices?: string[];
|
|
2394
|
-
/**
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2446
|
+
/**
|
|
2447
|
+
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2448
|
+
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2449
|
+
*/
|
|
2450
|
+
turns?: 'last' | 'all';
|
|
2398
2451
|
};
|
|
2399
2452
|
/**
|
|
2400
|
-
* What a question is shown of each attempt:
|
|
2401
|
-
*
|
|
2402
|
-
*
|
|
2403
|
-
* message and the transcript.
|
|
2453
|
+
* What a question is shown of each attempt: `c.judge()`'s inputs but `contents` (the attempts'
|
|
2454
|
+
* workspaces are gone when questions are asked). Every input is off unless the question turns it
|
|
2455
|
+
* on, and a question turns on at least one.
|
|
2404
2456
|
*/
|
|
2405
|
-
type
|
|
2406
|
-
/** The prompts the attempt was sent, once for both sides when they match.
|
|
2457
|
+
type QuestionInputs = {
|
|
2458
|
+
/** The prompts the attempt was sent, once for both sides when they match. */
|
|
2407
2459
|
task?: boolean;
|
|
2408
|
-
/** The final message: the agent's own account of its work.
|
|
2460
|
+
/** The final message: the agent's own account of its work. */
|
|
2409
2461
|
output?: boolean;
|
|
2410
|
-
/** `status path` lines for the changed files, under `diff`'s patterns.
|
|
2462
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. */
|
|
2411
2463
|
files?: boolean;
|
|
2412
2464
|
/**
|
|
2413
2465
|
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
2414
|
-
* the last matching pattern wins and `!` excludes.
|
|
2466
|
+
* the last matching pattern wins and `!` excludes.
|
|
2415
2467
|
*/
|
|
2416
2468
|
diff?: boolean | string[];
|
|
2417
2469
|
/**
|
|
2418
2470
|
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
2419
2471
|
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2420
|
-
* harness's tools, so a pair of two harnesses isn't blind.
|
|
2421
|
-
* attempt.
|
|
2472
|
+
* harness's tools, so a pair of two harnesses isn't blind.
|
|
2422
2473
|
*/
|
|
2423
2474
|
transcript?: boolean | 'tools' | 'full';
|
|
2424
|
-
/**
|
|
2425
|
-
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2426
|
-
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2427
|
-
*/
|
|
2428
|
-
turns?: 'last' | 'all';
|
|
2429
|
-
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
2475
|
+
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. */
|
|
2430
2476
|
checks?: boolean | string[];
|
|
2477
|
+
/** Each attempt's own captures, by name; `true` sends all of them. */
|
|
2478
|
+
artifacts?: true | string[];
|
|
2431
2479
|
};
|
|
2432
2480
|
/** `QuestionStateOptions` with every key set: what each side of one question showed. */
|
|
2433
2481
|
type ResolvedQuestionStateOptions = {
|
|
@@ -2439,14 +2487,10 @@ type ResolvedQuestionStateOptions = {
|
|
|
2439
2487
|
turns: 'last' | 'all';
|
|
2440
2488
|
checks: boolean | string[];
|
|
2441
2489
|
};
|
|
2442
|
-
/** Defaults for every question
|
|
2490
|
+
/** Defaults for every question below it. What a question is shown is its own, never a default. */
|
|
2443
2491
|
type QuestionOptions = {
|
|
2444
2492
|
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2445
2493
|
model?: string;
|
|
2446
|
-
/** What it's shown of each attempt. */
|
|
2447
|
-
state?: QuestionStateOptions;
|
|
2448
|
-
/** Each attempt's own captures, by name; `true` sends all of them. Default none. */
|
|
2449
|
-
artifacts?: true | string[];
|
|
2450
2494
|
};
|
|
2451
2495
|
/** A matrix factor a pairwise question varies; the others are held. */
|
|
2452
2496
|
type PairFactor = 'environment' | 'harness';
|
|
@@ -2484,7 +2528,7 @@ type CompareAnswer = {
|
|
|
2484
2528
|
model: string;
|
|
2485
2529
|
/** The model id the provider says served the calls, when it reports one. */
|
|
2486
2530
|
resolvedModel?: string;
|
|
2487
|
-
/**
|
|
2531
|
+
/** The inputs each side showed, every key resolved; unset on answers recorded before reeval wrote it, which showed the task and the diff. */
|
|
2488
2532
|
stateOptions?: ResolvedQuestionStateOptions;
|
|
2489
2533
|
/** The captures it sent of each side, as asked: `true` for all, or their names. */
|
|
2490
2534
|
artifacts?: true | string[];
|
|
@@ -2537,8 +2581,21 @@ type GatePolicy = 'all' | {
|
|
|
2537
2581
|
type CellGate = GatePolicy | 'none';
|
|
2538
2582
|
/** A compare object: `baseline` and `variants`. Same as `CompareOptions`. */
|
|
2539
2583
|
type CompareSpec<E = Environment> = CompareOptions<E>;
|
|
2584
|
+
type OddDigit = '1' | '3' | '5' | '7' | '9';
|
|
2585
|
+
/**
|
|
2586
|
+
* `N` when it's an odd whole number of 3 or more, else a sentence the compiler shows as the type
|
|
2587
|
+
* it expected. A `number` that isn't a literal stays `number`: only planning can check it.
|
|
2588
|
+
*/
|
|
2589
|
+
type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${string}` | `${string}.${string}` | `${string}e${string}` | '1' ? 'an odd whole number of 3 or more, e.g. 3 or 5' : `${N}` extends `${string}${OddDigit}` ? N : 'an odd whole number of 3 or more, e.g. 3 or 5';
|
|
2590
|
+
/**
|
|
2591
|
+
* Attempts per cell: a fixed count, or `{ bestOf }`, an odd number of 3 or more, which stops a
|
|
2592
|
+
* cell once one side (pass or fail) has a majority: `{ bestOf: 5 }` runs 3 to 5 attempts.
|
|
2593
|
+
*/
|
|
2594
|
+
type Runs<N extends number = number> = number | {
|
|
2595
|
+
bestOf: OddAtLeast3<N>;
|
|
2596
|
+
};
|
|
2540
2597
|
/** Options other than `environment` and `compare`, which are exclusive of each other. */
|
|
2541
|
-
type EvalOptionFields = {
|
|
2598
|
+
type EvalOptionFields<N extends number = number> = {
|
|
2542
2599
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2543
2600
|
harness?: HarnessOption;
|
|
2544
2601
|
/**
|
|
@@ -2547,11 +2604,12 @@ type EvalOptionFields = {
|
|
|
2547
2604
|
*/
|
|
2548
2605
|
sandbox?: SandboxDescriptor;
|
|
2549
2606
|
/**
|
|
2550
|
-
* Attempts per cell
|
|
2551
|
-
*
|
|
2552
|
-
*
|
|
2607
|
+
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
|
|
2608
|
+
* a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
|
|
2609
|
+
* Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
|
|
2610
|
+
* a verdict, and anything else runs once, a pooled `evaluate.each` case included.
|
|
2553
2611
|
*/
|
|
2554
|
-
runs?:
|
|
2612
|
+
runs?: Runs<N>;
|
|
2555
2613
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2556
2614
|
gate?: GatePolicy;
|
|
2557
2615
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
@@ -2565,7 +2623,7 @@ type EvalOptionFields = {
|
|
|
2565
2623
|
judge?: JudgeOptions;
|
|
2566
2624
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
2567
2625
|
capture?: CaptureOptions;
|
|
2568
|
-
/**
|
|
2626
|
+
/** The model every question that names none of its own is answered by, down the cascade. */
|
|
2569
2627
|
question?: QuestionOptions;
|
|
2570
2628
|
/**
|
|
2571
2629
|
* What a language model is asked about the attempts, always a list; they never change pass,
|
|
@@ -2576,6 +2634,11 @@ type EvalOptionFields = {
|
|
|
2576
2634
|
questions?: Question[];
|
|
2577
2635
|
/** Per attempt. */
|
|
2578
2636
|
timeout?: Duration;
|
|
2637
|
+
/**
|
|
2638
|
+
* When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
|
|
2639
|
+
* the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
|
|
2640
|
+
*/
|
|
2641
|
+
snapshot?: SnapshotOption;
|
|
2579
2642
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
2580
2643
|
tags?: string[];
|
|
2581
2644
|
/**
|
|
@@ -2622,8 +2685,9 @@ type EnvironmentOrCompare<E> = {
|
|
|
2622
2685
|
baseline?: undefined;
|
|
2623
2686
|
};
|
|
2624
2687
|
};
|
|
2625
|
-
|
|
2626
|
-
type
|
|
2688
|
+
/** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
|
|
2689
|
+
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
|
|
2690
|
+
type GroupOptions<N extends number = number> = EvalOptions<N>;
|
|
2627
2691
|
/**
|
|
2628
2692
|
* `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
|
|
2629
2693
|
* by name or a fake judge, and environments without `extend`. The same `environment` / compare
|
|
@@ -2663,8 +2727,8 @@ type ProfileDefaults = {
|
|
|
2663
2727
|
sandbox?: SandboxDescriptor;
|
|
2664
2728
|
/** What every eval runs in when it names no environment. */
|
|
2665
2729
|
environment?: Environment;
|
|
2666
|
-
/** Attempts per cell. */
|
|
2667
|
-
runs?:
|
|
2730
|
+
/** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
|
|
2731
|
+
runs?: Runs;
|
|
2668
2732
|
/** How each cell's attempts decide its outcome. */
|
|
2669
2733
|
gate?: GatePolicy;
|
|
2670
2734
|
/** Variables and secrets for the sandbox, merged by name over the config's. */
|
|
@@ -2679,6 +2743,8 @@ type ProfileDefaults = {
|
|
|
2679
2743
|
capture?: CaptureOptions;
|
|
2680
2744
|
/** Defaults for every question, merged over the config's. */
|
|
2681
2745
|
question?: QuestionOptions;
|
|
2746
|
+
/** When each attempt's workspace is copied; merged over the config's. */
|
|
2747
|
+
snapshot?: SnapshotOption;
|
|
2682
2748
|
};
|
|
2683
2749
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
2684
2750
|
type ProfileLimit = {
|
|
@@ -2755,15 +2821,15 @@ type BlindAttempt = {
|
|
|
2755
2821
|
choices?: string[];
|
|
2756
2822
|
attempt: BlindSide;
|
|
2757
2823
|
};
|
|
2758
|
-
/** A turn of one side, with the
|
|
2824
|
+
/** A turn of one side, with the inputs its question turned on. */
|
|
2759
2825
|
type BlindTurn = {
|
|
2760
2826
|
task?: string;
|
|
2761
2827
|
output?: string;
|
|
2762
2828
|
transcript?: string;
|
|
2763
2829
|
};
|
|
2764
2830
|
/**
|
|
2765
|
-
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the
|
|
2766
|
-
* its question
|
|
2831
|
+
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the inputs
|
|
2832
|
+
* its question turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
|
|
2767
2833
|
* under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
|
|
2768
2834
|
*/
|
|
2769
2835
|
type BlindSide = BlindTurn & {
|
|
@@ -2817,8 +2883,11 @@ type ReevalConfig = {
|
|
|
2817
2883
|
sandbox?: SandboxDescriptor;
|
|
2818
2884
|
/** What every eval runs in when it names no environment, and a baseline-less compare's control. */
|
|
2819
2885
|
environment?: Environment;
|
|
2820
|
-
/**
|
|
2821
|
-
|
|
2886
|
+
/**
|
|
2887
|
+
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
|
|
2888
|
+
* variants runs 4 a side and anything else once.
|
|
2889
|
+
*/
|
|
2890
|
+
runs?: Runs;
|
|
2822
2891
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2823
2892
|
gate?: GatePolicy;
|
|
2824
2893
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
@@ -2836,6 +2905,11 @@ type ReevalConfig = {
|
|
|
2836
2905
|
capture?: CaptureOptions;
|
|
2837
2906
|
/** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
|
|
2838
2907
|
question?: QuestionOptions;
|
|
2908
|
+
/**
|
|
2909
|
+
* When each attempt's workspace is copied into its directory in `.reeval/runs`: `true` after
|
|
2910
|
+
* the checks, `'turns'` after each `c.send` too, `false` never. Default `true`.
|
|
2911
|
+
*/
|
|
2912
|
+
snapshot?: SnapshotOption;
|
|
2839
2913
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
2840
2914
|
concurrency?: number;
|
|
2841
2915
|
/** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
|
|
@@ -2920,9 +2994,9 @@ type AttemptError = {
|
|
|
2920
2994
|
};
|
|
2921
2995
|
/**
|
|
2922
2996
|
* One start of a stdio MCP server during a turn (`turns[].mcpStartup`, in start order), timed by
|
|
2923
|
-
* reeval's MCP timer, which every stdio server runs through. An agent CLI that
|
|
2924
|
-
* turn
|
|
2925
|
-
*
|
|
2997
|
+
* reeval's MCP timer, which every stdio server runs through. An agent CLI that starts one process
|
|
2998
|
+
* per turn starts its servers on every turn; a CLI that keeps a session starts them on the first
|
|
2999
|
+
* turn only. A server that doesn't answer within 60s fails the attempt as an infra
|
|
2926
3000
|
* error naming it; one whose client closes it first doesn't. `mcpStartup` is absent when the
|
|
2927
3001
|
* environment has no stdio MCP server, or the sandbox has no `node` to run the timer.
|
|
2928
3002
|
*/
|
|
@@ -2935,7 +3009,7 @@ type McpStartup = {
|
|
|
2935
3009
|
*/
|
|
2936
3010
|
ms: number | null;
|
|
2937
3011
|
};
|
|
2938
|
-
/**
|
|
3012
|
+
/** The duration of each evaluation phase, in ms. `agentMs` is the harness turns; `diffMs` reading changes. */
|
|
2939
3013
|
type PhaseTimings = {
|
|
2940
3014
|
setupMs: number;
|
|
2941
3015
|
agentMs: number;
|
|
@@ -2945,16 +3019,28 @@ type PhaseTimings = {
|
|
|
2945
3019
|
};
|
|
2946
3020
|
type CellRole = 'control' | 'variant';
|
|
2947
3021
|
/**
|
|
2948
|
-
*
|
|
2949
|
-
*
|
|
2950
|
-
*
|
|
2951
|
-
*
|
|
3022
|
+
* A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
|
|
3023
|
+
* environment's sandbox id. With `compare`, the control is `baseline` (or the cascaded
|
|
3024
|
+
* environment, or an empty `baseline`) and each variant is that environment on its own,
|
|
3025
|
+
* including its sandbox.
|
|
2952
3026
|
*/
|
|
2953
3027
|
type Cell = {
|
|
3028
|
+
/** The harness id, the environment name, and the sandbox id, joined with `/`. */
|
|
2954
3029
|
key: string;
|
|
3030
|
+
/**
|
|
3031
|
+
* The harness id: `id` when set, otherwise `<kind>:<model>` (`<kind>` alone without a model),
|
|
3032
|
+
* with `[effort=<value>]` appended whenever `effort` is set.
|
|
3033
|
+
*/
|
|
2955
3034
|
harness: string;
|
|
2956
|
-
/** The environment's own name, also on a variant cell (`with-
|
|
3035
|
+
/** The environment's own name, also on a variant cell (`with-skill`, with no prefix). */
|
|
2957
3036
|
environment: string;
|
|
3037
|
+
/**
|
|
3038
|
+
* The sandbox id is `id` when set, or `fake` for the fake sandbox. A Docker sandbox starts with
|
|
3039
|
+
* the engine (`docker` by default). It adds `:` and the sanitized final component of `image` or
|
|
3040
|
+
* `from`, or for `dockerfile` the sanitized filename (the parent directory name when that
|
|
3041
|
+
* filename is `Dockerfile` and a parent is present). Non-empty `install` steps append `+` and an
|
|
3042
|
+
* eight-character hash.
|
|
3043
|
+
*/
|
|
2958
3044
|
sandbox: string;
|
|
2959
3045
|
role: CellRole;
|
|
2960
3046
|
/** The variant's name, for variant cells. */
|
|
@@ -2997,9 +3083,9 @@ type EnvironmentRecord = {
|
|
|
2997
3083
|
name: string;
|
|
2998
3084
|
/** The previous `extendEnvironment` layer's name, else null. */
|
|
2999
3085
|
parent: string | null;
|
|
3000
|
-
/** Keys a later layer replaced,
|
|
3086
|
+
/** Keys a later layer replaced, for example `env.KEY`, `skills.typescript`, `mcp.docs`, or `rules`. */
|
|
3001
3087
|
overrides: string[];
|
|
3002
|
-
/** Covers every field but the rules file's path; secret names are included
|
|
3088
|
+
/** Covers every field but the rules file's path; secret names are included; values never are. The image is the sandbox's. */
|
|
3003
3089
|
digest: string;
|
|
3004
3090
|
setup: string[];
|
|
3005
3091
|
skills: Array<{
|
|
@@ -3013,7 +3099,7 @@ type EnvironmentRecord = {
|
|
|
3013
3099
|
/** The plugin the skill came with; unset for a skill from `skills`. */
|
|
3014
3100
|
plugin?: string;
|
|
3015
3101
|
}>;
|
|
3016
|
-
/**
|
|
3102
|
+
/** The names of the MCP servers the agent was given. */
|
|
3017
3103
|
mcp: string[];
|
|
3018
3104
|
/** The plugins placed, each with what it placed; in the digest. Unset when there are none. */
|
|
3019
3105
|
plugins?: PluginRecord[];
|
|
@@ -3038,9 +3124,9 @@ type EnvironmentRecord = {
|
|
|
3038
3124
|
/** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
|
|
3039
3125
|
egress: EgressPolicy;
|
|
3040
3126
|
/**
|
|
3041
|
-
*
|
|
3042
|
-
* project root), the ref and path asked for (null when unset), the commit the
|
|
3043
|
-
* and the hash of the tree that was copied in, which is in the digest.
|
|
3127
|
+
* This records the Git repository the workspace started as: the repo as written (a local one
|
|
3128
|
+
* relative to the project root), the ref and path asked for (null when unset), the commit the
|
|
3129
|
+
* ref resolved to, and the hash of the tree that was copied in, which is in the digest.
|
|
3044
3130
|
*/
|
|
3045
3131
|
workspace?: {
|
|
3046
3132
|
source: string;
|
|
@@ -3057,8 +3143,8 @@ type EnvironmentRecord = {
|
|
|
3057
3143
|
* and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
|
|
3058
3144
|
* can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3059
3145
|
* the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3060
|
-
* harness always tries (its CLI's telemetry, say), kept out of `blocked
|
|
3061
|
-
*
|
|
3146
|
+
* harness always tries (its CLI's telemetry, say), kept out of `blocked`, so `blocked` contains
|
|
3147
|
+
* only unexpected refusals.
|
|
3062
3148
|
*/
|
|
3063
3149
|
type EgressRecord = {
|
|
3064
3150
|
policy: EgressPolicy;
|
|
@@ -3096,7 +3182,7 @@ type AttemptImage = {
|
|
|
3096
3182
|
/** Content-addressed image id (`sha256:…`). */
|
|
3097
3183
|
id: string;
|
|
3098
3184
|
agentCli: string | null;
|
|
3099
|
-
/** The harness image `install`
|
|
3185
|
+
/** The harness image the `install` layers were baked onto: `ref` itself without any. */
|
|
3100
3186
|
base: string;
|
|
3101
3187
|
/** The `docker({ install })` commands, in order. */
|
|
3102
3188
|
install: string[];
|
|
@@ -3131,11 +3217,26 @@ type SetupProcess = {
|
|
|
3131
3217
|
/** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
|
|
3132
3218
|
secrets: string[];
|
|
3133
3219
|
};
|
|
3134
|
-
/** One `
|
|
3220
|
+
/** One `c.serve` call that listened. */
|
|
3221
|
+
type ServeRecord = {
|
|
3222
|
+
/** The command as the eval passed it. */
|
|
3223
|
+
command: string;
|
|
3224
|
+
/** The port it listens on, which the handle and `c.capture` use. */
|
|
3225
|
+
port: number;
|
|
3226
|
+
/** The `PORT` it was given: the port the eval passed, or the one reeval allocated. */
|
|
3227
|
+
portEnv: number;
|
|
3228
|
+
/** It listened on a port other than `PORT`, the one that started listening after it did. */
|
|
3229
|
+
detected: boolean;
|
|
3230
|
+
/** The address it listens on there, e.g. `127.0.0.1`, `::1` or `0.0.0.0`; empty when the sandbox can't tell. */
|
|
3231
|
+
address: string;
|
|
3232
|
+
/** From the command's start until it listened. */
|
|
3233
|
+
readyMs: number;
|
|
3234
|
+
};
|
|
3235
|
+
/** One `expect` or `expect.require` matcher call an attempt made, or a check another language made. */
|
|
3135
3236
|
type AssertionRecord = {
|
|
3136
3237
|
/**
|
|
3137
|
-
* `
|
|
3138
|
-
*
|
|
3238
|
+
* `label` is the supplied `expect` label. Without one, it is the matcher plus its first
|
|
3239
|
+
* argument or named subject, cut at 60 characters. Repeats get ` #2`, ` #3`, and later suffixes.
|
|
3139
3240
|
*/
|
|
3140
3241
|
label: string;
|
|
3141
3242
|
/** The eval gave the label. */
|
|
@@ -3144,12 +3245,61 @@ type AssertionRecord = {
|
|
|
3144
3245
|
matcher: string;
|
|
3145
3246
|
/** `error`: the matcher couldn't answer (a subject it can't read, an event the harness doesn't report). */
|
|
3146
3247
|
status: 'pass' | 'fail' | 'error';
|
|
3248
|
+
/**
|
|
3249
|
+
* A failure didn't end the eval's body: an `expect`. False for `expect.require` and for a check
|
|
3250
|
+
* that threw, such as Python's `assert`, which end the body when they fail.
|
|
3251
|
+
*/
|
|
3147
3252
|
soft: boolean;
|
|
3253
|
+
/** `path:line` of the call in the eval file, relative to the project root, when reeval saw it. */
|
|
3254
|
+
at?: string;
|
|
3148
3255
|
/** Why it failed or couldn't answer, cut at 1 KB. */
|
|
3149
3256
|
message?: string;
|
|
3150
3257
|
/** Async matchers only (`toHaveFile`, `toSucceed`, `.resolves`, `expect.poll`). */
|
|
3151
3258
|
durationMs?: number;
|
|
3152
3259
|
};
|
|
3260
|
+
/**
|
|
3261
|
+
* A copy of the attempt's workspace, taken in the sandbox (never recorded into the history
|
|
3262
|
+
* folder), or why it wasn't.
|
|
3263
|
+
*/
|
|
3264
|
+
type Snapshot = {
|
|
3265
|
+
/** `final` after the checks, or `turn-<n>` after the nth `c.send`. */
|
|
3266
|
+
name: string;
|
|
3267
|
+
/** A plain directory, or a gzipped tarball with `snapshot: { compress: true }`. */
|
|
3268
|
+
form: 'dir' | 'tar.gz';
|
|
3269
|
+
/**
|
|
3270
|
+
* Where it is, relative to the run's directory in `.reeval/runs`:
|
|
3271
|
+
* `<eval>/<cell>/<n>/snapshots/<name>` (with `.tar.gz` when compressed); null when it wasn't
|
|
3272
|
+
* taken.
|
|
3273
|
+
*/
|
|
3274
|
+
path: string | null;
|
|
3275
|
+
/** Its size on disk: the directory's files, or the tarball; 0 when it wasn't taken. */
|
|
3276
|
+
bytes: number;
|
|
3277
|
+
/** Regular files and symbolic links in it. */
|
|
3278
|
+
files: number;
|
|
3279
|
+
/** Directories left out by name, such as `node_modules/` and `.git/`, at most 50. */
|
|
3280
|
+
excluded: string[];
|
|
3281
|
+
/** Text files whose secret-looking values were masked, at most 50. */
|
|
3282
|
+
masked: string[];
|
|
3283
|
+
/** Binary files left out because they held a secret-looking value, at most 50. */
|
|
3284
|
+
dropped: string[];
|
|
3285
|
+
/**
|
|
3286
|
+
* Why it wasn't taken: `size` when the workspace was over its form's cap, `error` when the
|
|
3287
|
+
* sandbox couldn't archive it; null when it was.
|
|
3288
|
+
*/
|
|
3289
|
+
skipped: {
|
|
3290
|
+
cause: 'size' | 'error';
|
|
3291
|
+
message: string;
|
|
3292
|
+
} | null;
|
|
3293
|
+
};
|
|
3294
|
+
/** `snapshot`: the shorthand, or the object that also picks the form. */
|
|
3295
|
+
type SnapshotOption = boolean | 'turns' | SnapshotOptions;
|
|
3296
|
+
/** `snapshot` as an object, merged field by field over the levels above. */
|
|
3297
|
+
type SnapshotOptions = {
|
|
3298
|
+
/** `true` after the checks, `'turns'` after each `c.send` too, `false` never. Default `true`. */
|
|
3299
|
+
when?: boolean | 'turns';
|
|
3300
|
+
/** Write each as `<name>.tar.gz` instead of a directory. Default `false`. */
|
|
3301
|
+
compress?: boolean;
|
|
3302
|
+
};
|
|
3153
3303
|
/** Which environment ran as the control, so the studio can show it. */
|
|
3154
3304
|
type EvalCompareMeta = {
|
|
3155
3305
|
baseline: string;
|
|
@@ -3160,16 +3310,20 @@ type EvalCompareMeta = {
|
|
|
3160
3310
|
from?: 'environment' | 'empty';
|
|
3161
3311
|
};
|
|
3162
3312
|
/**
|
|
3163
|
-
* A sandbox with more privilege than the default `reeval` user:
|
|
3164
|
-
* `docker({ capabilities })`. One per level that set such a sandbox,
|
|
3313
|
+
* A `SandboxWarning` describes a sandbox with more privilege than the default `reeval` user:
|
|
3314
|
+
* `docker({ root: true })` or `docker({ capabilities })`. One per level that set such a sandbox,
|
|
3315
|
+
* however many evaluations inherit it.
|
|
3165
3316
|
*/
|
|
3166
3317
|
type SandboxWarning = {
|
|
3167
3318
|
kind: 'root' | 'capabilities';
|
|
3168
3319
|
/** Which level set the sandbox. */
|
|
3169
3320
|
level: 'config' | 'profile' | 'group' | 'eval';
|
|
3170
|
-
/**
|
|
3321
|
+
/**
|
|
3322
|
+
* The value is the level as reeval names it: the `reeval.config.ts` file, `profiles.pr.defaults`,
|
|
3323
|
+
* `group("net")`, or `evaluate("installs deps")`.
|
|
3324
|
+
*/
|
|
3171
3325
|
where: string;
|
|
3172
|
-
/** The
|
|
3326
|
+
/** The evaluation file that sets it, for a group or evaluation; null for the config or a profile. */
|
|
3173
3327
|
file: string | null;
|
|
3174
3328
|
/** The capabilities added back, for `capabilities`. */
|
|
3175
3329
|
capabilities?: string[];
|
|
@@ -3185,5 +3339,5 @@ type ResolvedGateway = {
|
|
|
3185
3339
|
}>;
|
|
3186
3340
|
};
|
|
3187
3341
|
//#endregion
|
|
3188
|
-
export {
|
|
3189
|
-
//# sourceMappingURL=types-
|
|
3342
|
+
export { FakeScriptStep as $, Verdict as $t, CursorHarnessDescriptor as A, Spread as An, PluginPart as At, EnvironmentHelpers as B, effortLabel as Bn, SandboxDescriptor as Bt, ClaudeCodeHarnessDescriptor as C, DIRECTIONAL_MIN as Cn, OpenCodeHarnessDescriptor as Ct, CompareOptions as D, EffortMetric as Dn, PairVote as Dt, CompareCut as E, EffortContrast as En, PairFactor as Et, Duration as F, cellEffort as Fn, Question as Ft, EvalOptions as G, formatCount as Gn, SetupStep as Gt, EnvironmentSpec as H, effortSample as Hn, Secret as Ht, EgressPolicy as I, cellSamples as In, QuestionInputs as It, FakeAnalysisModel as J, tokenValues as Jn, TranscriptEvent as Jt, ExecResult as K, formatRatio as Kn, SkillSource as Kt, EgressRecord as L, contrastEffort as Ln, QuestionOptions as Lt, DirectGatewayDescriptor as M, VERDICT_TEXT as Mn, Profile as Mt, DirectoryOptions as N, bothUnderATenth as Nn, ProfileDefaults as Nt, ConfigHelpers as O, EffortSample as On, PiHarnessDescriptor as Ot, DockerSandboxDescriptor as P, cellContrast as Pn, ProfileLimit as Pt, FakeSandboxDescriptor as Q, VercelGatewayDescriptor as Qt, EnvVars as R, directional as Rn, ReevalConfig as Rt, ClaudeCodeEffort as S, CellEffort as Sn, McpServer as St, CompareAnswer as T, EFFORT_MIN_RUNS as Tn, Outcome as Tt, ErrorKind as U, effortSummary as Un, SecretTemplate as Ut, EnvironmentInput as V, effortOf as Vn, SandboxWarning as Vt, EvalFn as W, effortVerdict as Wn, ServeHandle as Wt, FakeHarnessDescriptor as X, DeclaredNode as Xn, TurnData as Xt, FakeAnswer as Y, DeclarationTree as Yn, Turn as Yt, FakeJudgeModel as Z, TurnStatus as Zt, CaptureUseScript as _, JudgmentRecord as _n, JudgeInputs as _t, AttemptContext as a, ArtifactRef as an, GrokBuildHarnessDescriptor as at, Cell as b, StackSummary as bn, JudgeState as bt, BlindPair as c, CheckTally as cn, HarnessDescriptor as ct, CaptureCall as d, FamilyLiftSummary as dn, HistoryConfig as dt, VertexGatewayDescriptor as en, FakeVote as et, CaptureInstruction as f, FamilyRate as fn, HostFile as ft, CaptureUseAction as g, HistoryEntry as gn, JudgeFailure as gt, CaptureUse as h, HistoryCell as hn, JudgeCallOptions as ht, AttemptAnswer as i, WebEventType as in, GooseHarnessDescriptor as it, Diff as j, TOKENS_UNREPORTED as jn, PluginSource as jt, ConfigJudgeOptions as k, EffortVerdict as kn, PinnableKey as kt, Budget as l, EvalEntry as ln, HarnessEdit as lt, CaptureOptions as m, FileRef as mn, JudgeArtifactRef as mt, AnalysisConfig as n, Viewport as nn, GatewayDescriptor as nt, BackgroundHandle as o, AttemptRecord as on, GroupOptions as ot, CaptureKey as p, FamilyRecord as pn, JSONValue as pt, ExecRetry as q, formatSeconds as qn, TokenUsage as qt, AntigravityHarnessDescriptor as r, Wait as rn, GatewayName as rt, BlindAttempt as s, CellRecord as sn, HarnessData as st, AgentHarnessDescriptor as t, VertexPublisher as tn, FxHarnessDescriptor as tt, Capture as u, FamilyLift as un, HarnessOption as ut, CaptureWait as v, Lift as vn, JudgeModel as vt, CodexHarnessDescriptor as w, EFFORT_ALPHA as wn, OpenRouterGatewayDescriptor as wt, CellRole as x, UpstreamCount as xn, JudgeSubject as xt, CaptureWindow as y, RunRecord as yn, JudgeOptions as yt, Environment as z, effortChangeLabel as zn, ResolvedGateway as zt };
|
|
3343
|
+
//# sourceMappingURL=types-DVvW66ci.d.ts.map
|