reeval 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -64
- package/THIRD_PARTY_NOTICES.md +288 -112
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +30 -30
- package/dist/adapter-Bwy3QxAK.js +279 -0
- package/dist/adapter-Bwy3QxAK.js.map +1 -0
- package/dist/{agent-6FefIhX7.js → agent-D483ru7D.js} +6 -6
- package/dist/{agent-6FefIhX7.js.map → agent-D483ru7D.js.map} +1 -1
- package/dist/{agent-WSmTaesf.js → agent-DpE80WPz.js} +46 -15
- package/dist/agent-DpE80WPz.js.map +1 -0
- package/dist/{analysis-DlGBDjTj.js → analysis-Bxub9LMT.js} +4 -4
- package/dist/{analysis-DlGBDjTj.js.map → analysis-Bxub9LMT.js.map} +1 -1
- package/dist/assemble-D5KGrwnJ.js +1 -0
- package/dist/{assemble-BCZzTyR7.js → assemble-D9GWr-HS.js} +69 -41
- package/dist/assemble-D9GWr-HS.js.map +1 -0
- package/dist/{client-Bbo9SPlK.js → assert-EUIoRdIt.js} +10 -113
- package/dist/assert-EUIoRdIt.js.map +1 -0
- package/dist/bridge/index.mjs +260 -60
- package/dist/{capture-Bp74Z6Fr.js → capture-BQlGHiMC.js} +146 -32
- package/dist/capture-BQlGHiMC.js.map +1 -0
- package/dist/capture-CYGLpWxR.js +1 -0
- package/dist/{cli-CVrMh_qz.js → cli-ClpPT8kn.js} +229 -537
- package/dist/cli-ClpPT8kn.js.map +1 -0
- package/dist/{cli-spec-CKEYa8V2.js → cli-spec-qmEnQ25_.js} +50 -22
- package/dist/cli-spec-qmEnQ25_.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/client-fcwgwOAD.js +106 -0
- package/dist/client-fcwgwOAD.js.map +1 -0
- package/dist/{command-DUT5l13I.js → command-Bj39y-Wm.js} +329 -87
- package/dist/command-Bj39y-Wm.js.map +1 -0
- package/dist/{command-BFP0hKEb.js → command-BlM0y8jN.js} +14 -14
- package/dist/{command-BFP0hKEb.js.map → command-BlM0y8jN.js.map} +1 -1
- package/dist/command-C-cVjlny.js +1 -0
- package/dist/{command-CzIFp1JQ.js → command-DbAe1Rzb.js} +19 -19
- package/dist/command-DbAe1Rzb.js.map +1 -0
- package/dist/{command-C5xFPKHM.js → command-TgbzPizb.js} +378 -74
- package/dist/command-TgbzPizb.js.map +1 -0
- package/dist/commands-CthqwkAn.js +1 -0
- package/dist/{commands-CLmPBrTS.js → commands-YUwlyTbr.js} +14 -11
- package/dist/commands-YUwlyTbr.js.map +1 -0
- package/dist/config-DrVSEVN0.js +774 -0
- package/dist/config-DrVSEVN0.js.map +1 -0
- package/dist/{context-DbSMEVKs.js → context-C1mxXh80.js} +11 -3
- package/dist/context-C1mxXh80.js.map +1 -0
- package/dist/contrasts-5zsj32gk.js +101 -0
- package/dist/contrasts-5zsj32gk.js.map +1 -0
- package/dist/db-CKg5NtiW.js +2 -0
- package/dist/{db-CfQFwJtq.js → db-DfhipHEF.js} +467 -38
- package/dist/db-DfhipHEF.js.map +1 -0
- package/dist/debug-D7bEv_UV.js +1 -0
- package/dist/{dist-CTzThfu_.js → dist-CuY4ehEk.js} +95 -15
- package/dist/dist-CuY4ehEk.js.map +1 -0
- package/dist/{docker-CRQGQbIt.js → docker-D1HZDS6q.js} +22 -21
- package/dist/docker-D1HZDS6q.js.map +1 -0
- package/dist/docker-DNmMPOBb.js +1 -0
- package/dist/{docker-DV2GNgle.js → docker-DOvjji9I.js} +3 -3
- package/dist/docker-DOvjji9I.js.map +1 -0
- package/dist/{doctor-DlKycx5H.js → doctor-D2MbNO8L.js} +262 -84
- package/dist/doctor-D2MbNO8L.js.map +1 -0
- package/dist/{egress-BhzNh7Yi.js → egress-2-qzPvGi.js} +146 -15
- package/dist/egress-2-qzPvGi.js.map +1 -0
- package/dist/environment-Cca6a4wB.js +1132 -0
- package/dist/environment-Cca6a4wB.js.map +1 -0
- package/dist/{errors-yDAEwmZo.js → errors-B6BSMs03.js} +2 -2
- package/dist/errors-B6BSMs03.js.map +1 -0
- package/dist/{errors-C9w_ud7g.js → errors-r025B0Zb.js} +4 -4
- package/dist/errors-r025B0Zb.js.map +1 -0
- package/dist/evaluate-DKIl54Hb.js +2348 -0
- package/dist/evaluate-DKIl54Hb.js.map +1 -0
- package/dist/evaluate-DMyaGQEj.d.ts +92 -0
- package/dist/files-BAt0i00X.js +1 -0
- package/dist/{files-DJUFg-4e.js → files-BXHZFWmx.js} +5 -5
- package/dist/{files-DJUFg-4e.js.map → files-BXHZFWmx.js.map} +1 -1
- package/dist/files-Bk1mC3AN.js +43 -0
- package/dist/files-Bk1mC3AN.js.map +1 -0
- package/dist/{files-CSzVADHn.js → files-BpUYBwGu.js} +11 -10
- package/dist/{files-CSzVADHn.js.map → files-BpUYBwGu.js.map} +1 -1
- package/dist/files-Bw8lWEjg.js +1 -0
- package/dist/files-D6J0iSdx.js +1 -0
- package/dist/{git-DfG-YJ1u.js → git-Crl3b87o.js} +2 -2
- package/dist/{git-DfG-YJ1u.js.map → git-Crl3b87o.js.map} +1 -1
- package/dist/harness-CEuntrla.js +323 -0
- package/dist/harness-CEuntrla.js.map +1 -0
- package/dist/harness.d.ts +16 -30
- package/dist/harness.js +1 -2
- package/dist/{header-CALhmafp.js → header-DHQv_ZrF.js} +2 -2
- package/dist/{header-CALhmafp.js.map → header-DHQv_ZrF.js.map} +1 -1
- package/dist/{images-BMgdpMAX.js → images-Bi3aGxb9.js} +114 -45
- package/dist/images-Bi3aGxb9.js.map +1 -0
- package/dist/index.d.ts +9 -37
- package/dist/index.js +6 -2138
- package/dist/{init-BIdLF0F9.js → init-Cl3f3NBT.js} +26 -8
- package/dist/init-Cl3f3NBT.js.map +1 -0
- package/dist/init-fhOXED8F.js +1 -0
- package/dist/{invocations-CLaL8AuW.js → invocations-BYDPe3fV.js} +4 -4
- package/dist/{invocations-CLaL8AuW.js.map → invocations-BYDPe3fV.js.map} +1 -1
- package/dist/{judge-CeYPN8P_.js → judge-DW_GDOmB.js} +11 -19
- package/dist/{judge-CeYPN8P_.js.map → judge-DW_GDOmB.js.map} +1 -1
- package/dist/judge.d.ts +2 -2
- package/dist/judge.js +1 -1
- package/dist/{live-S0qODb6o.js → live-jlfszAbb.js} +9 -20
- package/dist/live-jlfszAbb.js.map +1 -0
- package/dist/matchers-Ddmc2z7H.js +118 -0
- package/dist/matchers-Ddmc2z7H.js.map +1 -0
- package/dist/model-CXhrziBe.js +1 -0
- package/dist/{model-B1BZoZ9b.js → model-D0IcOQao.js} +22 -20
- package/dist/model-D0IcOQao.js.map +1 -0
- package/dist/{model-DEQo3FkZ.js → model-DaVCY3rb.js} +2 -2
- package/dist/{model-DEQo3FkZ.js.map → model-DaVCY3rb.js.map} +1 -1
- package/dist/{model-BOnlAPzs.js → model-DaX4bNLZ.js} +123 -22
- package/dist/model-DaX4bNLZ.js.map +1 -0
- package/dist/model-_bxNrk2h.js +1 -0
- package/dist/{node-Co7EXge0.js → node-B3ydHo6t.js} +10 -47
- package/dist/node-B3ydHo6t.js.map +1 -0
- package/dist/object-CLf2-tNq.js +9 -0
- package/dist/object-CLf2-tNq.js.map +1 -0
- package/dist/{opencode-providers-ByD85Q0l.js → opencode-providers-J9colP0-.js} +96 -21
- package/dist/opencode-providers-J9colP0-.js.map +1 -0
- package/dist/options-C0d0AzeR.js +1118 -0
- package/dist/options-C0d0AzeR.js.map +1 -0
- package/dist/{pairs-DDsjgm5F.js → pairs-CnInRu-G.js} +76 -2
- package/dist/{pairs-DDsjgm5F.js.map → pairs-CnInRu-G.js.map} +1 -1
- package/dist/{patch-bDV8YFW0.js → patch-B1bnfhN-.js} +6 -6
- package/dist/{patch-bDV8YFW0.js.map → patch-B1bnfhN-.js.map} +1 -1
- package/dist/{plan-DUydFc_1.js → plan-DgQxp9yy.js} +34 -13
- package/dist/plan-DgQxp9yy.js.map +1 -0
- package/dist/{preflight-BkoSxf4C.js → preflight-BRzD30L5.js} +8 -7
- package/dist/preflight-BRzD30L5.js.map +1 -0
- package/dist/preflight-DlQvbySb.js +1 -0
- package/dist/{prepare-CUAWHebF.js → prepare-B_y-AFMv.js} +38 -30
- package/dist/prepare-B_y-AFMv.js.map +1 -0
- package/dist/prepare-DCKTLHN3.js +2 -0
- package/dist/{provenance-D7HK3vzt.js → provenance-DQ0AXn6c.js} +5 -5
- package/dist/{provenance-D7HK3vzt.js.map → provenance-DQ0AXn6c.js.map} +1 -1
- package/dist/provider-DehR4CMN.js +1 -0
- package/dist/providers-B9yTdIqF.js +1 -0
- package/dist/providers-DoTq1jHW.js +165 -0
- package/dist/providers-DoTq1jHW.js.map +1 -0
- package/dist/query-BO-ux7RB.js +139 -0
- package/dist/query-BO-ux7RB.js.map +1 -0
- package/dist/query-CRQfsw0e.js +1 -0
- package/dist/questions-C2dym2ce.js +524 -0
- package/dist/questions-C2dym2ce.js.map +1 -0
- package/dist/{record-BRP9vZrj.js → record-BKmMvZ6O.js} +9 -1
- package/dist/{record-BRP9vZrj.js.map → record-BKmMvZ6O.js.map} +1 -1
- package/dist/remote-CPPW86Q9.js +378 -0
- package/dist/remote-CPPW86Q9.js.map +1 -0
- package/dist/{results-Di7m8dQ3.js → results-BE2f-1XS.js} +6 -5
- package/dist/results-BE2f-1XS.js.map +1 -0
- package/dist/{resume-SUz1GskV.js → resume-Ca1c8vix.js} +10 -10
- package/dist/{resume-SUz1GskV.js.map → resume-Ca1c8vix.js.map} +1 -1
- package/dist/{root-CvdRlRZX.js → root-DICDHCtp.js} +4 -4
- package/dist/{root-CvdRlRZX.js.map → root-DICDHCtp.js.map} +1 -1
- package/dist/{runtime-DdsfnRz_.js → runtime-PX_oXiCS.js} +23 -71
- package/dist/runtime-PX_oXiCS.js.map +1 -0
- package/dist/sandbox-DLrsvu7U.js +70 -0
- package/dist/sandbox-DLrsvu7U.js.map +1 -0
- package/dist/sandbox.d.ts +19 -6
- package/dist/sandbox.js +1 -2
- package/dist/server-Dp8eOeK5.js +1 -0
- package/dist/{server-BNnXtmS8.js → server-TUY5xA_A.js} +11 -11
- package/dist/server-TUY5xA_A.js.map +1 -0
- package/dist/shared-CA61BuPJ.js +188 -0
- package/dist/shared-CA61BuPJ.js.map +1 -0
- package/dist/{source-Dof0NQBm.js → source-DJGS0a0u.js} +2 -2
- package/dist/{source-Dof0NQBm.js.map → source-DJGS0a0u.js.map} +1 -1
- package/dist/source-DdbLGR1M.js +1 -0
- package/dist/src-BxfCKA3_.js +2 -0
- package/dist/{src-C1E9HDWO.js → src-C7_1SqDP.js} +3 -3
- package/dist/{src-C1E9HDWO.js.map → src-C7_1SqDP.js.map} +1 -1
- package/dist/{src-DDNXsD9H.js → src-CF17Ug08.js} +5 -4
- package/dist/src-CF17Ug08.js.map +1 -0
- package/dist/{src-C_4Umgh2.js → src-CFp86GtV.js} +52 -51
- package/dist/src-CFp86GtV.js.map +1 -0
- package/dist/src-CLoOMVWI.js +3 -0
- package/dist/src-CcrQbpG8.js +2 -0
- package/dist/src-DCFtJRp0.js +2 -0
- package/dist/src-DIL0jST0.js +368 -0
- package/dist/src-DIL0jST0.js.map +1 -0
- package/dist/src-DSYYPEOj.js +2 -0
- package/dist/src-DYO3DdnQ.js +1 -0
- package/dist/src-DvCg0NBs.js +324 -0
- package/dist/src-DvCg0NBs.js.map +1 -0
- package/dist/{src-COgE7991.js → src-DvK0EIZx.js} +5 -5
- package/dist/{src-COgE7991.js.map → src-DvK0EIZx.js.map} +1 -1
- package/dist/{src-DVy58-zZ.js → src-FGXhkEkH.js} +5 -5
- package/dist/{src-DVy58-zZ.js.map → src-FGXhkEkH.js.map} +1 -1
- package/dist/src-WD-zgQYZ.js +2 -0
- package/dist/src-_oI9AePb.js +2 -0
- package/dist/{src-8rRXy4u9.js → src-ftu5wPSt.js} +1652 -182
- package/dist/src-ftu5wPSt.js.map +1 -0
- package/dist/src-qsKCqv7a.js +2 -0
- package/dist/{src-D95O-Smv.js → src-u8_ixn_F.js} +15 -11
- package/dist/src-u8_ixn_F.js.map +1 -0
- package/dist/state-CeR6oz0k.js +66 -0
- package/dist/state-CeR6oz0k.js.map +1 -0
- package/dist/{state-CiXyLqg0.js → state-DfpHox6b.js} +36 -24
- package/dist/state-DfpHox6b.js.map +1 -0
- package/dist/state-DpAxYyED.js +1 -0
- package/dist/studio-app/assets/index-CBkhNAv8.js +58 -0
- package/dist/studio-app/assets/index-CwQN3f_h.css +2 -0
- package/dist/studio-app/index.html +2 -2
- package/dist/suite-Cw_7G2RM.js +429 -0
- package/dist/suite-Cw_7G2RM.js.map +1 -0
- package/dist/suite-DamAEnNF.js +1 -0
- package/dist/summary-BUKfU-9l.js +611 -0
- package/dist/summary-BUKfU-9l.js.map +1 -0
- package/dist/{summary-2OAWF2N7.js → summary-Baa4rTHS.js} +7 -7
- package/dist/{summary-2OAWF2N7.js.map → summary-Baa4rTHS.js.map} +1 -1
- package/dist/testing.d.ts +3 -2
- package/dist/testing.js +3 -2
- package/dist/{tool-target-BU1JnSE9.js → tool-target-D2mcYlSQ.js} +14 -7
- package/dist/tool-target-D2mcYlSQ.js.map +1 -0
- package/dist/{tools-B10Uj7OI.js → tools-DLHgetpx.js} +6 -6
- package/dist/tools-DLHgetpx.js.map +1 -0
- package/dist/transcript-BNNANU8A.js +64 -0
- package/dist/transcript-BNNANU8A.js.map +1 -0
- package/dist/{types-D8RLfkFP.d.ts → types-BYdlwMwo.d.ts} +398 -46
- package/dist/{usage-DDF_6tkV.js → usage-CleNKkly.js} +4 -2
- package/dist/{usage-DDF_6tkV.js.map → usage-CleNKkly.js.map} +1 -1
- package/dist/view/index.d.ts +290 -8
- package/dist/view/index.js +7 -7
- package/dist/{views-Dqa5ojfZ.js → views-B8w2J-gD.js} +19 -24
- package/dist/views-B8w2J-gD.js.map +1 -0
- package/dist/views-DblnI3kA.js +2 -0
- package/dist/{worker-entry-BAikHl9F.js → worker-entry-DNnveLqx.js} +16 -229
- package/dist/worker-entry-DNnveLqx.js.map +1 -0
- package/dist/{workspace-C-TrNIUc.js → workspace-C4RA3srV.js} +7 -7
- package/dist/{workspace-C-TrNIUc.js.map → workspace-C4RA3srV.js.map} +1 -1
- package/dist/workspace-C_X0tdv6.js +1 -0
- package/package.json +12 -12
- package/dist/agent-WSmTaesf.js.map +0 -1
- package/dist/assemble-BCZzTyR7.js.map +0 -1
- package/dist/assemble-C5qq5yT2.js +0 -2
- package/dist/capture-Bp74Z6Fr.js.map +0 -1
- package/dist/capture-DS-cPyuE.js +0 -2
- package/dist/cli-CVrMh_qz.js.map +0 -1
- package/dist/cli-spec-CKEYa8V2.js.map +0 -1
- package/dist/client-Bbo9SPlK.js.map +0 -1
- package/dist/command-BrmXUC-W.js +0 -2
- package/dist/command-C5xFPKHM.js.map +0 -1
- package/dist/command-CzIFp1JQ.js.map +0 -1
- package/dist/command-DUT5l13I.js.map +0 -1
- package/dist/commands-CLmPBrTS.js.map +0 -1
- package/dist/commands-fI3G2Wnh.js +0 -2
- package/dist/config-CGvBeewH.js +0 -457
- package/dist/config-CGvBeewH.js.map +0 -1
- package/dist/context-DbSMEVKs.js.map +0 -1
- package/dist/db-CSSFtGcN.js +0 -3
- package/dist/db-CfQFwJtq.js.map +0 -1
- package/dist/debug-SEmW4w8f.js +0 -2
- package/dist/dist-CTzThfu_.js.map +0 -1
- package/dist/docker-CRQGQbIt.js.map +0 -1
- package/dist/docker-DV2GNgle.js.map +0 -1
- package/dist/docker-Df_YOxxB.js +0 -2
- package/dist/doctor-DlKycx5H.js.map +0 -1
- package/dist/duration-JsJrIWXq.js +0 -26
- package/dist/duration-JsJrIWXq.js.map +0 -1
- package/dist/egress-BhzNh7Yi.js.map +0 -1
- package/dist/env-BQqj93gu.js +0 -21
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-1jE-HhO-.js +0 -657
- package/dist/environment-1jE-HhO-.js.map +0 -1
- package/dist/errors-C9w_ud7g.js.map +0 -1
- package/dist/errors-yDAEwmZo.js.map +0 -1
- package/dist/fake-instructions-CUI33I_F.js +0 -32
- package/dist/fake-instructions-CUI33I_F.js.map +0 -1
- package/dist/family-D2b7WD10.js +0 -67
- package/dist/family-D2b7WD10.js.map +0 -1
- package/dist/files-AiEKuEZe.js +0 -2
- package/dist/files-BM78_nnF.js +0 -2
- package/dist/files-BMCvfB4X.js +0 -2
- package/dist/harness-CeCp2GSW.js +0 -171
- package/dist/harness-CeCp2GSW.js.map +0 -1
- package/dist/images-BMgdpMAX.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/init-B6vegaP_.js +0 -2
- package/dist/init-BIdLF0F9.js.map +0 -1
- package/dist/jiti-BcJHCtrb.js +0 -46
- package/dist/jiti-BcJHCtrb.js.map +0 -1
- package/dist/live-S0qODb6o.js.map +0 -1
- package/dist/model-B1BZoZ9b.js.map +0 -1
- package/dist/model-BOnlAPzs.js.map +0 -1
- package/dist/model-DWeg5J7z.js +0 -2
- package/dist/model-FXko2kl7.js +0 -2
- package/dist/node-Co7EXge0.js.map +0 -1
- package/dist/opencode-providers-ByD85Q0l.js.map +0 -1
- package/dist/options-CKqyPGxX.js +0 -739
- package/dist/options-CKqyPGxX.js.map +0 -1
- package/dist/plan-DUydFc_1.js.map +0 -1
- package/dist/preflight-BkoSxf4C.js.map +0 -1
- package/dist/preflight-D5LW7Oqi.js +0 -2
- package/dist/prepare-CUAWHebF.js.map +0 -1
- package/dist/prepare-oji2oO8W.js +0 -3
- package/dist/process-CSVO_jsc.js +0 -119
- package/dist/process-CSVO_jsc.js.map +0 -1
- package/dist/provider-BDbNOQhQ.js +0 -2
- package/dist/providers-B1DzMvmH.js +0 -107
- package/dist/providers-B1DzMvmH.js.map +0 -1
- package/dist/providers-CpEfNU1F.js +0 -2
- package/dist/query-BjBMLa-C.js +0 -91
- package/dist/query-BjBMLa-C.js.map +0 -1
- package/dist/query-oPO_XrTy.js +0 -2
- package/dist/questions-B8by4Bbt.js +0 -234
- package/dist/questions-B8by4Bbt.js.map +0 -1
- package/dist/results-Di7m8dQ3.js.map +0 -1
- package/dist/runtime-DdsfnRz_.js.map +0 -1
- package/dist/sandbox-D5v-of6d.js +0 -139
- package/dist/sandbox-D5v-of6d.js.map +0 -1
- package/dist/server-BNnXtmS8.js.map +0 -1
- package/dist/server-BNwoKzlm.js +0 -2
- package/dist/source-Bk_0ZaEY.js +0 -2
- package/dist/src-8rRXy4u9.js.map +0 -1
- package/dist/src-BGWuWp6I.js +0 -3
- package/dist/src-BRt0O02i.js +0 -3
- package/dist/src-Bx2x1ekD.js +0 -3
- package/dist/src-C_4Umgh2.js.map +0 -1
- package/dist/src-D95O-Smv.js.map +0 -1
- package/dist/src-DDNXsD9H.js.map +0 -1
- package/dist/src-DLa-ESW0.js +0 -4
- package/dist/src-cN0aB-WR.js +0 -3
- package/dist/src-cdIVjC5b.js +0 -2
- package/dist/src-sIsDHWYq.js +0 -3
- package/dist/state-Bo5zg4da.js +0 -2
- package/dist/state-CiXyLqg0.js.map +0 -1
- package/dist/studio-app/assets/index-B9MCTfuu.css +0 -2
- package/dist/studio-app/assets/index-CROH2NB3.js +0 -58
- package/dist/summary-BTksrIxn.js +0 -254
- package/dist/summary-BTksrIxn.js.map +0 -1
- package/dist/tool-target-BU1JnSE9.js.map +0 -1
- package/dist/tools-B10Uj7OI.js.map +0 -1
- package/dist/views-Cs2Oqayd.js +0 -3
- package/dist/views-Dqa5ojfZ.js.map +0 -1
- package/dist/worker-entry-BAikHl9F.js.map +0 -1
- package/dist/workspace-Do_SiWBs.js +0 -2
|
@@ -41,6 +41,10 @@ type EffortContrast = {
|
|
|
41
41
|
agentMsP: number | null;
|
|
42
42
|
/** The variant's and control's tool-call ranges don't overlap. */
|
|
43
43
|
separated: boolean;
|
|
44
|
+
/** Their agent-time ranges don't overlap. */
|
|
45
|
+
timeSeparated: boolean;
|
|
46
|
+
/** Their token ranges don't overlap; null when the token test was omitted. */
|
|
47
|
+
tokensSeparated: boolean | null;
|
|
44
48
|
/** On tool calls, the steadier measure; `timeVerdict` on agent time. */
|
|
45
49
|
verdict: EffortVerdict;
|
|
46
50
|
timeVerdict: EffortVerdict;
|
|
@@ -67,6 +71,21 @@ declare function effortVerdict(xs: number[], ys: number[]): {
|
|
|
67
71
|
p: number | null;
|
|
68
72
|
verdict: EffortVerdict;
|
|
69
73
|
};
|
|
74
|
+
/**
|
|
75
|
+
* The fewest attempts a side for "no overlap" to say anything: at 3 against 3 one ordering in
|
|
76
|
+
* ten separates by chance; at 1 against 1 every pair of unequal values does.
|
|
77
|
+
*/
|
|
78
|
+
declare const DIRECTIONAL_MIN = 3;
|
|
79
|
+
/**
|
|
80
|
+
* The fewest runs per cell whose effort test can reach `EFFORT_ALPHA` (4 against 4: p = 2/70).
|
|
81
|
+
*/
|
|
82
|
+
declare const EFFORT_MIN_RUNS: number;
|
|
83
|
+
type EffortMetric = 'tools' | 'agentS' | 'tokens';
|
|
84
|
+
/**
|
|
85
|
+
* The metrics whose test is `insufficient` but whose ranges don't overlap at
|
|
86
|
+
* `DIRECTIONAL_MIN` a side or more: a direction, not an effect.
|
|
87
|
+
*/
|
|
88
|
+
declare function directional(c: EffortContrast): EffortMetric[];
|
|
70
89
|
declare function contrastEffort(variant: EffortSample[], control: EffortSample[]): EffortContrast | null;
|
|
71
90
|
/** The counted attempts of one cell of an eval, as effort samples. */
|
|
72
91
|
declare const cellSamples: (e: Pick<EvalEntry, 'attempts'>, cellKey: string) => EffortSample[];
|
|
@@ -76,13 +95,17 @@ declare function cellContrast(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey:
|
|
|
76
95
|
/** `4`, `4.5`: medians of counts are whole or halves. */
|
|
77
96
|
declare const formatCount: (n: number) => string;
|
|
78
97
|
declare const formatRatio: (r: number | null) => string;
|
|
79
|
-
/**
|
|
98
|
+
/** Two agent times that both print `<0.1s` have no ratio worth printing. */
|
|
99
|
+
declare const bothUnderATenth: (a: number, b: number) => boolean;
|
|
100
|
+
/** Seconds for agent time: `<0.1s`, `0.4s`, `21s`, `1m 05s`. */
|
|
80
101
|
declare function formatSeconds(ms: number): string;
|
|
81
102
|
declare const VERDICT_TEXT: Record<EffortVerdict, (what: string) => string>;
|
|
82
103
|
/**
|
|
83
104
|
* `tools 5 vs 43 (×0.11), agent 21s vs 46s (×0.47): too few attempts to test` for a variant
|
|
84
105
|
* against its control; with enough attempts, `fewer tool calls (p = 0.029)`. Tokens fold in
|
|
85
106
|
* when both sides have usage samples; they are omitted (not `insufficient`) when either does not.
|
|
107
|
+
* A metric too small to test whose ranges don't overlap says so: `(×0.37, no overlap)`. Agent
|
|
108
|
+
* times both under a tenth of a second have no ratio: `agent <0.1s vs <0.1s`.
|
|
86
109
|
*/
|
|
87
110
|
declare function effortLabel(c: EffortContrast): string;
|
|
88
111
|
/** One cell's effort on its own: `tools 43 · agent 46s`, plus tokens when any attempt reported them. */
|
|
@@ -281,6 +304,10 @@ type AttemptRecord = {
|
|
|
281
304
|
webSearches: number | null;
|
|
282
305
|
webFetches: number | null;
|
|
283
306
|
reasoningMs: number;
|
|
307
|
+
/** Transcript tool events with `stalled: true` (long shell or edit). */
|
|
308
|
+
stalledTools: number;
|
|
309
|
+
/** Longest tool `durationMs` in the transcript; null when none reported a duration. */
|
|
310
|
+
maxToolMs: number | null;
|
|
284
311
|
};
|
|
285
312
|
judgements: JudgementRecord[];
|
|
286
313
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
@@ -335,7 +362,9 @@ type EvalEntry = {
|
|
|
335
362
|
* The compare's questions, answered after its attempts by pairwise votes, one per question;
|
|
336
363
|
* unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
337
364
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
338
|
-
* (rates by cut, votes, ties, a failed call, a sample of reasons).
|
|
365
|
+
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
366
|
+
* `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
|
|
367
|
+
* masked and screenshots as refs; unset where the eval's files weren't kept.
|
|
339
368
|
*/
|
|
340
369
|
answers?: CompareAnswer[];
|
|
341
370
|
/** Set on each case of a pooled `evaluate.each`; `RunRecord.families` has the suite. */
|
|
@@ -358,7 +387,7 @@ type FamilyInterval = {
|
|
|
358
387
|
*/
|
|
359
388
|
method: 'wilson' | 'newcombe-paired' | 't-cases';
|
|
360
389
|
};
|
|
361
|
-
/** One cell's pass
|
|
390
|
+
/** One cell's pass@1 over a family's cases. */
|
|
362
391
|
type FamilyRate = {
|
|
363
392
|
/** The cell's key, the same in every case. */
|
|
364
393
|
cell: string;
|
|
@@ -479,7 +508,7 @@ type FingerprintParts = {
|
|
|
479
508
|
environment: string | null;
|
|
480
509
|
image: string | null;
|
|
481
510
|
agentCli: string | null;
|
|
482
|
-
/** Over the judge's model, threshold and version. */
|
|
511
|
+
/** Over the judge's model, threshold and version, and the config's `state` when it sets one. */
|
|
483
512
|
judge: string;
|
|
484
513
|
/** Over the attempt's context files. */
|
|
485
514
|
context: string;
|
|
@@ -531,9 +560,10 @@ type CheckTally = {
|
|
|
531
560
|
notRun: number;
|
|
532
561
|
};
|
|
533
562
|
/**
|
|
534
|
-
* One check over a cell's counted attempts. `verdict` (variant cells)
|
|
535
|
-
* passes on every
|
|
536
|
-
* when it does the same on both sides, so it can't tell them
|
|
563
|
+
* One check over a cell's counted attempts. `verdict` (variant cells), over the attempts that
|
|
564
|
+
* reached the check: `discriminates` when it passes on every one of one side and none of the
|
|
565
|
+
* other; `both-pass` or `both-fail` when it does the same on both sides, so it can't tell them
|
|
566
|
+
* apart; `mixed` otherwise. No verdict when a side never reached it: not run is not a fail.
|
|
537
567
|
*/
|
|
538
568
|
type CellCheck = CheckTally & {
|
|
539
569
|
label: string;
|
|
@@ -563,7 +593,7 @@ type Cost = {
|
|
|
563
593
|
source: 'reported' | 'unreported';
|
|
564
594
|
};
|
|
565
595
|
type RunRecord = {
|
|
566
|
-
kind: '
|
|
596
|
+
kind: 'reeval-run';
|
|
567
597
|
schemaVersion: typeof SCHEMA_VERSION;
|
|
568
598
|
/** Also the file name: `<UTC start>_<short sha>_<author or ci>`. */
|
|
569
599
|
runId: string;
|
|
@@ -644,7 +674,8 @@ type RunRecord = {
|
|
|
644
674
|
};
|
|
645
675
|
/** What ran and graded the agents: the machine, images, harnesses and judge. */
|
|
646
676
|
type Stack = {
|
|
647
|
-
|
|
677
|
+
/** The reeval version that wrote the run. */
|
|
678
|
+
reeval: string;
|
|
648
679
|
/** `process.versions.node`: under Bun or Deno, the Node version they're compatible with. */
|
|
649
680
|
node: string;
|
|
650
681
|
/**
|
|
@@ -710,6 +741,8 @@ type Stack = {
|
|
|
710
741
|
threshold: number;
|
|
711
742
|
/** Changes when reeval, the AI SDK or the provider adapter (and so the judge prompt) change. */
|
|
712
743
|
version: string;
|
|
744
|
+
/** What the judge sees by default, every key resolved; absent when the config and profile leave `state` at its defaults. */
|
|
745
|
+
state?: ResolvedJudgeStateOptions;
|
|
713
746
|
};
|
|
714
747
|
/** Union of every attempt's context files. */
|
|
715
748
|
context: ContextFile[];
|
|
@@ -801,7 +834,8 @@ type FileEditEvent = {
|
|
|
801
834
|
};
|
|
802
835
|
/**
|
|
803
836
|
* Normalised agent events. Each tool call is one event, merged with its result; `durationMs`
|
|
804
|
-
* is
|
|
837
|
+
* is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
|
|
838
|
+
* when the harness never reported the end. Long shell or edit calls set `stalled: true`.
|
|
805
839
|
*/
|
|
806
840
|
type TranscriptEvent = EventTime & ({
|
|
807
841
|
type: 'text' | 'reasoning';
|
|
@@ -809,6 +843,8 @@ type TranscriptEvent = EventTime & ({
|
|
|
809
843
|
durationMs: number | null;
|
|
810
844
|
} | (FileEditEvent & {
|
|
811
845
|
durationMs: number | null;
|
|
846
|
+
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
847
|
+
stalled?: boolean;
|
|
812
848
|
}) | {
|
|
813
849
|
type: 'web-search';
|
|
814
850
|
toolCallId: string;
|
|
@@ -818,12 +854,16 @@ type TranscriptEvent = EventTime & ({
|
|
|
818
854
|
/** Whether the search succeeded. Absent means it did. */
|
|
819
855
|
ok?: boolean;
|
|
820
856
|
durationMs: number | null;
|
|
857
|
+
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
858
|
+
stalled?: boolean;
|
|
821
859
|
} | {
|
|
822
860
|
type: 'web-fetch';
|
|
823
861
|
toolCallId: string;
|
|
824
862
|
url: string;
|
|
825
863
|
ok: boolean;
|
|
826
864
|
durationMs: number | null;
|
|
865
|
+
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
866
|
+
stalled?: boolean;
|
|
827
867
|
} | {
|
|
828
868
|
type: 'tool';
|
|
829
869
|
toolCallId: string;
|
|
@@ -832,6 +872,8 @@ type TranscriptEvent = EventTime & ({
|
|
|
832
872
|
output: unknown;
|
|
833
873
|
ok: boolean;
|
|
834
874
|
durationMs: number | null;
|
|
875
|
+
/** True when a shell-like tool or edit ran at least two minutes. */
|
|
876
|
+
stalled?: boolean;
|
|
835
877
|
} | {
|
|
836
878
|
type: 'error';
|
|
837
879
|
message: string;
|
|
@@ -850,7 +892,9 @@ type ExecResult = {
|
|
|
850
892
|
};
|
|
851
893
|
/** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
|
|
852
894
|
type CaptureWindow = {
|
|
895
|
+
/** In CSS pixels. */
|
|
853
896
|
width: number;
|
|
897
|
+
/** In CSS pixels. */
|
|
854
898
|
height: number;
|
|
855
899
|
};
|
|
856
900
|
/**
|
|
@@ -912,6 +956,7 @@ type CaptureUse = {
|
|
|
912
956
|
* pass it as the first argument to `c.capture`.
|
|
913
957
|
*/
|
|
914
958
|
type CaptureOptions = {
|
|
959
|
+
/** The window and saved PNG size, or `'full-page'`. Default 1280×720. */
|
|
915
960
|
viewport?: Viewport;
|
|
916
961
|
/** How long to poll before failing. Default `'60s'`. */
|
|
917
962
|
timeout?: Duration;
|
|
@@ -1017,6 +1062,7 @@ type Turn = TurnData & {
|
|
|
1017
1062
|
exec(command: string): Promise<ExecResult>;
|
|
1018
1063
|
readFile(path: string): Promise<string | null>;
|
|
1019
1064
|
};
|
|
1065
|
+
/** Any JSON value. */
|
|
1020
1066
|
type JSONValue = null | boolean | number | string | JSONValue[] | {
|
|
1021
1067
|
[key: string]: JSONValue;
|
|
1022
1068
|
};
|
|
@@ -1053,10 +1099,11 @@ type JudgeArtifactRef = {
|
|
|
1053
1099
|
*/
|
|
1054
1100
|
type FakeJudgeModel = {
|
|
1055
1101
|
kind: 'fake-judge';
|
|
1102
|
+
/** The score (0–1) every call gets, or a function of the state it would see. */
|
|
1056
1103
|
score: number | ((state: JudgeState) => number);
|
|
1057
1104
|
};
|
|
1058
1105
|
/**
|
|
1059
|
-
* `'provider/model'` (e.g. `'typesafe-ai/jev-1.
|
|
1106
|
+
* `'provider/model'` (e.g. `'typesafe-ai/jev-1.14.0'`, `'google/gemini-3.8-flash'`), an AI SDK
|
|
1060
1107
|
* decision model object (per group, eval or call only), or a fake judge.
|
|
1061
1108
|
*/
|
|
1062
1109
|
type JudgeModel = string | FakeJudgeModel | {
|
|
@@ -1064,28 +1111,77 @@ type JudgeModel = string | FakeJudgeModel | {
|
|
|
1064
1111
|
};
|
|
1065
1112
|
/** Merged field by field down the cascade, so per-model settings can join as new fields. */
|
|
1066
1113
|
type JudgeOptions = {
|
|
1114
|
+
/** The model that grades. */
|
|
1067
1115
|
model?: JudgeModel;
|
|
1068
1116
|
/** Minimum score (0–1) for a `pass` verdict. */
|
|
1069
1117
|
threshold?: number;
|
|
1118
|
+
/** AI SDK provider options for the judge's calls, keyed by provider. */
|
|
1070
1119
|
providerOptions?: Record<string, Record<string, JSONValue>>;
|
|
1071
1120
|
/** The most judge calls to this model's provider at once, across the run. */
|
|
1072
1121
|
maxConcurrent?: number;
|
|
1073
1122
|
/**
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1123
|
+
* The judge's state: what the judge is given beside the question, not the workspace's or the
|
|
1124
|
+
* attempt's state. Merged key by key down the cascade.
|
|
1076
1125
|
*/
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1126
|
+
state?: JudgeStateOptions;
|
|
1127
|
+
};
|
|
1128
|
+
/**
|
|
1129
|
+
* What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
|
|
1130
|
+
* keeps its default, which is what the judge gets with no `state` option at all.
|
|
1131
|
+
*/
|
|
1132
|
+
type JudgeStateOptions = {
|
|
1133
|
+
/** The graded turn's prompt. Default true. */
|
|
1134
|
+
task?: boolean;
|
|
1135
|
+
/** The graded turn's final message, or `on`'s value. Default true. */
|
|
1136
|
+
output?: boolean;
|
|
1137
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. Default true. */
|
|
1138
|
+
files?: boolean;
|
|
1139
|
+
/**
|
|
1140
|
+
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
1141
|
+
* the last matching pattern wins and `!` excludes. Default true.
|
|
1142
|
+
*/
|
|
1143
|
+
diff?: boolean | string[];
|
|
1144
|
+
/**
|
|
1145
|
+
* Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style
|
|
1146
|
+
* patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in
|
|
1147
|
+
* the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing.
|
|
1148
|
+
* These files leave the diff. Default none.
|
|
1149
|
+
*/
|
|
1150
|
+
contents?: string[];
|
|
1151
|
+
/**
|
|
1152
|
+
* The graded turn's transcript as text lines: `true` is messages and tool calls with tool
|
|
1153
|
+
* output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output.
|
|
1154
|
+
* Default false.
|
|
1155
|
+
*/
|
|
1156
|
+
transcript?: boolean | 'tools' | 'full';
|
|
1157
|
+
/**
|
|
1158
|
+
* `'all'`: every turn up to the graded one under `turns`, each with its task, output and (with
|
|
1159
|
+
* `transcript`) transcript. Default `'last'`.
|
|
1160
|
+
*/
|
|
1161
|
+
turns?: 'last' | 'all';
|
|
1162
|
+
/** `c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
1163
|
+
checks?: boolean | string[];
|
|
1164
|
+
};
|
|
1165
|
+
/** `JudgeStateOptions` with every key set: what one judge call saw. */
|
|
1166
|
+
type ResolvedJudgeStateOptions = {
|
|
1167
|
+
task: boolean;
|
|
1168
|
+
output: boolean;
|
|
1169
|
+
files: boolean;
|
|
1170
|
+
diff: boolean | string[];
|
|
1171
|
+
contents: string[];
|
|
1172
|
+
transcript: boolean | 'tools' | 'full';
|
|
1173
|
+
turns: 'last' | 'all';
|
|
1174
|
+
checks: boolean | string[];
|
|
1080
1175
|
};
|
|
1081
1176
|
type JudgeCallOptions = JudgeOptions & {
|
|
1082
1177
|
/** What to grade instead of the latest turn: another turn, or a value used as the output. */
|
|
1083
1178
|
on?: Turn | JSONValue;
|
|
1084
1179
|
/**
|
|
1085
|
-
* Captures this call sends the judge as file parts
|
|
1086
|
-
* Call-only: the bytes are on this attempt.
|
|
1180
|
+
* Captures this call sends the judge as file parts: `c.capture()` handles or their names, or
|
|
1181
|
+
* `true` for every capture so far. Call-only: the bytes are on this attempt. Jev takes PNG
|
|
1182
|
+
* captures; a provider that cannot take files fails the call.
|
|
1087
1183
|
*/
|
|
1088
|
-
artifacts?: Array<Capture | string>;
|
|
1184
|
+
artifacts?: true | Array<Capture | string>;
|
|
1089
1185
|
};
|
|
1090
1186
|
type Verdict = {
|
|
1091
1187
|
verdict: 'pass' | 'fail';
|
|
@@ -1108,12 +1204,16 @@ type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
|
|
|
1108
1204
|
score: null;
|
|
1109
1205
|
error: string;
|
|
1110
1206
|
};
|
|
1207
|
+
/**
|
|
1208
|
+
* What an eval's body gets. The sandbox and the agent session start on the first call that needs
|
|
1209
|
+
* them (`send`, `exec`, `readFile`, `writeFile`, `serve` or `capture`), once however many ask.
|
|
1210
|
+
*/
|
|
1111
1211
|
type AttemptContext = {
|
|
1112
|
-
/** Sends the next turn of this eval's agent session
|
|
1212
|
+
/** Sends the next turn of this eval's agent session. */
|
|
1113
1213
|
send(prompt: string): Promise<Turn>;
|
|
1114
1214
|
/**
|
|
1115
1215
|
* Asks the judge a yes/no question about the latest turn (or `options.on`).
|
|
1116
|
-
* `options.artifacts` is a list of `c.capture()` handles or names.
|
|
1216
|
+
* `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
|
|
1117
1217
|
*/
|
|
1118
1218
|
judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1119
1219
|
exec(command: string): Promise<ExecResult>;
|
|
@@ -1132,7 +1232,6 @@ type AttemptContext = {
|
|
|
1132
1232
|
* Screenshots a path on a tunneled sandbox port and writes `artifacts/<name>.png`.
|
|
1133
1233
|
* First argument is the container port or a `ServeHandle` from `c.serve`.
|
|
1134
1234
|
* `instructions` runs a vision journey on that same tab first.
|
|
1135
|
-
* Needs a sandbox, same as `c.exec`.
|
|
1136
1235
|
*/
|
|
1137
1236
|
capture(target: number | ServeHandle, options?: CaptureCall): Promise<Capture>;
|
|
1138
1237
|
/** Workspace-relative, or `~/` for the home directory; `null` when there's no such file. */
|
|
@@ -1140,14 +1239,9 @@ type AttemptContext = {
|
|
|
1140
1239
|
/**
|
|
1141
1240
|
* Writes a file into the sandbox (workspace-relative, or `~/`), creating its directories.
|
|
1142
1241
|
* Later turns' `diff` leaves it out, as it does the environment's files, but shows any change
|
|
1143
|
-
* made to it afterwards.
|
|
1242
|
+
* made to it afterwards.
|
|
1144
1243
|
*/
|
|
1145
1244
|
writeFile(path: string, contents: string | Uint8Array): Promise<void>;
|
|
1146
|
-
/**
|
|
1147
|
-
* Starts the sandbox and the agent session without a turn, for checks on what the environment
|
|
1148
|
-
* set up. `c.send()` starts them too; calling both starts them once.
|
|
1149
|
-
*/
|
|
1150
|
-
sandbox(): Promise<Pick<AttemptContext, 'exec' | 'readFile'>>;
|
|
1151
1245
|
/** Every turn so far, oldest first. */
|
|
1152
1246
|
readonly turns: readonly Turn[];
|
|
1153
1247
|
/** Aborted on timeout or Ctrl-C. */
|
|
@@ -1160,12 +1254,14 @@ type AttemptContext = {
|
|
|
1160
1254
|
};
|
|
1161
1255
|
};
|
|
1162
1256
|
type EvalFn = (c: AttemptContext) => Promise<void>;
|
|
1257
|
+
/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
|
|
1163
1258
|
type FakeHarnessDescriptor = {
|
|
1164
1259
|
kind: 'fake';
|
|
1165
1260
|
/** Cell id; defaults to `fake`. */
|
|
1166
1261
|
id?: string;
|
|
1167
1262
|
/** Files the fake agent writes into the sandbox on every turn. */
|
|
1168
1263
|
files?: Record<string, string>;
|
|
1264
|
+
/** The fake agent's final message on every turn; default `Done.`. */
|
|
1169
1265
|
text?: string;
|
|
1170
1266
|
/** Simulated turn length, honours the abort signal. */
|
|
1171
1267
|
delayMs?: number;
|
|
@@ -1217,10 +1313,11 @@ type FakeHarnessDescriptor = {
|
|
|
1217
1313
|
* Behave like this agent: its image, and where it reads rules and skills (`CLAUDE.md` and
|
|
1218
1314
|
* `.claude/skills` for `claude-code`, `~/.gemini/config/skills` for `antigravity`,
|
|
1219
1315
|
* `~/.config/opencode/skills` for `opencode`, Goose's private home for `goose`,
|
|
1220
|
-
* `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex
|
|
1221
|
-
*
|
|
1316
|
+
* `~/.pi/agent/skills` for `pi`, `~/.agents/skills` for `codex`, fx's private home for
|
|
1317
|
+
* `fx`, Grok's private home for `grok-build`). Defaults to the fake's own (`AGENTS.md`,
|
|
1318
|
+
* `.agents/skills`) in Cursor's image.
|
|
1222
1319
|
*/
|
|
1223
|
-
layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex';
|
|
1320
|
+
layout?: 'cursor' | 'claude-code' | 'antigravity' | 'opencode' | 'goose' | 'pi' | 'codex' | 'fx' | 'grok-build';
|
|
1224
1321
|
};
|
|
1225
1322
|
/**
|
|
1226
1323
|
* One action of a scripted fake agent. `exec` is reported as a `shell` tool call whose output is
|
|
@@ -1406,28 +1503,82 @@ type CodexHarnessDescriptor = {
|
|
|
1406
1503
|
*/
|
|
1407
1504
|
startTimeout?: Duration;
|
|
1408
1505
|
};
|
|
1506
|
+
/**
|
|
1507
|
+
* fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
|
|
1508
|
+
* OpenAI's API by the model id, and reports each turn's tokens.
|
|
1509
|
+
*/
|
|
1510
|
+
type FxHarnessDescriptor = {
|
|
1511
|
+
kind: 'fx';
|
|
1512
|
+
/**
|
|
1513
|
+
* A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
|
|
1514
|
+
* `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
|
|
1515
|
+
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-4.1`.
|
|
1516
|
+
*/
|
|
1517
|
+
model: string;
|
|
1518
|
+
/** Cell id; defaults to `fx:<model>`. */
|
|
1519
|
+
id?: string;
|
|
1520
|
+
/** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
|
|
1521
|
+
maxConcurrent?: number;
|
|
1522
|
+
/**
|
|
1523
|
+
* How long the harness's session may take to become ready (its bootstrap and first session)
|
|
1524
|
+
* before the attempt is an infra error, after one retry; default 90s.
|
|
1525
|
+
*/
|
|
1526
|
+
startTimeout?: Duration;
|
|
1527
|
+
};
|
|
1528
|
+
/**
|
|
1529
|
+
* xAI's Grok Build through `@reeval/harness-grok-build` (`grok agent stdio`), made by
|
|
1530
|
+
* `grokBuild()`. It runs xAI models on an xAI API key and reports each turn's tokens.
|
|
1531
|
+
*/
|
|
1532
|
+
type GrokBuildHarnessDescriptor = {
|
|
1533
|
+
kind: 'grok-build';
|
|
1534
|
+
/** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
|
|
1535
|
+
model: string;
|
|
1536
|
+
/**
|
|
1537
|
+
* Reasoning effort as Grok's `default_reasoning_effort`, e.g. `low`, `medium` or `high`, for a
|
|
1538
|
+
* model that takes one.
|
|
1539
|
+
*/
|
|
1540
|
+
effort?: string;
|
|
1541
|
+
/** Cell id; defaults to `grok-build:<model>`, or `grok-build:<model>[effort=<value>]` when `effort` is set. */
|
|
1542
|
+
id?: string;
|
|
1543
|
+
/** The most Grok Build attempts at once, across the run, e.g. for the key's rate limits. */
|
|
1544
|
+
maxConcurrent?: number;
|
|
1545
|
+
/**
|
|
1546
|
+
* How long the harness's session may take to become ready (its bootstrap and first session)
|
|
1547
|
+
* before the attempt is an infra error, after one retry; default 90s.
|
|
1548
|
+
*/
|
|
1549
|
+
startTimeout?: Duration;
|
|
1550
|
+
};
|
|
1409
1551
|
/** A real coding agent, driven through its AI SDK harness adapter. */
|
|
1410
|
-
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor;
|
|
1552
|
+
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
|
|
1553
|
+
/** The agent an attempt runs: a real harness, or the fake one. */
|
|
1411
1554
|
type HarnessDescriptor = FakeHarnessDescriptor | AgentHarnessDescriptor;
|
|
1555
|
+
/** A temp directory on the host instead of a container, made by `fakeSandbox()`. */
|
|
1412
1556
|
type FakeSandboxDescriptor = {
|
|
1413
1557
|
kind: 'fake';
|
|
1558
|
+
/** Cell id; defaults to `fake`. */
|
|
1414
1559
|
id?: string;
|
|
1415
1560
|
/** Parent directory for the per-eval workspace (defaults to the OS temp dir). */
|
|
1416
1561
|
dir?: string;
|
|
1417
1562
|
/** The most attempts on this sandbox kind at once, across the run. */
|
|
1418
1563
|
maxConcurrent?: number;
|
|
1419
1564
|
};
|
|
1565
|
+
/** A local Docker container, made by `docker()`, or a Podman one, made by `podman()`. */
|
|
1420
1566
|
type DockerSandboxDescriptor = {
|
|
1421
1567
|
kind: 'docker';
|
|
1422
|
-
/**
|
|
1568
|
+
/** The container CLI the sandbox runs on: `podman` for `podman()`, Docker when unset. */
|
|
1569
|
+
engine?: 'docker' | 'podman';
|
|
1570
|
+
/**
|
|
1571
|
+
* Cell id; defaults to `docker` (`podman`), or `docker:<image>` for other images, plus an
|
|
1572
|
+
* install hash.
|
|
1573
|
+
*/
|
|
1423
1574
|
id?: string;
|
|
1424
1575
|
/** An image that already has the harness baked in, used as is. */
|
|
1425
1576
|
image?: string;
|
|
1426
1577
|
/** A Debian or Ubuntu based toolchain image; reeval bakes Node and the harness on top, cached. */
|
|
1427
1578
|
from?: string;
|
|
1428
1579
|
/**
|
|
1429
|
-
*
|
|
1430
|
-
*
|
|
1580
|
+
* A Dockerfile, absolute or relative to the directory of the file it's written in, built from its
|
|
1581
|
+
* directory with `REEVAL_BASE` set to reeval's sandbox image and cached by content hash.
|
|
1431
1582
|
*/
|
|
1432
1583
|
dockerfile?: string;
|
|
1433
1584
|
/** Baked onto the sandbox image in one build container: in order, no secrets, no workspace. */
|
|
@@ -1485,13 +1636,17 @@ type SecretOptions = {
|
|
|
1485
1636
|
/** `secret.template('Bearer ${TOKEN}')`: host variables substituted into a string, for MCP headers. */
|
|
1486
1637
|
type SecretTemplate = {
|
|
1487
1638
|
kind: 'secret-template';
|
|
1639
|
+
/** The string, with `${NAME}` for each host variable to substitute. */
|
|
1488
1640
|
template: string;
|
|
1489
1641
|
};
|
|
1642
|
+
/** A variable's value: a literal string, or a host secret. */
|
|
1490
1643
|
type EnvValue = string | Secret;
|
|
1644
|
+
/** Environment variables by name. */
|
|
1491
1645
|
type EnvVars = Record<string, EnvValue>;
|
|
1492
1646
|
/** A file on the host, read when an attempt starts. Made by the `file` helper. */
|
|
1493
1647
|
type HostFile = {
|
|
1494
1648
|
kind: 'file';
|
|
1649
|
+
/** Its path on the host, absolute or relative to the directory of the file it's written in. */
|
|
1495
1650
|
path: string;
|
|
1496
1651
|
};
|
|
1497
1652
|
/** Options for the `directory` helper. */
|
|
@@ -1509,62 +1664,92 @@ type DirectoryOptions = {
|
|
|
1509
1664
|
/** `false` walks into `DEFAULT_DIRECTORY_IGNORES` too. `.git` is always skipped. */
|
|
1510
1665
|
defaultIgnores?: boolean;
|
|
1511
1666
|
};
|
|
1667
|
+
/** An MCP server the agent gets, made by the `mcp` helper: a command in the sandbox, or a URL. */
|
|
1512
1668
|
type McpServer = {
|
|
1513
1669
|
kind: 'mcp';
|
|
1514
1670
|
transport: 'stdio';
|
|
1671
|
+
/** The command that starts the server in the sandbox. */
|
|
1515
1672
|
command: string;
|
|
1516
1673
|
args: string[];
|
|
1674
|
+
/** Variables the server process gets. */
|
|
1517
1675
|
env: EnvVars;
|
|
1518
1676
|
} | {
|
|
1519
1677
|
kind: 'mcp';
|
|
1678
|
+
/** Streamable HTTP, or the older SSE transport. */
|
|
1520
1679
|
transport: 'http' | 'sse';
|
|
1521
1680
|
url: string;
|
|
1681
|
+
/** Request headers, with secrets brokered by the egress gateway. */
|
|
1522
1682
|
headers: Record<string, string | Secret | SecretTemplate>;
|
|
1523
1683
|
};
|
|
1524
1684
|
/** `harness` (default): the agent's own skills, via its adapter. `workspace`: its repo skill dir. */
|
|
1525
1685
|
type SkillPlacement = 'harness' | 'workspace';
|
|
1686
|
+
/** Skills to place for the agent, made by the `skill` helper. */
|
|
1526
1687
|
type SkillSource = {
|
|
1527
1688
|
kind: 'skill';
|
|
1528
1689
|
placement: SkillPlacement;
|
|
1529
1690
|
} & ({
|
|
1530
1691
|
source: 'local';
|
|
1692
|
+
/** A host directory with a `SKILL.md`, or a directory of them. */
|
|
1531
1693
|
path: string;
|
|
1694
|
+
/** The skill names to keep; default every one found. */
|
|
1532
1695
|
only?: string[];
|
|
1696
|
+
/** Paths in each skill directory to leave out, gitignore-style. */
|
|
1533
1697
|
exclude?: string[];
|
|
1534
1698
|
} | {
|
|
1535
1699
|
source: 'git';
|
|
1700
|
+
/** As written: a URL, `github:owner/repo`, or a GitHub `/tree/<ref>/<path>` URL. */
|
|
1536
1701
|
repo: string;
|
|
1702
|
+
/** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
|
|
1537
1703
|
ref?: string;
|
|
1704
|
+
/** The directory in the repository to read skills from. */
|
|
1538
1705
|
path?: string;
|
|
1706
|
+
/** The skill names to keep; default every one found. */
|
|
1539
1707
|
only?: string[];
|
|
1708
|
+
/** Paths in each skill directory to leave out, gitignore-style. */
|
|
1540
1709
|
exclude?: string[];
|
|
1541
1710
|
} | {
|
|
1542
1711
|
source: 'registry';
|
|
1712
|
+
/** A skills.sh id, `owner/repo@skill`. */
|
|
1543
1713
|
id: string;
|
|
1714
|
+
/** A git ref of the skill's repository. */
|
|
1544
1715
|
version?: string;
|
|
1545
1716
|
} | {
|
|
1546
1717
|
source: 'npm';
|
|
1718
|
+
/** A pinned npm package spec. */
|
|
1547
1719
|
spec: string;
|
|
1720
|
+
/** The directory in the package to read skills from; default `skills`. */
|
|
1548
1721
|
path: string;
|
|
1722
|
+
/** The skill names to keep; default every one found. */
|
|
1549
1723
|
only?: string[];
|
|
1724
|
+
/** Paths in each skill directory to leave out, gitignore-style. */
|
|
1550
1725
|
exclude?: string[];
|
|
1551
1726
|
} | {
|
|
1552
1727
|
source: 'inline';
|
|
1728
|
+
/** The skill's name, as in `SKILL.md`'s frontmatter. */
|
|
1553
1729
|
name: string;
|
|
1730
|
+
/** The skill's description, as in `SKILL.md`'s frontmatter. */
|
|
1554
1731
|
description: string;
|
|
1732
|
+
/** The body of `SKILL.md`. */
|
|
1555
1733
|
content: string;
|
|
1734
|
+
/** Other files in the skill directory, by relative path. */
|
|
1556
1735
|
files: Record<string, string>;
|
|
1557
1736
|
});
|
|
1558
1737
|
/** A `docker({ install })` command, baked into a cached image layer: no secrets, root, cwd `/`. */
|
|
1559
1738
|
type InstallStep = string | {
|
|
1739
|
+
/** The shell command. */
|
|
1560
1740
|
run: string;
|
|
1561
1741
|
timeout?: Duration;
|
|
1562
1742
|
};
|
|
1563
|
-
/**
|
|
1564
|
-
|
|
1743
|
+
/**
|
|
1744
|
+
* Runs in every attempt before the agent starts, with secrets, outside the eval timeout: a shell
|
|
1745
|
+
* command, or `{ run, timeout?, cwd? }`.
|
|
1746
|
+
*/
|
|
1565
1747
|
type SetupStep = string | {
|
|
1748
|
+
/** The shell command. */
|
|
1566
1749
|
run: string;
|
|
1750
|
+
/** Default 5m. */
|
|
1567
1751
|
timeout?: Duration;
|
|
1752
|
+
/** Relative to the workspace, or absolute; default the workspace. */
|
|
1568
1753
|
cwd?: string;
|
|
1569
1754
|
};
|
|
1570
1755
|
/**
|
|
@@ -1584,6 +1769,7 @@ type GitWorkspace = {
|
|
|
1584
1769
|
kind: 'git';
|
|
1585
1770
|
/** As written: a URL, `github:owner/repo`, or a local path. */
|
|
1586
1771
|
repo: string;
|
|
1772
|
+
/** What reeval fetches: the clone URL, or an absolute path for a local repository. */
|
|
1587
1773
|
url: string;
|
|
1588
1774
|
/** A branch, tag or commit SHA; unset is the remote's `HEAD`. */
|
|
1589
1775
|
ref?: string;
|
|
@@ -1609,6 +1795,7 @@ type EnvironmentSpec = {
|
|
|
1609
1795
|
sandbox?: SandboxDescriptor;
|
|
1610
1796
|
/** The code the workspace starts as, before `files`, `rules` and `setup`; a child's replaces its parent's. */
|
|
1611
1797
|
workspace?: GitWorkspace;
|
|
1798
|
+
/** Variables and secrets for the sandbox, merged by name over the parent's. */
|
|
1612
1799
|
env?: EnvVars;
|
|
1613
1800
|
/** Replaces the inherited policy; an `extend` child's replaces its parent's. */
|
|
1614
1801
|
egress?: EgressPolicy;
|
|
@@ -1616,13 +1803,18 @@ type EnvironmentSpec = {
|
|
|
1616
1803
|
files?: Record<string, string | HostFile>;
|
|
1617
1804
|
/** Project rules, written where the harness's agent reads them: `AGENTS.md`, or `CLAUDE.md` for Claude Code. */
|
|
1618
1805
|
rules?: string | HostFile;
|
|
1806
|
+
/** Shell commands run in every attempt before the agent starts; a child's run after its parent's. */
|
|
1619
1807
|
setup?: SetupStep[];
|
|
1808
|
+
/** Skills placed for the agent. */
|
|
1620
1809
|
skills?: SkillSource[];
|
|
1810
|
+
/** MCP servers the agent gets, by name. */
|
|
1621
1811
|
mcp?: Record<string, McpServer>;
|
|
1622
1812
|
};
|
|
1623
1813
|
/** One environment in an `extend` chain. */
|
|
1624
1814
|
type EnvironmentLayer = {
|
|
1815
|
+
/** The name passed to `defineEnvironment` or `extend`. */
|
|
1625
1816
|
name: string;
|
|
1817
|
+
/** The spec as that layer wrote it, before merging. */
|
|
1626
1818
|
spec: EnvironmentSpec;
|
|
1627
1819
|
};
|
|
1628
1820
|
/**
|
|
@@ -1631,7 +1823,9 @@ type EnvironmentLayer = {
|
|
|
1631
1823
|
*/
|
|
1632
1824
|
type Environment = {
|
|
1633
1825
|
readonly kind: 'environment';
|
|
1826
|
+
/** The name it was defined or extended under, which names its cells. */
|
|
1634
1827
|
readonly name: string;
|
|
1828
|
+
/** Its `extend` chain, root first. */
|
|
1635
1829
|
readonly layers: readonly EnvironmentLayer[];
|
|
1636
1830
|
/**
|
|
1637
1831
|
* A child environment layered on this one; wherever it sets a key this one has, it wins.
|
|
@@ -1767,14 +1961,71 @@ type CompareOptions<E = Environment> = {
|
|
|
1767
1961
|
};
|
|
1768
1962
|
/**
|
|
1769
1963
|
* A question for pairwise judgement: its text, or the text and the `'provider/model'` that
|
|
1770
|
-
* answers it. Without a model, `
|
|
1964
|
+
* answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the
|
|
1965
|
+
* judge's when it's a language model.
|
|
1771
1966
|
* `across` is the factor it varies when pairing; without it (and without a one-factor
|
|
1772
|
-
* `compare.across`) both pair kinds run.
|
|
1967
|
+
* `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.
|
|
1773
1968
|
*/
|
|
1774
1969
|
type CompareQuestion = string | {
|
|
1775
1970
|
question: string;
|
|
1971
|
+
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
1776
1972
|
model?: string;
|
|
1973
|
+
/** The factor it varies when pairing. */
|
|
1777
1974
|
across?: PairFactor;
|
|
1975
|
+
/** What each side shows, over the cascaded `question.state`, key by key. */
|
|
1976
|
+
state?: QuestionStateOptions;
|
|
1977
|
+
/** Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
|
|
1978
|
+
artifacts?: true | string[];
|
|
1979
|
+
};
|
|
1980
|
+
/**
|
|
1981
|
+
* What each side of a compare question shows: the keys of `judge.state` but `contents` (the
|
|
1982
|
+
* attempts' workspaces are gone when questions are asked). An unset key keeps its default, which
|
|
1983
|
+
* is what a question shows with no `state` at all: the task and the diff.
|
|
1984
|
+
*/
|
|
1985
|
+
type QuestionStateOptions = {
|
|
1986
|
+
/** The prompts the attempt was sent, once for both sides when they match. Default true. */
|
|
1987
|
+
task?: boolean;
|
|
1988
|
+
/** The final message: the agent's own account of its work. Default false. */
|
|
1989
|
+
output?: boolean;
|
|
1990
|
+
/** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
|
|
1991
|
+
files?: boolean;
|
|
1992
|
+
/**
|
|
1993
|
+
* The diff since the eval started. A list is gitignore-style patterns over workspace paths:
|
|
1994
|
+
* the last matching pattern wins and `!` excludes. Default true.
|
|
1995
|
+
*/
|
|
1996
|
+
diff?: boolean | string[];
|
|
1997
|
+
/**
|
|
1998
|
+
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
1999
|
+
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2000
|
+
* harness's tools, so a pair of two harnesses isn't blind. Default false.
|
|
2001
|
+
*/
|
|
2002
|
+
transcript?: boolean | 'tools' | 'full';
|
|
2003
|
+
/**
|
|
2004
|
+
* `'all'`: every turn under `turns`, each with its task, output and (with `transcript`)
|
|
2005
|
+
* transcript; `'last'`: the last turn only. Default `'all'`.
|
|
2006
|
+
*/
|
|
2007
|
+
turns?: 'last' | 'all';
|
|
2008
|
+
/** `c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false. */
|
|
2009
|
+
checks?: boolean | string[];
|
|
2010
|
+
};
|
|
2011
|
+
/** `QuestionStateOptions` with every key set: what each side of one question showed. */
|
|
2012
|
+
type ResolvedQuestionStateOptions = {
|
|
2013
|
+
task: boolean;
|
|
2014
|
+
output: boolean;
|
|
2015
|
+
files: boolean;
|
|
2016
|
+
diff: boolean | string[];
|
|
2017
|
+
transcript: boolean | 'tools' | 'full';
|
|
2018
|
+
turns: 'last' | 'all';
|
|
2019
|
+
checks: boolean | string[];
|
|
2020
|
+
};
|
|
2021
|
+
/** Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key. */
|
|
2022
|
+
type QuestionOptions = {
|
|
2023
|
+
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2024
|
+
model?: string;
|
|
2025
|
+
/** What each side shows. */
|
|
2026
|
+
state?: QuestionStateOptions;
|
|
2027
|
+
/** Each side's own captures, by name; `true` sends all of them. Default none. */
|
|
2028
|
+
artifacts?: true | string[];
|
|
1778
2029
|
};
|
|
1779
2030
|
/** A matrix factor a compare question varies when pairing; the others are held. */
|
|
1780
2031
|
type PairFactor = 'environment' | 'harness';
|
|
@@ -1799,6 +2050,11 @@ type PairVote = {
|
|
|
1799
2050
|
swapped: boolean;
|
|
1800
2051
|
/** The model's short reason for this pick; omitted when it gave none. Capped. */
|
|
1801
2052
|
reason?: string;
|
|
2053
|
+
/** The captures each named side showed, by name; set when the question sends `artifacts`. */
|
|
2054
|
+
artifacts?: {
|
|
2055
|
+
a: string[];
|
|
2056
|
+
b: string[];
|
|
2057
|
+
};
|
|
1802
2058
|
};
|
|
1803
2059
|
/** A compare question's answer: every vote, and each side's wins over decided votes per cut. */
|
|
1804
2060
|
type CompareAnswer = {
|
|
@@ -1807,6 +2063,10 @@ type CompareAnswer = {
|
|
|
1807
2063
|
model: string;
|
|
1808
2064
|
/** The model id the provider says served the calls, when it reports one. */
|
|
1809
2065
|
resolvedModel?: string;
|
|
2066
|
+
/** What each side showed, every key resolved; unset on answers from before questions took `state`, which showed the defaults. */
|
|
2067
|
+
stateOptions?: ResolvedQuestionStateOptions;
|
|
2068
|
+
/** The captures it sent of each side, as asked: `true` for all, or their names. */
|
|
2069
|
+
artifacts?: true | string[];
|
|
1810
2070
|
votes: PairVote[];
|
|
1811
2071
|
/** Wins over decided pairs, per named side, per cut. */
|
|
1812
2072
|
rates: Array<{
|
|
@@ -1817,7 +2077,10 @@ type CompareAnswer = {
|
|
|
1817
2077
|
}>;
|
|
1818
2078
|
/** Why some or all of its votes are missing: the model call failed. Outcomes never change. */
|
|
1819
2079
|
error?: string;
|
|
2080
|
+
/** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
|
|
2081
|
+
input?: FileRef;
|
|
1820
2082
|
};
|
|
2083
|
+
/** An option a profile's `limit` can narrow, and `pin` can protect. */
|
|
1821
2084
|
type PinnableKey = 'harness' | 'runs' | 'compare';
|
|
1822
2085
|
/**
|
|
1823
2086
|
* How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
|
|
@@ -1833,7 +2096,7 @@ type CellGate = GatePolicy | 'none';
|
|
|
1833
2096
|
type CompareSpec<E = Environment> = CompareOptions<E>;
|
|
1834
2097
|
/** Options other than `environment` and `compare`, which are exclusive of each other. */
|
|
1835
2098
|
type EvalOptionFields = {
|
|
1836
|
-
/**
|
|
2099
|
+
/** Which agents run the eval; each harness is its own cell. */
|
|
1837
2100
|
harness?: HarnessOption;
|
|
1838
2101
|
/**
|
|
1839
2102
|
* The default machine when the resolved environment omits `sandbox`. Cascades like `harness`.
|
|
@@ -1851,11 +2114,15 @@ type EvalOptionFields = {
|
|
|
1851
2114
|
* environment, then groups and the eval.
|
|
1852
2115
|
*/
|
|
1853
2116
|
egress?: EgressPolicy;
|
|
2117
|
+
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
1854
2118
|
judge?: JudgeOptions;
|
|
1855
2119
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
1856
2120
|
capture?: CaptureOptions;
|
|
2121
|
+
/** Defaults for every compare question: `model`, and `state` merged key by key, down the cascade. */
|
|
2122
|
+
question?: QuestionOptions;
|
|
1857
2123
|
/** Per attempt. */
|
|
1858
2124
|
timeout?: Duration;
|
|
2125
|
+
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
1859
2126
|
tags?: string[];
|
|
1860
2127
|
/** Keys a profile's `limit` may not narrow for this group or eval. */
|
|
1861
2128
|
pin?: PinnableKey[];
|
|
@@ -1873,20 +2140,36 @@ type EnvironmentOrCompare<E> = {
|
|
|
1873
2140
|
} | {
|
|
1874
2141
|
/** Forbidden beside `compare.baseline`, which names the control. */
|
|
1875
2142
|
environment?: never;
|
|
2143
|
+
/** Environments to run side by side; `baseline` is the control. */
|
|
1876
2144
|
compare: CompareSpec<E> & {
|
|
1877
2145
|
baseline: E;
|
|
1878
2146
|
};
|
|
1879
2147
|
} | {
|
|
1880
2148
|
/** The control: the compare has no `baseline`. */
|
|
1881
2149
|
environment?: E;
|
|
2150
|
+
/** Environments to run side by side against `environment`, the control. */
|
|
1882
2151
|
compare: CompareSpec<E> & {
|
|
1883
2152
|
baseline?: undefined;
|
|
1884
2153
|
};
|
|
1885
2154
|
};
|
|
1886
2155
|
type EvalOptions = EvalOptionFields & EnvironmentOrCompare<Environment>;
|
|
1887
2156
|
type GroupOptions = EvalOptions;
|
|
2157
|
+
/**
|
|
2158
|
+
* `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
|
|
2159
|
+
* by name or a fake judge, and environments without `extend`. The same `environment` / compare
|
|
2160
|
+
* object split as `EvalOptions`.
|
|
2161
|
+
*/
|
|
2162
|
+
type EvalOptionsData = Omit<EvalOptionFields, 'harness' | 'judge'> & {
|
|
2163
|
+
/** Which agents run the eval; each harness is its own cell. */
|
|
2164
|
+
harness?: HarnessData;
|
|
2165
|
+
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
2166
|
+
judge?: ConfigJudgeOptions;
|
|
2167
|
+
} & EnvironmentOrCompare<EnvironmentData>;
|
|
2168
|
+
/** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
|
|
2169
|
+
type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
|
|
1888
2170
|
/** The config file is plain data sent to workers, so its judge model can't be an object. */
|
|
1889
2171
|
type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
2172
|
+
/** `'provider/model'`, or a fake judge. */
|
|
1890
2173
|
model?: string | FakeJudgeModel;
|
|
1891
2174
|
};
|
|
1892
2175
|
type HistoryConfig = {
|
|
@@ -1897,22 +2180,35 @@ type HistoryConfig = {
|
|
|
1897
2180
|
};
|
|
1898
2181
|
/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
|
|
1899
2182
|
type Budget = {
|
|
2183
|
+
/** The most attempts the plan may have. */
|
|
1900
2184
|
attempts?: number;
|
|
2185
|
+
/** How long the run may go on, from the first attempt's start. */
|
|
1901
2186
|
duration?: Duration;
|
|
1902
2187
|
};
|
|
1903
2188
|
/** What evals get when they don't say otherwise: under groups and evals. */
|
|
1904
2189
|
type ProfileDefaults = {
|
|
2190
|
+
/** The agents every eval runs on, unless a group or eval replaces them. */
|
|
1905
2191
|
harness?: HarnessDescriptor | HarnessDescriptor[];
|
|
1906
2192
|
/** The default machine when the resolved environment omits `sandbox`. */
|
|
1907
2193
|
sandbox?: SandboxDescriptor;
|
|
2194
|
+
/** What every eval runs in when it names no environment. */
|
|
1908
2195
|
environment?: Environment;
|
|
2196
|
+
/** Attempts per cell. */
|
|
1909
2197
|
runs?: number;
|
|
2198
|
+
/** How each cell's attempts decide its outcome. */
|
|
1910
2199
|
gate?: GatePolicy;
|
|
2200
|
+
/** Variables and secrets for the sandbox, merged by name over the config's. */
|
|
1911
2201
|
env?: EnvVars;
|
|
2202
|
+
/** Where the sandbox may connect; replaces the config's policy. */
|
|
1912
2203
|
egress?: EgressPolicy;
|
|
2204
|
+
/** Defaults for `c.judge()`, merged field by field over the config's. */
|
|
1913
2205
|
judge?: ConfigJudgeOptions;
|
|
2206
|
+
/** Per attempt. */
|
|
1914
2207
|
timeout?: Duration;
|
|
2208
|
+
/** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
|
|
1915
2209
|
capture?: CaptureOptions;
|
|
2210
|
+
/** Defaults for every compare question, merged over the config's. */
|
|
2211
|
+
question?: QuestionOptions;
|
|
1916
2212
|
};
|
|
1917
2213
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
1918
2214
|
type ProfileLimit = {
|
|
@@ -1927,10 +2223,15 @@ type ProfileLimit = {
|
|
|
1927
2223
|
};
|
|
1928
2224
|
/** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
|
|
1929
2225
|
type Profile = {
|
|
2226
|
+
/** What evals get when they don't say otherwise: under groups and evals. */
|
|
1930
2227
|
defaults?: ProfileDefaults;
|
|
2228
|
+
/** Caps over every eval, which only narrow; `pin` on a group or eval skips them. */
|
|
1931
2229
|
limit?: ProfileLimit;
|
|
2230
|
+
/** Judge settings for this profile, the same as `defaults.judge`, which wins field by field. */
|
|
1932
2231
|
judge?: ConfigJudgeOptions;
|
|
2232
|
+
/** Merged over the config's `budget`. */
|
|
1933
2233
|
budget?: Budget;
|
|
2234
|
+
/** When this profile's runs are copied into the history folder. */
|
|
1934
2235
|
record?: RecordMode;
|
|
1935
2236
|
/** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
|
|
1936
2237
|
tags?: string[];
|
|
@@ -1974,12 +2275,48 @@ type FakePick = FakeVote | {
|
|
|
1974
2275
|
winner: FakeVote;
|
|
1975
2276
|
reason?: string;
|
|
1976
2277
|
};
|
|
1977
|
-
/** What a compare question's model is shown: the prompt, and the two
|
|
2278
|
+
/** What a compare question's model is shown: the prompt, and the two sides in slot order. */
|
|
1978
2279
|
type BlindPair = {
|
|
1979
2280
|
question: string;
|
|
1980
2281
|
prompt: string;
|
|
1981
|
-
a:
|
|
1982
|
-
b:
|
|
2282
|
+
a: BlindSide;
|
|
2283
|
+
b: BlindSide;
|
|
2284
|
+
};
|
|
2285
|
+
/** A turn of one side, with the keys its question's `state` turned on. */
|
|
2286
|
+
type BlindTurn = {
|
|
2287
|
+
task?: string;
|
|
2288
|
+
output?: string;
|
|
2289
|
+
transcript?: string;
|
|
2290
|
+
};
|
|
2291
|
+
/**
|
|
2292
|
+
* One side of a blinded pair: only the keys its question's `state` turned on. With
|
|
2293
|
+
* `turns: 'all'`, `task`, `output` and `transcript` are under `turns`; a task both sides share is
|
|
2294
|
+
* the pair's prompt's, not either side's.
|
|
2295
|
+
*/
|
|
2296
|
+
type BlindSide = BlindTurn & {
|
|
2297
|
+
turns?: BlindTurn[];
|
|
2298
|
+
files?: string[];
|
|
2299
|
+
/** The diff as the model sees it: dependency and binary noise dropped. */
|
|
2300
|
+
diff?: string;
|
|
2301
|
+
/** A command the other side ran and this one didn't is `{ command, notRun: true }`. */
|
|
2302
|
+
checks?: Array<{
|
|
2303
|
+
command: string;
|
|
2304
|
+
exitCode: number;
|
|
2305
|
+
stdout: string;
|
|
2306
|
+
stderr: string;
|
|
2307
|
+
} | {
|
|
2308
|
+
command: string;
|
|
2309
|
+
notRun: true;
|
|
2310
|
+
}>;
|
|
2311
|
+
/** Its captures, in the order sent, labelled by position; a name it lacks is `missing`. */
|
|
2312
|
+
artifacts?: Array<{
|
|
2313
|
+
label: string;
|
|
2314
|
+
mediaType: 'image/png';
|
|
2315
|
+
bytes: number;
|
|
2316
|
+
} | {
|
|
2317
|
+
label: string;
|
|
2318
|
+
missing: true;
|
|
2319
|
+
}>;
|
|
1983
2320
|
};
|
|
1984
2321
|
/** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
|
|
1985
2322
|
type AnalysisConfig = {
|
|
@@ -1997,33 +2334,46 @@ type AnalysisConfig = {
|
|
|
1997
2334
|
/** The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap. */
|
|
1998
2335
|
maxOutputTokens?: number;
|
|
1999
2336
|
};
|
|
2000
|
-
type
|
|
2337
|
+
type ReevalConfig = {
|
|
2338
|
+
/** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
|
|
2001
2339
|
include?: string[];
|
|
2340
|
+
/** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
|
|
2002
2341
|
harness?: HarnessDescriptor | HarnessDescriptor[];
|
|
2003
2342
|
/**
|
|
2004
2343
|
* The default machine when the resolved environment omits `sandbox`. Not an environment:
|
|
2005
2344
|
* it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
|
|
2006
2345
|
*/
|
|
2007
2346
|
sandbox?: SandboxDescriptor;
|
|
2347
|
+
/** What every eval runs in when it names no environment, and a baseline-less compare's control. */
|
|
2008
2348
|
environment?: Environment;
|
|
2349
|
+
/** Attempts per cell; default 1. */
|
|
2009
2350
|
runs?: number;
|
|
2010
2351
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2011
2352
|
gate?: GatePolicy;
|
|
2353
|
+
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
2012
2354
|
env?: EnvVars;
|
|
2355
|
+
/** Where the sandbox may connect; default `'allow-all'`. */
|
|
2013
2356
|
egress?: EgressPolicy;
|
|
2357
|
+
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
2014
2358
|
judge?: ConfigJudgeOptions;
|
|
2359
|
+
/** Per attempt; default `'15m'`. */
|
|
2015
2360
|
timeout?: Duration;
|
|
2016
2361
|
/**
|
|
2017
2362
|
* Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host
|
|
2018
2363
|
* binary when `c.capture()` runs; it does not change this object.
|
|
2019
2364
|
*/
|
|
2020
2365
|
capture?: CaptureOptions;
|
|
2366
|
+
/** Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade. */
|
|
2367
|
+
question?: QuestionOptions;
|
|
2021
2368
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
2022
2369
|
concurrency?: number;
|
|
2023
2370
|
/** When runs are copied into the history folder: `reeval record` only, in CI, or on every run. */
|
|
2024
2371
|
record?: RecordMode;
|
|
2372
|
+
/** The history folder, and which attempts keep their artifacts there. */
|
|
2025
2373
|
history?: HistoryConfig;
|
|
2374
|
+
/** Caps on a run; a profile's `budget` is merged over it. */
|
|
2026
2375
|
budget?: Budget;
|
|
2376
|
+
/** Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`. */
|
|
2027
2377
|
profiles?: Record<string, Profile>;
|
|
2028
2378
|
/** The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set. */
|
|
2029
2379
|
defaultProfile?: {
|
|
@@ -2042,6 +2392,7 @@ type EsConfig = {
|
|
|
2042
2392
|
keep?: number;
|
|
2043
2393
|
traceBufferMb?: number;
|
|
2044
2394
|
};
|
|
2395
|
+
/** `reeval analyze`, compare questions without a model, and `agent`'s notes during a run. */
|
|
2045
2396
|
analysis?: AnalysisConfig;
|
|
2046
2397
|
};
|
|
2047
2398
|
/** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
|
|
@@ -2108,6 +2459,7 @@ type CellRole = 'control' | 'variant';
|
|
|
2108
2459
|
type Cell = {
|
|
2109
2460
|
key: string;
|
|
2110
2461
|
harness: string;
|
|
2462
|
+
/** The environment's own name, also on a variant cell (`with-house-greeting`, with no prefix). */
|
|
2111
2463
|
environment: string;
|
|
2112
2464
|
sandbox: string;
|
|
2113
2465
|
role: CellRole;
|
|
@@ -2269,5 +2621,5 @@ type SandboxWarning = {
|
|
|
2269
2621
|
message: string;
|
|
2270
2622
|
};
|
|
2271
2623
|
//#endregion
|
|
2272
|
-
export {
|
|
2273
|
-
//# sourceMappingURL=types-
|
|
2624
|
+
export { GooseHarnessDescriptor as $, JudgementRecord as $t, DirectoryOptions as A, ServeHandle as At, ErrorKind as B, Wait as Bt, CompareCut as C, effortSample as Cn, ProfileDefaults as Ct, ConfigJudgeOptions as D, formatRatio as Dn, SandboxWarning as Dt, ConfigHelpers as E, formatCount as En, SandboxDescriptor as Et, EnvVars as F, Turn as Ft, FakeAnalysisModel as G, CheckTally as Gt, EvalOptions as H, ArtifactRef as Ht, Environment as I, TurnData as It, FakeNoteInput as J, FamilyRate as Jt, FakeHarnessDescriptor as K, EvalEntry as Kt, EnvironmentHelpers as L, TurnStatus as Lt, Duration as M, SkillSource as Mt, EgressPolicy as N, TokenUsage as Nt, CursorHarnessDescriptor as O, formatSeconds as On, Secret as Ot, EgressRecord as P, TranscriptEvent as Pt, FxHarnessDescriptor as Q, HistoryEntry as Qt, EnvironmentInput as R, Verdict as Rt, CompareAnswer as S, effortOf as Sn, Profile as St, CompareQuestion as T, effortVerdict as Tn, ReevalConfig as Tt, EvalOptionsData as U, AttemptRecord as Ut, EvalFn as V, WebEventType as Vt, ExecResult as W, CellRecord as Wt, FakeScriptStep as X, FileRef as Xt, FakeSandboxDescriptor as Y, FamilyRecord as Yt, FakeVote as Z, HistoryCell as Zt, Cell as _, cellSamples as _n, Outcome as _t, BackgroundHandle as a, EFFORT_ALPHA as an, HarnessOption as at, ClaudeCodeHarnessDescriptor as b, effortChangeLabel as bn, PiHarnessDescriptor as bt, Capture as c, EffortMetric as cn, JudgeArtifactRef as ct, CaptureOptions as d, Spread as dn, JudgeModel as dt, Lift as en, GrokBuildHarnessDescriptor as et, CaptureUse as f, TOKENS_UNREPORTED as fn, JudgeOptions as ft, CaptureWindow as g, cellEffort as gn, OpenCodeHarnessDescriptor as gt, CaptureWait as h, cellContrast as hn, McpServer as ht, AttemptContext as i, DIRECTIONAL_MIN as in, HarnessEdit as it, DockerSandboxDescriptor as j, SetupStep as jt, Diff as k, tokenValues as kn, SecretTemplate as kt, CaptureCall as l, EffortSample as ln, JudgeCallOptions as lt, CaptureUseScript as m, bothUnderATenth as mn, JudgeStateOptions as mt, AnalysisConfig as n, StackSummary as nn, HarnessData as nt, BlindPair as o, EFFORT_MIN_RUNS as on, HistoryConfig as ot, CaptureUseAction as p, VERDICT_TEXT as pn, JudgeState as pt, FakeJudgeModel as q, FamilyLift as qt, AntigravityHarnessDescriptor as r, CellEffort as rn, HarnessDescriptor as rt, Budget as s, EffortContrast as sn, HostFile as st, AgentHarnessDescriptor as t, RunRecord as tn, GroupOptions as tt, CaptureInstruction as u, EffortVerdict as un, JudgeFailure as ut, CellRole as v, contrastEffort as vn, PairFactor as vt, CompareOptions as w, effortSummary as wn, ProfileLimit as wt, CodexHarnessDescriptor as x, effortLabel as xn, PinnableKey as xt, ClaudeCodeEffort as y, directional as yn, PairVote as yt, EnvironmentSpec as z, Viewport as zt };
|
|
2625
|
+
//# sourceMappingURL=types-BYdlwMwo.d.ts.map
|