reeval 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +5 -5
- package/dist/{adapter-CptK032x.js → adapter-C60z3wLq.js} +5 -4
- package/dist/adapter-C60z3wLq.js.map +1 -0
- package/dist/{agent-B63OzUX9.js → agent-BmsMXZXx.js} +30 -10
- package/dist/{agent-B63OzUX9.js.map → agent-BmsMXZXx.js.map} +1 -1
- package/dist/agents-DXuFxtGC.js +1151 -0
- package/dist/agents-DXuFxtGC.js.map +1 -0
- package/dist/{app-files-sgqwfjtn.js → app-files-DT5mJyV4.js} +3 -3
- package/dist/app-files-DT5mJyV4.js.map +1 -0
- package/dist/assemble-CE4pHKIa.js +1 -0
- package/dist/{assemble-Me7Whp-y.js → assemble-DM2PNU4R.js} +152 -129
- package/dist/assemble-DM2PNU4R.js.map +1 -0
- package/dist/{attempt-Dp7Wtm8M.js → attempt-CAdlwlNu.js} +2 -2
- package/dist/{attempt-Dp7Wtm8M.js.map → attempt-CAdlwlNu.js.map} +1 -1
- package/dist/{attempt-host-Cik0SoIn.js → attempt-host-7VviAJrl.js} +6 -6
- package/dist/{attempt-host-Cik0SoIn.js.map → attempt-host-7VviAJrl.js.map} +1 -1
- package/dist/boat-BjLqFJIZ.js +108 -0
- package/dist/boat-BjLqFJIZ.js.map +1 -0
- package/dist/boat-DSNgZt-t.js +322 -0
- package/dist/boat-DSNgZt-t.js.map +1 -0
- package/dist/bridge/index.mjs +401 -79
- package/dist/capture-BzotD2hs.js +1 -0
- package/dist/{capture-C3RmjPph.js → capture-Ccf0Hmyx.js} +27 -25
- package/dist/capture-Ccf0Hmyx.js.map +1 -0
- package/dist/cases-Dy0mifal.js +1 -0
- package/dist/cases-z9E6DCd5.js +639 -0
- package/dist/cases-z9E6DCd5.js.map +1 -0
- package/dist/{cli-BBOlXKPG.js → cli-DTuLSlmW.js} +442 -707
- package/dist/cli-DTuLSlmW.js.map +1 -0
- package/dist/{cli-spec-DZ0bnP0_.js → cli-spec-Dsr79Qy1.js} +38 -11
- package/dist/cli-spec-Dsr79Qy1.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-tZ2WBDxf.js → client-KM4C5r5_.js} +93 -3
- package/dist/client-KM4C5r5_.js.map +1 -0
- package/dist/{command-DhiwsZl1.js → command-C2C7P5qo.js} +9 -9
- package/dist/{command-DhiwsZl1.js.map → command-C2C7P5qo.js.map} +1 -1
- package/dist/{command-CFQ9c1hk.js → command-D8oMB2fa.js} +16 -16
- package/dist/command-D8oMB2fa.js.map +1 -0
- package/dist/{command-fO0BFsTq.js → command-DSL7I-SI.js} +37 -37
- package/dist/command-DSL7I-SI.js.map +1 -0
- package/dist/{command-qGS802uX.js → command-Ff0T-1Hi.js} +21 -80
- package/dist/command-Ff0T-1Hi.js.map +1 -0
- package/dist/{commands-xYTMNTPv.js → commands-Bk9NG1s_.js} +1 -1
- package/dist/{commands-CC60TOT5.js → commands-xAOi_VEk.js} +186 -85
- package/dist/commands-xAOi_VEk.js.map +1 -0
- package/dist/{config-C2UtS9vy.js → config-SvK8pjEK.js} +261 -1708
- package/dist/config-SvK8pjEK.js.map +1 -0
- package/dist/{context-BajfiH0S.js → context-BdgCao9Q.js} +3 -2
- package/dist/context-BdgCao9Q.js.map +1 -0
- package/dist/{cost-BuVDamui.js → cost-BdRgySQl.js} +2 -2
- package/dist/{cost-BuVDamui.js.map → cost-BdRgySQl.js.map} +1 -1
- package/dist/cost-Cr26Alhq.js +1 -0
- package/dist/{cursor-catalog-BTK_apL-.js → cursor-catalog-DEzan4IT.js} +3 -3
- package/dist/{cursor-catalog-BTK_apL-.js.map → cursor-catalog-DEzan4IT.js.map} +1 -1
- package/dist/{db-C5SZ0510.js → db-BzOJ4Js4.js} +130 -84
- package/dist/db-BzOJ4Js4.js.map +1 -0
- package/dist/db-CQRZ3URX.js +2 -0
- package/dist/debug-CF4p4Lp_.js +1 -0
- package/dist/{debug-B5jDImcW.js → debug-CIXzB4wK.js} +2 -2
- package/dist/{debug-B5jDImcW.js.map → debug-CIXzB4wK.js.map} +1 -1
- package/dist/{default-effort-FEidNAIi.js → default-effort-BBDPbJ4D.js} +1 -1
- package/dist/{default-effort-BVmx7r7L.js → default-effort-BDKIxJyC.js} +4 -4
- package/dist/{default-effort-BVmx7r7L.js.map → default-effort-BDKIxJyC.js.map} +1 -1
- package/dist/{dist-ByYk_wAs.js → dist-Db7ifchA.js} +37 -8
- package/dist/dist-Db7ifchA.js.map +1 -0
- package/dist/{docker-BlSopHpZ.js → docker--mukta3L.js} +1 -1
- package/dist/{docker-B2F8-PGC.js → docker-DFrjoNs-.js} +3 -3
- package/dist/{docker-B2F8-PGC.js.map → docker-DFrjoNs-.js.map} +1 -1
- package/dist/{docker-Dto8ROCq.js → docker-DG_uFgn2.js} +109 -77
- package/dist/docker-DG_uFgn2.js.map +1 -0
- package/dist/{doctor-BcvfKpBX.js → doctor-DWK4ttex.js} +84 -27
- package/dist/doctor-DWK4ttex.js.map +1 -0
- package/dist/{environment-CC1RV21n.js → environment-cPMCOk-h.js} +79 -39
- package/dist/environment-cPMCOk-h.js.map +1 -0
- package/dist/{errors-BQV__TzH.js → errors-BZgLSZtV.js} +2 -2
- package/dist/{errors-BQV__TzH.js.map → errors-BZgLSZtV.js.map} +1 -1
- package/dist/{eval-source-CzxdgFBW.js → eval-source-C9b1h8L6.js} +10 -10
- package/dist/{eval-source-CzxdgFBW.js.map → eval-source-C9b1h8L6.js.map} +1 -1
- package/dist/{evaluate-NTNJ8T9H.js → evaluate-B12Tosvq.js} +377 -142
- package/dist/evaluate-B12Tosvq.js.map +1 -0
- package/dist/{evaluate-DToSB2bt.d.ts → evaluate-COdzf7Hl.d.ts} +17 -26
- package/dist/{files-aAzoGNCB.js → files-BOBdC0uI.js} +5 -5
- package/dist/{files-aAzoGNCB.js.map → files-BOBdC0uI.js.map} +1 -1
- package/dist/files-C1Rk9FYA.js +1 -0
- package/dist/files-DMQ9-Vv5.js +1 -0
- package/dist/{files-BGlHnC46.js → files-DgXyaAg6.js} +2 -2
- package/dist/{files-BGlHnC46.js.map → files-DgXyaAg6.js.map} +1 -1
- package/dist/files-Di0xDbCc.js +1 -0
- package/dist/{files-p-uTdOjA.js → files-DlQshsks.js} +3 -2
- package/dist/files-DlQshsks.js.map +1 -0
- package/dist/{files-DsXULUi5.js → files-F5EG0WPT.js} +5 -5
- package/dist/{files-DsXULUi5.js.map → files-F5EG0WPT.js.map} +1 -1
- package/dist/files-G5jf8wBp.js +1 -0
- package/dist/{files-CZXrkPja.js → files-s28a6czq.js} +54 -13
- package/dist/{files-CZXrkPja.js.map → files-s28a6czq.js.map} +1 -1
- package/dist/files-tiNWe8Ih.js +64 -0
- package/dist/files-tiNWe8Ih.js.map +1 -0
- package/dist/gate-CgYdaprG.js +49 -0
- package/dist/gate-CgYdaprG.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-C1MB5bjr.js → git-BpoqEgDV.js} +2 -2
- package/dist/{git-C1MB5bjr.js.map → git-BpoqEgDV.js.map} +1 -1
- package/dist/{harness-DGsRZDSo.js → harness-C2berMiM.js} +3 -3
- package/dist/{harness-DGsRZDSo.js.map → harness-C2berMiM.js.map} +1 -1
- package/dist/{harness-DYSC4zMh.js → harness-DXv2gnaT.js} +4 -2
- package/dist/harness-DXv2gnaT.js.map +1 -0
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{ids-DuK6xl4w.js → ids-C_advdJA.js} +3 -8
- package/dist/ids-C_advdJA.js.map +1 -0
- package/dist/{images-CNayXCp2.js → images-D2jFze2v.js} +98 -10
- package/dist/images-D2jFze2v.js.map +1 -0
- package/dist/index.d.ts +23 -3
- package/dist/index.js +5 -5
- package/dist/{init-D5HtmyMI.js → init-Bi1Q3lRi.js} +7 -6
- package/dist/init-Bi1Q3lRi.js.map +1 -0
- package/dist/init-CUw65o36.js +1 -0
- package/dist/{ink-list-Ca-5dunO.js → ink-list-BpfidQ1h.js} +121 -31
- package/dist/ink-list-BpfidQ1h.js.map +1 -0
- package/dist/{invocations-DrGXF9dT.js → invocations-CxGNOlh0.js} +3 -3
- package/dist/{invocations-DrGXF9dT.js.map → invocations-CxGNOlh0.js.map} +1 -1
- package/dist/judge-COMXT1Hx.js +230 -0
- package/dist/judge-COMXT1Hx.js.map +1 -0
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-m-mb0zv8.js → judgments-CzgpsTqv.js} +2 -2
- package/dist/{judgments-m-mb0zv8.js.map → judgments-CzgpsTqv.js.map} +1 -1
- package/dist/kinds-Bh2I35id.js +698 -0
- package/dist/kinds-Bh2I35id.js.map +1 -0
- package/dist/{summary-Uzc9osfV.js → layout-Cm85tQYk.js} +39 -150
- package/dist/layout-Cm85tQYk.js.map +1 -0
- package/dist/list-DuThsE28.js +1348 -0
- package/dist/list-DuThsE28.js.map +1 -0
- package/dist/{live-B71o2TkX.js → live-CgYbO4Te.js} +6 -4
- package/dist/{live-B71o2TkX.js.map → live-CgYbO4Te.js.map} +1 -1
- package/dist/{live-uRURFDMZ.js → live-DL0se4Zd.js} +7 -7
- package/dist/{live-uRURFDMZ.js.map → live-DL0se4Zd.js.map} +1 -1
- package/dist/live-DZLmj0iG.js +1 -0
- package/dist/{matchers-Cs8aDqdU.js → matchers-CXLn-1jm.js} +3 -2
- package/dist/{matchers-Cs8aDqdU.js.map → matchers-CXLn-1jm.js.map} +1 -1
- package/dist/{model-CXVVB4RK.js → model-CkNVAxob.js} +1 -1
- package/dist/model-Cv4GuTsZ.js +1 -0
- package/dist/{model-1L19Zukh.js → model-CwwkrCb7.js} +5 -5
- package/dist/{model-1L19Zukh.js.map → model-CwwkrCb7.js.map} +1 -1
- package/dist/{model-Ce9l9_rF.js → model-V3mMXp6x.js} +203 -313
- package/dist/model-V3mMXp6x.js.map +1 -0
- package/dist/{model-BxBWFOZn.js → model-cgt5uvu8.js} +5 -5
- package/dist/{model-BxBWFOZn.js.map → model-cgt5uvu8.js.map} +1 -1
- package/dist/{node-DhIRSrrK.js → node-D7pAU--0.js} +14 -10
- package/dist/node-D7pAU--0.js.map +1 -0
- package/dist/{opencode-providers-Co7plCea.js → opencode-providers-BGiXt8_B.js} +104 -35
- package/dist/opencode-providers-BGiXt8_B.js.map +1 -0
- package/dist/{openrouter-models-DNvLRYM7.js → openrouter-models-Cw1z8-sq.js} +2 -2
- package/dist/{openrouter-models-DNvLRYM7.js.map → openrouter-models-Cw1z8-sq.js.map} +1 -1
- package/dist/options-DPUmk7v_.js +1 -0
- package/dist/{gate-Cp_pQoEe.js → pairs-BGb134Vu.js} +2 -48
- package/dist/pairs-BGb134Vu.js.map +1 -0
- package/dist/{patch-ug5HpMOj.js → patch-DAi1nCuH.js} +2 -2
- package/dist/{patch-ug5HpMOj.js.map → patch-DAi1nCuH.js.map} +1 -1
- package/dist/plain-BYUf_fUV.js +267 -0
- package/dist/plain-BYUf_fUV.js.map +1 -0
- package/dist/{plan-DqnzfDH1.js → plan-CsPilzpk.js} +110 -39
- package/dist/plan-CsPilzpk.js.map +1 -0
- package/dist/{plugins-B2KQ0y3P.js → plugins-BfnhazOv.js} +10 -267
- package/dist/plugins-BfnhazOv.js.map +1 -0
- package/dist/{preflight-DyzmvK7i.js → preflight-BixaE4BA.js} +16 -9
- package/dist/preflight-BixaE4BA.js.map +1 -0
- package/dist/preflight-Bn3ianye.js +1 -0
- package/dist/{preflight-DVXEvzFU.js → preflight-D1C-iH0U.js} +1 -1
- package/dist/preflight-GkpgrBxP.js +68 -0
- package/dist/preflight-GkpgrBxP.js.map +1 -0
- package/dist/{prepare-BcSRa7-3.js → prepare-C--rM_wO.js} +2 -2
- package/dist/{prepare-21WMQZyt.js → prepare-DQEDXoDp.js} +9 -9
- package/dist/{prepare-21WMQZyt.js.map → prepare-DQEDXoDp.js.map} +1 -1
- package/dist/{prices-DYJA0tJN.js → prices-5m3SzMZ3.js} +2 -2
- package/dist/{prices-DYJA0tJN.js.map → prices-5m3SzMZ3.js.map} +1 -1
- package/dist/{provenance-iRxjp4Pu.js → provenance-BXyYwUC8.js} +4 -3
- package/dist/provenance-BXyYwUC8.js.map +1 -0
- package/dist/provider-BnUoz4KB.js +1 -0
- package/dist/{provider-DZd8t_XW.js → provider-BxzZq3Th.js} +2 -2
- package/dist/{provider-DZd8t_XW.js.map → provider-BxzZq3Th.js.map} +1 -1
- package/dist/{providers-vXMXuwn-.js → providers-1p4HKYIp.js} +152 -9
- package/dist/providers-1p4HKYIp.js.map +1 -0
- package/dist/providers-B-ijCktf.js +1 -0
- package/dist/query-BYhyf0LR.js +1 -0
- package/dist/{query-BhaKQ07P.js → query-DhhU9ADI.js} +2 -2
- package/dist/{query-BhaKQ07P.js.map → query-DhhU9ADI.js.map} +1 -1
- package/dist/{questions-BzdLdV3C.js → questions-CMAq4pr8.js} +9 -9
- package/dist/{questions-BzdLdV3C.js.map → questions-CMAq4pr8.js.map} +1 -1
- package/dist/{record-DhzwkSXc.js → record-6IUzwP9K.js} +5 -3
- package/dist/record-6IUzwP9K.js.map +1 -0
- package/dist/{remarks-CauU408t.js → remarks-BNABzemt.js} +115 -35
- package/dist/remarks-BNABzemt.js.map +1 -0
- package/dist/{remarks-NMgPd30U.js → remarks-DPCONFxZ.js} +71 -50
- package/dist/remarks-DPCONFxZ.js.map +1 -0
- package/dist/{remote-Dwy3fyo3.js → remote-U7fNt0UP.js} +6 -6
- package/dist/{remote-Dwy3fyo3.js.map → remote-U7fNt0UP.js.map} +1 -1
- package/dist/resolve-C14Pmtej.js +1 -0
- package/dist/{resolve-B1zMnPwK.js → resolve-Cx3a9UQ2.js} +10 -9
- package/dist/resolve-Cx3a9UQ2.js.map +1 -0
- package/dist/{results-CfC0id_u.js → results-CQKXmHym.js} +6 -6
- package/dist/{results-CfC0id_u.js.map → results-CQKXmHym.js.map} +1 -1
- package/dist/{resume-Dkhrpjow.js → resume-POpZa1ge.js} +169 -27
- package/dist/resume-POpZa1ge.js.map +1 -0
- package/dist/retry-CmG809Mp.js +105 -0
- package/dist/retry-CmG809Mp.js.map +1 -0
- package/dist/{runtime-DcvQcr36.js → runtime-Xm9oeC7o.js} +26 -7
- package/dist/runtime-Xm9oeC7o.js.map +1 -0
- package/dist/sandbox/boat.d.ts +26 -0
- package/dist/sandbox/boat.js +37 -0
- package/dist/sandbox/boat.js.map +1 -0
- package/dist/{sandbox-8zCrtuCf.js → sandbox-DrlL4yqb.js} +31 -3
- package/dist/sandbox-DrlL4yqb.js.map +1 -0
- package/dist/sandbox.d.ts +20 -2
- package/dist/sandbox.js +1 -1
- package/dist/{server-Dplq8g0r.js → server-C00zd6zl.js} +9 -9
- package/dist/{server-Dplq8g0r.js.map → server-C00zd6zl.js.map} +1 -1
- package/dist/server-C1UJcpUh.js +1 -0
- package/dist/{shared-ogqPJ0uo.js → shared-Cfi45Bvy.js} +9 -2
- package/dist/shared-Cfi45Bvy.js.map +1 -0
- package/dist/slug-IHh3f770.js +136 -0
- package/dist/slug-IHh3f770.js.map +1 -0
- package/dist/{source-NWHqn5-o.js → source-DDgr48Gn.js} +2 -2
- package/dist/{source-NWHqn5-o.js.map → source-DDgr48Gn.js.map} +1 -1
- package/dist/source-DJHmQv_j.js +1 -0
- package/dist/src-1dBuXzs4.js +3 -0
- package/dist/{src-CUpoVVb8.js → src-BKpioayC.js} +2 -2
- package/dist/{src-B-L8O1Ld.js → src-BNQznj4z.js} +2 -2
- package/dist/{src-B-L8O1Ld.js.map → src-BNQznj4z.js.map} +1 -1
- package/dist/src-BY1cCaoJ.js +2 -0
- package/dist/src-BbZYaQ8b.js +2 -0
- package/dist/src-CDFH2vVN.js +2 -0
- package/dist/src-CMW5R9Q0.js +1 -0
- package/dist/{src-BQXdf-Gw.js → src-CV_7DkPa.js} +47 -45
- package/dist/{src-BQXdf-Gw.js.map → src-CV_7DkPa.js.map} +1 -1
- package/dist/{src-KlDOGD0B.js → src-Cb2MWozp.js} +3 -1
- package/dist/{src-KlDOGD0B.js.map → src-Cb2MWozp.js.map} +1 -1
- package/dist/{src-B9oE_mNS.js → src-CrV2WA2i.js} +2 -2
- package/dist/src-CsFaawcZ.js +2 -0
- package/dist/{src-HTG2SoNz.js → src-DAzaCTcN.js} +2 -2
- package/dist/{src-HTG2SoNz.js.map → src-DAzaCTcN.js.map} +1 -1
- package/dist/{src-uluei5nU.js → src-DvGns8cz.js} +2 -2
- package/dist/{src-uluei5nU.js.map → src-DvGns8cz.js.map} +1 -1
- package/dist/{src-B_igzM1O.js → src-Dw79VLa3.js} +3 -1
- package/dist/{src-B_igzM1O.js.map → src-Dw79VLa3.js.map} +1 -1
- package/dist/{src-D5t4OiRt.js → src-KxS91P7L.js} +3 -1
- package/dist/{src-D5t4OiRt.js.map → src-KxS91P7L.js.map} +1 -1
- package/dist/{src-BvM0jWuZ.js → src-YvblOrBU.js} +2 -2
- package/dist/{src-BvM0jWuZ.js.map → src-YvblOrBU.js.map} +1 -1
- package/dist/src-wKwznPgt.js +2 -0
- package/dist/{src-BQ6BiXoc.js → src-y0nOeRRE.js} +610 -72
- package/dist/src-y0nOeRRE.js.map +1 -0
- package/dist/{state-DDCGUhva.js → state-328JpnHA.js} +17 -17
- package/dist/{state-DDCGUhva.js.map → state-328JpnHA.js.map} +1 -1
- package/dist/state-Ca55Gk_O.js +1 -0
- package/dist/suite-CKBa6MKt.js +2 -0
- package/dist/{suite-M8_7xio7.js → suite-COglUhPd.js} +70 -92
- package/dist/suite-COglUhPd.js.map +1 -0
- package/dist/summary-DDy9DfyJ.js +148 -0
- package/dist/summary-DDy9DfyJ.js.map +1 -0
- package/dist/tart-cli-C-GsbVsQ.js +1 -0
- package/dist/tart-cli-CU0nFS-L.js +55 -0
- package/dist/tart-cli-CU0nFS-L.js.map +1 -0
- package/dist/tart-dWoJvWHc.js +586 -0
- package/dist/tart-dWoJvWHc.js.map +1 -0
- package/dist/tart-images-DZ_Mbh-g.js +233 -0
- package/dist/tart-images-DZ_Mbh-g.js.map +1 -0
- package/dist/tart-ref-7Qv3cm3B.js +1 -0
- package/dist/tart-ref-DYZijjrU.js +55 -0
- package/dist/tart-ref-DYZijjrU.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +4 -4
- package/dist/trace-Dx2NBE1O.js +311 -0
- package/dist/trace-Dx2NBE1O.js.map +1 -0
- package/dist/traits-DKnYkqeK.js +66 -0
- package/dist/traits-DKnYkqeK.js.map +1 -0
- package/dist/{transcript-Cg4SeqRX.js → transcript-CCf07UmU.js} +2 -2
- package/dist/{transcript-Cg4SeqRX.js.map → transcript-CCf07UmU.js.map} +1 -1
- package/dist/{types-C4OkYhZg.js → types-C2mLpph3.js} +3 -2
- package/dist/types-C2mLpph3.js.map +1 -0
- package/dist/{types-DGvexkcp.d.ts → types-DBwFsGem.d.ts} +352 -76
- package/dist/{judge-DYOF69Oe.js → vercel-CGlSBv63.js} +3 -229
- package/dist/vercel-CGlSBv63.js.map +1 -0
- package/dist/{vertex-BLQMayc1.js → vertex-Eb2wqahk.js} +2 -2
- package/dist/{vertex-BLQMayc1.js.map → vertex-Eb2wqahk.js.map} +1 -1
- package/dist/vertex-O88TCwgY.js +1 -0
- package/dist/view/index.d.ts +101 -58
- package/dist/view/index.js +9 -8
- package/dist/views-B0Yt_Q1S.js +2 -0
- package/dist/{views-BISrsIjw.js → views-BROacjBv.js} +7 -6
- package/dist/views-BROacjBv.js.map +1 -0
- package/dist/{worker-entry-DJD_nHkS.js → worker-entry-Smb4bNDc.js} +11 -8
- package/dist/{worker-entry-DJD_nHkS.js.map → worker-entry-Smb4bNDc.js.map} +1 -1
- package/dist/workspace-C0dDP75l.js +1 -0
- package/dist/{workspace-BO1VhUR9.js → workspace-DNDTEk8v.js} +6 -6
- package/dist/{workspace-BO1VhUR9.js.map → workspace-DNDTEk8v.js.map} +1 -1
- package/package.json +24 -11
- package/skills/eval-rubric/SKILL.md +15 -7
- package/dist/adapter-CptK032x.js.map +0 -1
- package/dist/app-files-sgqwfjtn.js.map +0 -1
- package/dist/assemble-CxBHkavM.js +0 -1
- package/dist/assemble-Me7Whp-y.js.map +0 -1
- package/dist/capture-C3RmjPph.js.map +0 -1
- package/dist/capture-YPwxp_iK.js +0 -1
- package/dist/cli-BBOlXKPG.js.map +0 -1
- package/dist/cli-spec-DZ0bnP0_.js.map +0 -1
- package/dist/client-tZ2WBDxf.js.map +0 -1
- package/dist/command-CFQ9c1hk.js.map +0 -1
- package/dist/command-fO0BFsTq.js.map +0 -1
- package/dist/command-qGS802uX.js.map +0 -1
- package/dist/commands-CC60TOT5.js.map +0 -1
- package/dist/config-C2UtS9vy.js.map +0 -1
- package/dist/context-BajfiH0S.js.map +0 -1
- package/dist/contrasts-BjOg7wyr.js +0 -101
- package/dist/contrasts-BjOg7wyr.js.map +0 -1
- package/dist/cost-BJU0MAji.js +0 -1
- package/dist/db-C5SZ0510.js.map +0 -1
- package/dist/db-puzLzX4k.js +0 -2
- package/dist/debug-BnXV7JbX.js +0 -1
- package/dist/dist-ByYk_wAs.js.map +0 -1
- package/dist/docker-Dto8ROCq.js.map +0 -1
- package/dist/doctor-BcvfKpBX.js.map +0 -1
- package/dist/environment-CC1RV21n.js.map +0 -1
- package/dist/evaluate-NTNJ8T9H.js.map +0 -1
- package/dist/files-COW5JBJM.js +0 -1
- package/dist/files-DFtuRQJx.js +0 -1
- package/dist/files-oOYNR7ZH.js +0 -1
- package/dist/files-p-uTdOjA.js.map +0 -1
- package/dist/gate-Cp_pQoEe.js.map +0 -1
- package/dist/harness-DYSC4zMh.js.map +0 -1
- package/dist/ids-DuK6xl4w.js.map +0 -1
- package/dist/images-CNayXCp2.js.map +0 -1
- package/dist/init-8o6W3_fn.js +0 -1
- package/dist/init-D5HtmyMI.js.map +0 -1
- package/dist/ink-list-Ca-5dunO.js.map +0 -1
- package/dist/judge-DYOF69Oe.js.map +0 -1
- package/dist/list-CM_YSJXq.js +0 -374
- package/dist/list-CM_YSJXq.js.map +0 -1
- package/dist/live-DQc8jr6o.js +0 -1
- package/dist/model-Ce9l9_rF.js.map +0 -1
- package/dist/model-Cxzo4cjw.js +0 -1
- package/dist/node-DhIRSrrK.js.map +0 -1
- package/dist/opencode-providers-Co7plCea.js.map +0 -1
- package/dist/plan-DqnzfDH1.js.map +0 -1
- package/dist/plugins-B2KQ0y3P.js.map +0 -1
- package/dist/preflight-DyzmvK7i.js.map +0 -1
- package/dist/provenance-iRxjp4Pu.js.map +0 -1
- package/dist/provider-DCdTHQpk.js +0 -1
- package/dist/providers-D0ob_tra.js +0 -1
- package/dist/providers-vXMXuwn-.js.map +0 -1
- package/dist/query-CvgUsvep.js +0 -1
- package/dist/record-DhzwkSXc.js.map +0 -1
- package/dist/remarks-CauU408t.js.map +0 -1
- package/dist/remarks-NMgPd30U.js.map +0 -1
- package/dist/resolve-B1zMnPwK.js.map +0 -1
- package/dist/resolve-BIIeVT12.js +0 -1
- package/dist/resume-Dkhrpjow.js.map +0 -1
- package/dist/runtime-DcvQcr36.js.map +0 -1
- package/dist/sandbox-8zCrtuCf.js.map +0 -1
- package/dist/server-0tuj8VSa.js +0 -1
- package/dist/shared-ogqPJ0uo.js.map +0 -1
- package/dist/source-llwLewOU.js +0 -1
- package/dist/src-BQ6BiXoc.js.map +0 -1
- package/dist/src-BXnv0ViU.js +0 -2
- package/dist/src-BiS8YNN_.js +0 -3
- package/dist/src-BkH4GOxu.js +0 -1
- package/dist/src-C-tH5ebd.js +0 -2
- package/dist/src-Cg1bFs9g.js +0 -2
- package/dist/src-DV_UorZv.js +0 -2
- package/dist/src-XxUpr2sa.js +0 -2
- package/dist/state-DZbAO95a.js +0 -1
- package/dist/suite-M8_7xio7.js.map +0 -1
- package/dist/suite-u2KJnGpY.js +0 -2
- package/dist/summary-Uzc9osfV.js.map +0 -1
- package/dist/types-C4OkYhZg.js.map +0 -1
- package/dist/vertex-h_peaZTN.js +0 -1
- package/dist/views-BISrsIjw.js.map +0 -1
- package/dist/views-Bfab8fM9.js +0 -2
- package/dist/workspace-B8EOCuxl.js +0 -1
|
@@ -257,13 +257,23 @@ type UpstreamCount = {
|
|
|
257
257
|
provider: string;
|
|
258
258
|
calls: number;
|
|
259
259
|
};
|
|
260
|
+
/** An infra error an attempt ended with before it ran again, and when that try started. */
|
|
261
|
+
type AttemptRetry = {
|
|
262
|
+
error: AttemptError;
|
|
263
|
+
at: string | null;
|
|
264
|
+
};
|
|
260
265
|
type AttemptRecord = {
|
|
261
|
-
/**
|
|
266
|
+
/**
|
|
267
|
+
* `<eval key> # <cell key>#<n>`, or `<eval key> [<case>] # <cell key>#<n>` on an eval with
|
|
268
|
+
* `cases`: unique in the run.
|
|
269
|
+
*/
|
|
262
270
|
key: string;
|
|
263
|
-
/** `<cell key>#<n
|
|
271
|
+
/** `<cell key>#<n>`, or `<cell key>#<case>#<n>` on an eval with `cases`: unique in its evaluation. */
|
|
264
272
|
id: string;
|
|
265
273
|
/** The cell's key; see `EvalEntry.cells`. */
|
|
266
274
|
cell: string;
|
|
275
|
+
/** The case's key, on an eval with `cases`; see `EvalEntry.cases`. */
|
|
276
|
+
case?: string;
|
|
267
277
|
/** Run number within the cell, from 1. */
|
|
268
278
|
n: number;
|
|
269
279
|
/**
|
|
@@ -277,6 +287,11 @@ type AttemptRecord = {
|
|
|
277
287
|
startedAt: string | null;
|
|
278
288
|
/** The image the container started from, in `stack.images`; null when the sandbox doesn't start from an image. */
|
|
279
289
|
imageId: string | null;
|
|
290
|
+
/**
|
|
291
|
+
* The image a `tart()` attempt's VM was cloned from, with the digest Tart had for it. Absent in
|
|
292
|
+
* other sandboxes, and in attempts that didn't reach their sandbox.
|
|
293
|
+
*/
|
|
294
|
+
vmImage?: VmImage;
|
|
280
295
|
/** Names of the variables and secrets set in the sandbox, and the brokered ones; never values. */
|
|
281
296
|
env: {
|
|
282
297
|
vars: string[];
|
|
@@ -405,6 +420,13 @@ type AttemptRecord = {
|
|
|
405
420
|
judgements: JudgmentRecord[];
|
|
406
421
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
407
422
|
startRetries?: StartRetry[];
|
|
423
|
+
/**
|
|
424
|
+
* The infra errors this attempt ended with before `reeval run --retry-errors` or `--resume` ran
|
|
425
|
+
* it again, oldest first: each `{ error, at }`, `at` being when that try started (null when it
|
|
426
|
+
* never did). The attempt's own `outcome` and `error` are its last try's. Absent when it never
|
|
427
|
+
* ran again.
|
|
428
|
+
*/
|
|
429
|
+
retries?: AttemptRetry[];
|
|
408
430
|
/**
|
|
409
431
|
* Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
|
|
410
432
|
* without `hosts` in their environment, so they hold its real value while the agent works and
|
|
@@ -497,9 +519,15 @@ type EvalEntry = {
|
|
|
497
519
|
/**
|
|
498
520
|
* Over gated cells, the outcome is `fail` if any failed, else `infra_error` if any is (a cell is
|
|
499
521
|
* `infra_error` only when nothing counted), else `skipped` if all were, else `pass`. An ungated
|
|
500
|
-
* cell's failures don't fail the evaluation.
|
|
522
|
+
* cell's failures don't fail the evaluation. With `cases`, a cell's outcome is over every case.
|
|
501
523
|
*/
|
|
502
524
|
outcome: Outcome;
|
|
525
|
+
/**
|
|
526
|
+
* In records from before an eval with `cases` was one entry, `family` on a case whose
|
|
527
|
+
* `gate: { passRate }` its family held over the mean of its cases. Never written now: see
|
|
528
|
+
* `CaseRecord.verdict`.
|
|
529
|
+
*/
|
|
530
|
+
verdict?: 'family';
|
|
503
531
|
/** A placeholder, `evaluate.todo` or `group.todo`: it never runs, and its outcome is `skipped`. */
|
|
504
532
|
todo: boolean;
|
|
505
533
|
/**
|
|
@@ -521,13 +549,63 @@ type EvalEntry = {
|
|
|
521
549
|
* with secrets masked and screenshots as refs; unset where the evaluation's files weren't kept.
|
|
522
550
|
*/
|
|
523
551
|
answers?: CompareAnswer[];
|
|
524
|
-
/**
|
|
552
|
+
/**
|
|
553
|
+
* In records from before an eval with `cases` was one entry, set on each case, which was an
|
|
554
|
+
* entry of its own. Never written now; reading such a record folds the cases into one entry.
|
|
555
|
+
*/
|
|
525
556
|
family?: {
|
|
526
557
|
id: string;
|
|
527
558
|
name: string;
|
|
528
559
|
case: number;
|
|
529
560
|
cases: number;
|
|
530
561
|
};
|
|
562
|
+
/**
|
|
563
|
+
* The environments (and harnesses) `where` left out, one entry per case and environment,
|
|
564
|
+
* including those of a case it left out everywhere; unset when none.
|
|
565
|
+
*/
|
|
566
|
+
excluded?: ExcludedPair[];
|
|
567
|
+
/**
|
|
568
|
+
* An eval with `cases`: each case that has attempts in this run, in list order, with its own
|
|
569
|
+
* cells. `cells` and `attempts` are over every case. Unset without cases.
|
|
570
|
+
*/
|
|
571
|
+
cases?: CaseRecord[];
|
|
572
|
+
/** With `cases`: how many the list has, `cases` holding those with attempts in this run. */
|
|
573
|
+
caseCount?: number;
|
|
574
|
+
/**
|
|
575
|
+
* With `cases` and `compare`: each variant cell against its control, paired by case. Its
|
|
576
|
+
* `minLift` gates the lift's lower bound, and a lift below it fails the variant cell.
|
|
577
|
+
*/
|
|
578
|
+
lifts?: FamilyLift[];
|
|
579
|
+
};
|
|
580
|
+
/** One case of an eval with `cases`, and how its attempts went. */
|
|
581
|
+
type CaseRecord = {
|
|
582
|
+
/** The case's key: `nextjs` for the case keyed `nextjs`. */
|
|
583
|
+
key: string;
|
|
584
|
+
/** Its place in the case list, from 0. */
|
|
585
|
+
index: number;
|
|
586
|
+
/**
|
|
587
|
+
* The name it ran under in a record from before an eval with `cases` was one entry, when that
|
|
588
|
+
* isn't `<eval name> [<key>]` (a name template's, `fixes GH-101` for `fixes $id`).
|
|
589
|
+
*/
|
|
590
|
+
name?: string;
|
|
591
|
+
/** `EvalEntry.source.case`: the hash of the case's data. */
|
|
592
|
+
hash?: string;
|
|
593
|
+
/**
|
|
594
|
+
* What the case did, over its own cells by each cell's gate. Under a `gate: { passRate }`, the
|
|
595
|
+
* eval's cell gates on the mean over the cases, so `verdict` says this outcome decides nothing.
|
|
596
|
+
*/
|
|
597
|
+
outcome: Outcome;
|
|
598
|
+
/** `family` when a cell's `gate: { passRate }` gates the mean over the cases, not this case. */
|
|
599
|
+
verdict?: 'family';
|
|
600
|
+
/** pass@1 over its gated cells' attempts. */
|
|
601
|
+
passRate: number | null;
|
|
602
|
+
fingerprint: string;
|
|
603
|
+
/** Its own cells: their attempts are those with this `case`. */
|
|
604
|
+
cells: CellRecord[];
|
|
605
|
+
/** The environments (and harnesses) `where` left this case out of; unset when none. */
|
|
606
|
+
excluded?: ExcludedPair[];
|
|
607
|
+
/** The eval's pairwise questions, answered over this case's attempts; unset without any. */
|
|
608
|
+
answers?: CompareAnswer[];
|
|
531
609
|
};
|
|
532
610
|
/** A 95% interval on a pooled rate or difference. */
|
|
533
611
|
type FamilyInterval = {
|
|
@@ -541,7 +619,7 @@ type FamilyInterval = {
|
|
|
541
619
|
*/
|
|
542
620
|
method: 'wilson' | 'newcombe-paired' | 't-cases';
|
|
543
621
|
};
|
|
544
|
-
/** One cell's pass@1 over
|
|
622
|
+
/** One cell's pass@1 over an eval's cases. */
|
|
545
623
|
type FamilyRate = {
|
|
546
624
|
/** The cell's key, the same in every case. */
|
|
547
625
|
cell: string;
|
|
@@ -553,13 +631,19 @@ type FamilyRate = {
|
|
|
553
631
|
/** The mean of the cases' pass@1; at one attempt a case, `pass / counted`. */
|
|
554
632
|
rate: number;
|
|
555
633
|
interval: FamilyInterval;
|
|
634
|
+
/** The cell's `gate: { passRate }`: with cases it gates `rate`, not each case's pass@1. */
|
|
635
|
+
passRate?: number;
|
|
636
|
+
/** Set with `passRate`: `fail` when `rate` is below it. */
|
|
637
|
+
outcome?: 'pass' | 'fail';
|
|
556
638
|
};
|
|
639
|
+
/** A cell's pass@1 over an eval's cases. */
|
|
640
|
+
type CellRate = Omit<FamilyRate, 'cell'>;
|
|
557
641
|
/**
|
|
558
642
|
* `lift` / `drop`: the interval excludes 0. `noise`: it doesn't. `insufficient`: fewer than 6
|
|
559
643
|
* discordant pairs (one attempt a case) or 6 paired cases (several), where no test can decide.
|
|
560
644
|
*/
|
|
561
645
|
type FamilyVerdict = 'lift' | 'drop' | 'noise' | 'insufficient';
|
|
562
|
-
/** One effort metric over
|
|
646
|
+
/** One effort metric over an eval's paired cases. */
|
|
563
647
|
type FamilyEffortMetric = {
|
|
564
648
|
/** Cases with a value on both sides. */
|
|
565
649
|
pairs: number;
|
|
@@ -580,7 +664,7 @@ type FamilyEffort = {
|
|
|
580
664
|
/** Over the cases where every counted attempt reported usage; omitted when none did. */
|
|
581
665
|
tokens?: FamilyEffortMetric;
|
|
582
666
|
};
|
|
583
|
-
/** A variant cell against its control over
|
|
667
|
+
/** A variant cell against its control over an eval's cases, paired by case. */
|
|
584
668
|
type FamilyLift = {
|
|
585
669
|
/** The variant cell's key. */
|
|
586
670
|
variant: string;
|
|
@@ -604,13 +688,17 @@ type FamilyLift = {
|
|
|
604
688
|
/** The interval spans 0. */
|
|
605
689
|
withinNoise: boolean;
|
|
606
690
|
verdict: FamilyVerdict;
|
|
607
|
-
/** The compare's `minLift`:
|
|
691
|
+
/** The compare's `minLift`: with cases it gates this lower bound, not each case's. */
|
|
608
692
|
minLift?: number;
|
|
609
693
|
/** `fail` when `interval.lo` is below `minLift`. */
|
|
610
694
|
outcome: 'pass' | 'fail';
|
|
611
695
|
effort?: FamilyEffort;
|
|
612
696
|
};
|
|
613
|
-
/**
|
|
697
|
+
/**
|
|
698
|
+
* In records from before an eval with `cases` was one entry: the eval over its cases, each case
|
|
699
|
+
* an entry of its own. Reading such a record folds it into the eval's `cases`, `lifts` and each
|
|
700
|
+
* cell's `rate`.
|
|
701
|
+
*/
|
|
614
702
|
type FamilyRecord = {
|
|
615
703
|
/** `file › group › … › template`. */
|
|
616
704
|
id: string;
|
|
@@ -620,13 +708,24 @@ type FamilyRecord = {
|
|
|
620
708
|
name: string;
|
|
621
709
|
/** Cases in the list. */
|
|
622
710
|
cases: number;
|
|
623
|
-
/**
|
|
711
|
+
/**
|
|
712
|
+
* Cases with an attempt that ran (not skipped) in this run; fewer when a filter or tag left
|
|
713
|
+
* some out, or an interrupted run or a filtered resume hasn't run them yet.
|
|
714
|
+
*/
|
|
624
715
|
ran: number;
|
|
625
716
|
/** One per cell. */
|
|
626
717
|
rates: FamilyRate[];
|
|
627
718
|
/** One per variant cell, with `compare`. */
|
|
628
719
|
lifts: FamilyLift[];
|
|
629
|
-
/**
|
|
720
|
+
/**
|
|
721
|
+
* Every case and environment (and harness) pair `where` left out, including those of a case it
|
|
722
|
+
* left out everywhere, which has no evaluation; unset when none.
|
|
723
|
+
*/
|
|
724
|
+
excluded?: ExcludedPair[];
|
|
725
|
+
/**
|
|
726
|
+
* `fail` when a lift is below its `minLift` or a rate below its `passRate`. A case still fails
|
|
727
|
+
* on its own under `gate: 'all'`.
|
|
728
|
+
*/
|
|
630
729
|
outcome: 'pass' | 'fail';
|
|
631
730
|
};
|
|
632
731
|
/** A variant cell's pass@1 against the control's, on the same harness and sandbox. */
|
|
@@ -663,6 +762,7 @@ type FingerprintParts = {
|
|
|
663
762
|
/** Over the harness descriptor, less concurrency caps. */
|
|
664
763
|
harness: string;
|
|
665
764
|
environment: string | null;
|
|
765
|
+
/** The Docker image's id, or the digest of the image a `tart()` VM was cloned from. */
|
|
666
766
|
image: string | null;
|
|
667
767
|
agentCli: string | null;
|
|
668
768
|
/** Over the judge's model, threshold and version. */
|
|
@@ -723,7 +823,13 @@ type CellRecord = Cell & {
|
|
|
723
823
|
* reports the worst outcome. A variant below its `minLift` fails too.
|
|
724
824
|
*/
|
|
725
825
|
outcome: Outcome;
|
|
826
|
+
/** Unset on an eval with several `cases`, whose variants have `EvalEntry.lifts`. */
|
|
726
827
|
lift?: Lift;
|
|
828
|
+
/**
|
|
829
|
+
* On an eval with several `cases`: the cell's pass@1 over them. A `gate: { passRate }` gates
|
|
830
|
+
* `rate.rate`, the mean of the cases' pass@1, and sets `rate.outcome`.
|
|
831
|
+
*/
|
|
832
|
+
rate?: CellRate;
|
|
727
833
|
/**
|
|
728
834
|
* Each assertion, by label, seen in any attempt of the evaluation, over this cell's counted
|
|
729
835
|
* attempts; on a variant cell, the control's beside it. Unset when no attempt recorded any.
|
|
@@ -785,6 +891,8 @@ type RunInvocation = {
|
|
|
785
891
|
filters: string[];
|
|
786
892
|
/** `--tag`, without the profile's own tags. */
|
|
787
893
|
tags: string[];
|
|
894
|
+
/** `--trait`, as given (`key=value`); unset without any. */
|
|
895
|
+
traits?: string[];
|
|
788
896
|
};
|
|
789
897
|
type RunRecord = {
|
|
790
898
|
kind: 'reeval-run';
|
|
@@ -795,7 +903,13 @@ type RunRecord = {
|
|
|
795
903
|
status: 'complete' | 'interrupted' | 'budget';
|
|
796
904
|
startedAt: string;
|
|
797
905
|
endedAt: string;
|
|
906
|
+
/** How long the run ran, counting each `--retry-errors` of it but not the time between them. */
|
|
798
907
|
durationMs: number;
|
|
908
|
+
/**
|
|
909
|
+
* When each `reeval run --retry-errors` of the run started, oldest first; unset for a run never
|
|
910
|
+
* retried. A `--resume` of the run while it's `interrupted` finishes the last one.
|
|
911
|
+
*/
|
|
912
|
+
retriedAt?: string[];
|
|
799
913
|
author: string;
|
|
800
914
|
git: GitInfo | null;
|
|
801
915
|
runner: RunnerInfo;
|
|
@@ -830,6 +944,7 @@ type RunRecord = {
|
|
|
830
944
|
maxVariants: number;
|
|
831
945
|
};
|
|
832
946
|
harness?: string[];
|
|
947
|
+
traits?: Record<string, string | string[]>;
|
|
833
948
|
};
|
|
834
949
|
} | null;
|
|
835
950
|
/** How `reeval run` was started; `--resume` plans with it again. Unset in older records. */
|
|
@@ -878,7 +993,10 @@ type RunRecord = {
|
|
|
878
993
|
};
|
|
879
994
|
summary: RunSummary;
|
|
880
995
|
evals: EvalEntry[];
|
|
881
|
-
/**
|
|
996
|
+
/**
|
|
997
|
+
* In records from before an eval with `cases` was one entry, each eval with `cases` that ran.
|
|
998
|
+
* Never written now; reading such a record folds it into its eval.
|
|
999
|
+
*/
|
|
882
1000
|
families?: FamilyRecord[];
|
|
883
1001
|
/**
|
|
884
1002
|
* These are redacted errors outside any attempt: an evaluation file that didn't load, a rejection
|
|
@@ -991,7 +1109,11 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
|
|
|
991
1109
|
/** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
|
|
992
1110
|
canceled?: number;
|
|
993
1111
|
};
|
|
994
|
-
/**
|
|
1112
|
+
/** One case of an eval with `cases` in a `HistoryEntry`. */
|
|
1113
|
+
type HistoryCase = Pick<CaseRecord, 'key' | 'outcome' | 'verdict' | 'fingerprint' | 'hash'> & {
|
|
1114
|
+
cells: HistoryCell[];
|
|
1115
|
+
};
|
|
1116
|
+
/** A pooled lift as the history listing keeps it: enough to compare its gate. */
|
|
995
1117
|
type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
|
|
996
1118
|
lo: number;
|
|
997
1119
|
hi: number;
|
|
@@ -1036,12 +1158,14 @@ type HistoryEntry = {
|
|
|
1036
1158
|
outcome: Outcome;
|
|
1037
1159
|
passRate: number | null;
|
|
1038
1160
|
cells: HistoryCell[];
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
lifts
|
|
1161
|
+
/** An eval with `cases`: each case that ran, with its own cells; unset without cases. */
|
|
1162
|
+
cases?: HistoryCase[];
|
|
1163
|
+
/** With `cases`: how many the list has. */
|
|
1164
|
+
caseCount?: number;
|
|
1165
|
+
/** With `cases` and `compare`: each variant's lift paired by case. */
|
|
1166
|
+
lifts?: FamilyLiftSummary[];
|
|
1167
|
+
/** With `cases`: its cells' rates over the cases that a `gate: { passRate }` gates; unset when none. */
|
|
1168
|
+
rates?: Array<Pick<FamilyRate, 'cell' | 'rate'> & Required<Pick<FamilyRate, 'passRate' | 'outcome'>>>;
|
|
1045
1169
|
}>;
|
|
1046
1170
|
/** The run's errors outside any attempt; unset when there were none. */
|
|
1047
1171
|
errors?: string[];
|
|
@@ -1621,6 +1745,49 @@ type AttemptContext = {
|
|
|
1621
1745
|
};
|
|
1622
1746
|
};
|
|
1623
1747
|
type EvalFn = (c: AttemptContext) => Promise<void>;
|
|
1748
|
+
/** The body of an eval with `cases`: the attempt and its case. */
|
|
1749
|
+
type CaseFn<T> = (c: AttemptContext, item: T) => Promise<void>;
|
|
1750
|
+
/**
|
|
1751
|
+
* One eval over several inputs: each case runs the body with every cell and run, and the cases
|
|
1752
|
+
* pool into one pass rate (and, with `compare`, one lift paired by case). Per eval only.
|
|
1753
|
+
*/
|
|
1754
|
+
type CaseOptions<T> = {
|
|
1755
|
+
/** The inputs, each JSON data; the body gets one as its second argument. */
|
|
1756
|
+
cases: readonly T[];
|
|
1757
|
+
/**
|
|
1758
|
+
* Each case's key, which names it in the output and the record. Without it, a string or a
|
|
1759
|
+
* finite number is its own key, and an object uses its `id`, else its `name`.
|
|
1760
|
+
*/
|
|
1761
|
+
caseKey?: (item: T) => string;
|
|
1762
|
+
/**
|
|
1763
|
+
* Which case and environment pairs run, decided at plan time. It sees each environment's
|
|
1764
|
+
* `traits`, and with `compare` the control, whose answer holds for every variant. Reading
|
|
1765
|
+
* `harness` decides each harness on its own. The pairs it leaves out are listed in `--dry-run`
|
|
1766
|
+
* and the record's `excluded`.
|
|
1767
|
+
*/
|
|
1768
|
+
where?: Where<T>;
|
|
1769
|
+
};
|
|
1770
|
+
/** Filter facts about an environment, for `where`: never sent to the agent, never prompt data. */
|
|
1771
|
+
type Traits = Readonly<Record<string, string | readonly string[]>>;
|
|
1772
|
+
/** `where`'s predicate: true runs the case on that environment (and harness). */
|
|
1773
|
+
type Where<T> = (pair: {
|
|
1774
|
+
case: T;
|
|
1775
|
+
environment: {
|
|
1776
|
+
readonly name: string;
|
|
1777
|
+
/** The environment's traits; reading a key that no listed environment defines is refused. */
|
|
1778
|
+
readonly traits: Readonly<Record<string, string | readonly string[] | undefined>>;
|
|
1779
|
+
};
|
|
1780
|
+
/** The harness id; reading it decides each harness on its own. */
|
|
1781
|
+
readonly harness: string;
|
|
1782
|
+
}) => boolean;
|
|
1783
|
+
/** A case and environment pair `where` left out. */
|
|
1784
|
+
type ExcludedPair = {
|
|
1785
|
+
/** The case key. */
|
|
1786
|
+
case: string;
|
|
1787
|
+
environment: string;
|
|
1788
|
+
/** Set only when `where` read the harness. */
|
|
1789
|
+
harness?: string;
|
|
1790
|
+
};
|
|
1624
1791
|
/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
|
|
1625
1792
|
type FakeHarnessDescriptor = {
|
|
1626
1793
|
kind: 'fake';
|
|
@@ -1639,6 +1806,11 @@ type FakeHarnessDescriptor = {
|
|
|
1639
1806
|
delayMs?: number;
|
|
1640
1807
|
/** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
|
|
1641
1808
|
ignoreAbort?: boolean;
|
|
1809
|
+
/**
|
|
1810
|
+
* Each turn's recorded agent time, exactly, whatever the wall clock says: a test can compare
|
|
1811
|
+
* agent time without scheduling jitter, which four runs a side can tell apart.
|
|
1812
|
+
*/
|
|
1813
|
+
turnMs?: number;
|
|
1642
1814
|
/** Throw before the first turn starts, to simulate an infra failure. */
|
|
1643
1815
|
failWith?: string;
|
|
1644
1816
|
/** The first this many session starts of an attempt never become ready, even when aborted (`true`: every one). */
|
|
@@ -1672,6 +1844,14 @@ type FakeHarnessDescriptor = {
|
|
|
1672
1844
|
};
|
|
1673
1845
|
/** Chance (0–1) that an attempt writes no files and says so, for flaky pass rates. */
|
|
1674
1846
|
flakeRate?: number;
|
|
1847
|
+
/** Tokens the fake agent reports for every turn, as `turn.usage`; without it, null. */
|
|
1848
|
+
usage?: {
|
|
1849
|
+
input: number;
|
|
1850
|
+
output: number;
|
|
1851
|
+
cacheRead: number;
|
|
1852
|
+
cacheWrite: number;
|
|
1853
|
+
reasoning: number | null;
|
|
1854
|
+
};
|
|
1675
1855
|
/**
|
|
1676
1856
|
* Real sandbox actions, one list per `c.send` (the last list repeats), run in order before
|
|
1677
1857
|
* `files`. `exec` runs as the agent: with the sandbox's own variables, none of the checks'.
|
|
@@ -2099,7 +2279,62 @@ type DockerSandboxDescriptor = {
|
|
|
2099
2279
|
/** Processes and threads at once. Default 1024. */
|
|
2100
2280
|
pids?: number;
|
|
2101
2281
|
};
|
|
2102
|
-
|
|
2282
|
+
/** A macOS VM on this Mac, cloned from `image` for each attempt by Tart, made by `tart()`. */
|
|
2283
|
+
type TartSandboxDescriptor = {
|
|
2284
|
+
kind: 'tart';
|
|
2285
|
+
/** Cell id; defaults to `tart:<image>`, the image's last path segment. */
|
|
2286
|
+
id?: string;
|
|
2287
|
+
/**
|
|
2288
|
+
* The VM each attempt clones: an OCI image such as
|
|
2289
|
+
* `ghcr.io/cirruslabs/macos-sequoia-base:latest`, or a local VM's name. Tart must have it
|
|
2290
|
+
* before the run: `reeval image build` (or `tart pull`) pulls it, and `reeval run` refuses one
|
|
2291
|
+
* `tart list` doesn't show.
|
|
2292
|
+
*/
|
|
2293
|
+
image: string;
|
|
2294
|
+
/**
|
|
2295
|
+
* The most Tart VMs at once, across the run: 1 or 2, and 2 when unset, since macOS runs at most
|
|
2296
|
+
* two macOS VMs at a time.
|
|
2297
|
+
*/
|
|
2298
|
+
maxConcurrent?: number;
|
|
2299
|
+
/** CPUs each clone gets; the image's when unset. */
|
|
2300
|
+
cpus?: number;
|
|
2301
|
+
/** Memory each clone gets: `'8g'`, `'4096m'`, or bytes; the image's when unset. */
|
|
2302
|
+
memory?: string | number;
|
|
2303
|
+
/**
|
|
2304
|
+
* Reach the VM over SSH when its Tart Guest Agent doesn't answer `tart exec`: as `user`
|
|
2305
|
+
* (`admin` when unset), with `key`, a private key the image trusts, absolute or relative to the
|
|
2306
|
+
* file it's written in.
|
|
2307
|
+
*/
|
|
2308
|
+
ssh?: {
|
|
2309
|
+
user?: string;
|
|
2310
|
+
key: string;
|
|
2311
|
+
};
|
|
2312
|
+
};
|
|
2313
|
+
/** A boat.dev machine size: `small` is 2 vCPUs and 4 GB. */
|
|
2314
|
+
type BoatMachineType = 'small' | 'default' | 'large' | 'xlarge';
|
|
2315
|
+
/** The host variable holding a Boat API key, made by `secret.fromEnv`. */
|
|
2316
|
+
type BoatApiKey = {
|
|
2317
|
+
kind: 'secret';
|
|
2318
|
+
from: string;
|
|
2319
|
+
};
|
|
2320
|
+
/** A boat.dev VM per attempt, made by `boat()` from `reeval/sandbox/boat`. */
|
|
2321
|
+
type BoatSandboxDescriptor = {
|
|
2322
|
+
kind: 'boat';
|
|
2323
|
+
/** Cell id; defaults to `boat`, or `boat:<type>` with a `type`. */
|
|
2324
|
+
id?: string;
|
|
2325
|
+
/** The host variable with the Boat API key, read on the host only. */
|
|
2326
|
+
apiKey: BoatApiKey;
|
|
2327
|
+
/**
|
|
2328
|
+
* The start of every sandbox name reeval gives Boat, so a run's cleanup finds only its own:
|
|
2329
|
+
* lowercase letters, digits and dashes.
|
|
2330
|
+
*/
|
|
2331
|
+
prefix: string;
|
|
2332
|
+
/** The machine size. Default `small`. */
|
|
2333
|
+
type?: BoatMachineType;
|
|
2334
|
+
/** The most Boat sandboxes at once, across the run. */
|
|
2335
|
+
maxConcurrent?: number;
|
|
2336
|
+
};
|
|
2337
|
+
type SandboxDescriptor = FakeSandboxDescriptor | DockerSandboxDescriptor | TartSandboxDescriptor | BoatSandboxDescriptor;
|
|
2103
2338
|
/**
|
|
2104
2339
|
* A host environment variable, redacted everywhere reeval writes and recorded by name only. Made by
|
|
2105
2340
|
* the `secret` helper of `defineEnvironment` and `defineConfig` callbacks.
|
|
@@ -2310,6 +2545,12 @@ type EnvironmentSpec = {
|
|
|
2310
2545
|
* parent's plugin of the same name.
|
|
2311
2546
|
*/
|
|
2312
2547
|
plugins?: PluginSource[];
|
|
2548
|
+
/**
|
|
2549
|
+
* Filter facts for `where`, such as `{ stack: 'next' }`: a key to a string or a list of
|
|
2550
|
+
* strings. A child's key replaces its parent's value whole; keys it doesn't set are inherited.
|
|
2551
|
+
* Recorded on the cell, and never written to the sandbox, the prompt or the judge.
|
|
2552
|
+
*/
|
|
2553
|
+
traits?: Traits;
|
|
2313
2554
|
};
|
|
2314
2555
|
/**
|
|
2315
2556
|
* A part of a plugin, as Claude Code's layout bundles it: `skills/<name>/SKILL.md`,
|
|
@@ -2420,6 +2661,8 @@ type Environment = {
|
|
|
2420
2661
|
readonly name: string;
|
|
2421
2662
|
/** Its `extend` chain, root first. */
|
|
2422
2663
|
readonly layers: readonly EnvironmentLayer[];
|
|
2664
|
+
/** Its layers' `traits`, merged by key with the child's winning; `{}` when none. */
|
|
2665
|
+
readonly traits: Traits;
|
|
2423
2666
|
/**
|
|
2424
2667
|
* A child environment layered on this one; wherever it sets a key this one has, it wins.
|
|
2425
2668
|
* `sandbox` is replaced entirely when the child sets one; when the child omits it, the
|
|
@@ -2521,26 +2764,39 @@ type HarnessEdit = {
|
|
|
2521
2764
|
type HarnessData = HarnessDescriptor | HarnessDescriptor[] | HarnessEdit;
|
|
2522
2765
|
/** `HarnessData`, or a function that edits the inherited list (TypeScript only). */
|
|
2523
2766
|
type HarnessOption = HarnessData | ((inherited: HarnessDescriptor[]) => HarnessDescriptor | HarnessDescriptor[]);
|
|
2524
|
-
type CompareGate = 'variants' | 'all' | 'none';
|
|
2525
2767
|
/**
|
|
2526
|
-
*
|
|
2768
|
+
* An edit of the inherited environment list: `add` appends environments, `only` keeps the
|
|
2769
|
+
* inherited ones with these names, in their order.
|
|
2770
|
+
*/
|
|
2771
|
+
type EnvironmentEdit<E = Environment> = {
|
|
2772
|
+
add: E | E[];
|
|
2773
|
+
} | {
|
|
2774
|
+
only: string | string[];
|
|
2775
|
+
};
|
|
2776
|
+
/** One environment, a list (each is its own cell), or an edit of the inherited list, as data. */
|
|
2777
|
+
type EnvironmentsData<E = EnvironmentData> = E | E[] | EnvironmentEdit<E>;
|
|
2778
|
+
/** `EnvironmentsData`, or a function that edits the inherited list (TypeScript only). */
|
|
2779
|
+
type EnvironmentsOption = EnvironmentsData<Environment> | ((inherited: Environment[]) => Environment | Environment[]);
|
|
2780
|
+
type CompareGate = 'variants' | 'all' | 'observe';
|
|
2781
|
+
/**
|
|
2782
|
+
* The control and each variant run as their own environments, with the same prompt and checks.
|
|
2527
2783
|
* `compare` does not layer them. Share setup first with `environment.extend` / `extendEnvironment`.
|
|
2528
2784
|
* `false` is not a value here: a profile turns compare off with `limit.compare`.
|
|
2529
2785
|
*/
|
|
2530
2786
|
type CompareOptions<E = Environment> = {
|
|
2531
2787
|
/**
|
|
2532
|
-
* The
|
|
2533
|
-
*
|
|
2788
|
+
* The environment each variant is measured against. Not layered under the variants, and it
|
|
2789
|
+
* replaces any `environments` the eval inherits.
|
|
2534
2790
|
*/
|
|
2535
|
-
|
|
2791
|
+
control: E;
|
|
2536
2792
|
/** Other environments, each planned and run as itself. */
|
|
2537
2793
|
variants: E[];
|
|
2538
|
-
/** Which cells decide pass/fail: `variants` (default), `all`, or `
|
|
2794
|
+
/** Which cells decide pass/fail: `variants` (default), `all`, or `observe` (none of them). */
|
|
2539
2795
|
gate?: CompareGate;
|
|
2540
2796
|
/** Lift in points from -1 to 1; a variant cell below it fails. */
|
|
2541
2797
|
minLift?: number;
|
|
2542
2798
|
/** Harnesses (ids or descriptors) to compare on; the others run the control only. */
|
|
2543
|
-
|
|
2799
|
+
harnesses?: Array<string | HarnessDescriptor>;
|
|
2544
2800
|
};
|
|
2545
2801
|
/**
|
|
2546
2802
|
* A question a language model answers after the attempts: its text, or the text and its options.
|
|
@@ -2691,18 +2947,22 @@ type AttemptAnswer = {
|
|
|
2691
2947
|
input?: FileRef;
|
|
2692
2948
|
};
|
|
2693
2949
|
/** An option a profile's `limit` can narrow, and `pin` can protect. */
|
|
2694
|
-
type PinnableKey = '
|
|
2950
|
+
type PinnableKey = 'harnesses' | 'runs' | 'compare';
|
|
2695
2951
|
/**
|
|
2696
2952
|
* How a cell's attempts decide its outcome: `'all'` (the default) passes only when every attempt
|
|
2697
2953
|
* passes (pass^k); `{ passRate }` passes when its pass@1 (passes over passes and fails) is at
|
|
2698
|
-
* least that share
|
|
2954
|
+
* least that share; `'observe'` gates nothing, so the eval's attempts run and are recorded, and
|
|
2955
|
+
* only run and infra errors fail it.
|
|
2699
2956
|
*/
|
|
2700
|
-
type GatePolicy = 'all' | {
|
|
2957
|
+
type GatePolicy = 'all' | 'observe' | {
|
|
2701
2958
|
passRate: number;
|
|
2702
2959
|
};
|
|
2703
|
-
/**
|
|
2704
|
-
|
|
2705
|
-
|
|
2960
|
+
/**
|
|
2961
|
+
* A cell's gate: its policy, or `'none'` when its failures don't fail the eval (a control's, or
|
|
2962
|
+
* any cell under `gate: 'observe'`).
|
|
2963
|
+
*/
|
|
2964
|
+
type CellGate = Exclude<GatePolicy, 'observe'> | 'none';
|
|
2965
|
+
/** A compare object: `control` and `variants`. Same as `CompareOptions`. */
|
|
2706
2966
|
type CompareSpec<E = Environment> = CompareOptions<E>;
|
|
2707
2967
|
type OddDigit = '1' | '3' | '5' | '7' | '9';
|
|
2708
2968
|
/**
|
|
@@ -2717,12 +2977,12 @@ type OddAtLeast3<N extends number> = number extends N ? N : `${N}` extends `-${s
|
|
|
2717
2977
|
type Runs<N extends number = number> = number | {
|
|
2718
2978
|
bestOf: OddAtLeast3<N>;
|
|
2719
2979
|
};
|
|
2720
|
-
/** Options other than `
|
|
2980
|
+
/** Options other than `environments` and `compare`, which are exclusive of each other. */
|
|
2721
2981
|
type EvalOptionFields<N extends number = number> = {
|
|
2722
2982
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2723
|
-
|
|
2983
|
+
harnesses?: HarnessOption;
|
|
2724
2984
|
/**
|
|
2725
|
-
* The default machine when the resolved environment omits `sandbox`. Cascades like `
|
|
2985
|
+
* The default machine when the resolved environment omits `sandbox`. Cascades like `harnesses`.
|
|
2726
2986
|
* Not an environment: it does not invent skills, files or a name.
|
|
2727
2987
|
*/
|
|
2728
2988
|
sandbox?: SandboxDescriptor;
|
|
@@ -2730,7 +2990,7 @@ type EvalOptionFields<N extends number = number> = {
|
|
|
2730
2990
|
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more), which stops each cell once
|
|
2731
2991
|
* a majority of its `n` attempts agree and gates it on that majority; not on a compared eval.
|
|
2732
2992
|
* Unset at every level, a compare with variants runs 4 a side, the fewest whose lift can reach
|
|
2733
|
-
* a verdict, and anything else runs once, a
|
|
2993
|
+
* a verdict, and anything else runs once, a case of an eval with `cases` included.
|
|
2734
2994
|
*/
|
|
2735
2995
|
runs?: Runs<N>;
|
|
2736
2996
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
@@ -2784,7 +3044,7 @@ type EvalSource = {
|
|
|
2784
3044
|
body: string;
|
|
2785
3045
|
/**
|
|
2786
3046
|
* sha256 of the case's data as canonical JSON, with secrets in its strings masked as the record
|
|
2787
|
-
* masks them; set on each case of an `
|
|
3047
|
+
* masks them; set on each case of an eval with `cases`.
|
|
2788
3048
|
*/
|
|
2789
3049
|
case?: string;
|
|
2790
3050
|
/** sha256 of the canonical options. */
|
|
@@ -2792,45 +3052,38 @@ type EvalSource = {
|
|
|
2792
3052
|
};
|
|
2793
3053
|
/**
|
|
2794
3054
|
* `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
|
|
2795
|
-
* A compare
|
|
2796
|
-
*
|
|
2797
|
-
* own, or an inherited one) is the control.
|
|
3055
|
+
* A compare names every arm, so an inherited `environments` is not applied, and the same object
|
|
3056
|
+
* can't set both.
|
|
2798
3057
|
*/
|
|
2799
|
-
type EnvironmentOrCompare<E> = {
|
|
2800
|
-
/**
|
|
2801
|
-
|
|
3058
|
+
type EnvironmentOrCompare<E, Envs> = {
|
|
3059
|
+
/**
|
|
3060
|
+
* What this group or eval runs in: one environment, or a list, each its own cell beside
|
|
3061
|
+
* each harness. `{ add }` and `{ only }` edit the inherited list.
|
|
3062
|
+
*/
|
|
3063
|
+
environments?: Envs;
|
|
2802
3064
|
compare?: never;
|
|
2803
3065
|
} | {
|
|
2804
|
-
/** Forbidden beside `compare
|
|
2805
|
-
|
|
2806
|
-
/** Environments to run side by side; `
|
|
2807
|
-
compare: CompareSpec<E
|
|
2808
|
-
baseline: E;
|
|
2809
|
-
};
|
|
2810
|
-
} | {
|
|
2811
|
-
/** The control: the compare has no `baseline`. */
|
|
2812
|
-
environment?: E;
|
|
2813
|
-
/** Environments to run side by side against `environment`, the control. */
|
|
2814
|
-
compare: CompareSpec<E> & {
|
|
2815
|
-
baseline?: undefined;
|
|
2816
|
-
};
|
|
3066
|
+
/** Forbidden beside `compare`, whose `control` names what the eval runs in. */
|
|
3067
|
+
environments?: never;
|
|
3068
|
+
/** Environments to run side by side; `control` is what each variant is measured against. */
|
|
3069
|
+
compare: CompareSpec<E>;
|
|
2817
3070
|
};
|
|
2818
3071
|
/** `N` is `runs.bestOf` as written, so an even or fractional literal is a type error. */
|
|
2819
|
-
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment>;
|
|
3072
|
+
type EvalOptions<N extends number = number> = EvalOptionFields<N> & EnvironmentOrCompare<Environment, EnvironmentsOption>;
|
|
2820
3073
|
type GroupOptions<N extends number = number> = EvalOptions<N>;
|
|
2821
3074
|
/**
|
|
2822
3075
|
* `EvalOptions` in their data forms, as JSON carries them: the harness as data, the judge's model
|
|
2823
3076
|
* by name or a fake judge, and environments without `extend`. The same `environment` / compare
|
|
2824
3077
|
* object split as `EvalOptions`.
|
|
2825
3078
|
*/
|
|
2826
|
-
type EvalOptionsData = Omit<EvalOptionFields, '
|
|
3079
|
+
type EvalOptionsData = Omit<EvalOptionFields, 'harnesses' | 'judge'> & {
|
|
2827
3080
|
/** Which agents run the eval; each harness is its own cell. */
|
|
2828
|
-
|
|
3081
|
+
harnesses?: HarnessData;
|
|
2829
3082
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
2830
3083
|
judge?: ConfigJudgeOptions;
|
|
2831
|
-
} & EnvironmentOrCompare<EnvironmentData>;
|
|
3084
|
+
} & EnvironmentOrCompare<EnvironmentData, EnvironmentsData>;
|
|
2832
3085
|
/** An `Environment` as data, as JSON carries it: without `extend` / `extendEnvironment`. */
|
|
2833
|
-
type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend'>;
|
|
3086
|
+
type EnvironmentData = Omit<Environment, 'extendEnvironment' | 'extend' | 'traits'>;
|
|
2834
3087
|
/** The config file is plain data sent to workers, so its judge model can't be an object. */
|
|
2835
3088
|
type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
2836
3089
|
/** `'provider/model'`, or a fake judge. */
|
|
@@ -2852,11 +3105,11 @@ type Budget = {
|
|
|
2852
3105
|
/** What evals get when they don't say otherwise: under groups and evals. */
|
|
2853
3106
|
type ProfileDefaults = {
|
|
2854
3107
|
/** The agents every eval runs on, unless a group or eval replaces them. */
|
|
2855
|
-
|
|
3108
|
+
harnesses?: HarnessDescriptor | HarnessDescriptor[];
|
|
2856
3109
|
/** The default machine when the resolved environment omits `sandbox`. */
|
|
2857
3110
|
sandbox?: SandboxDescriptor;
|
|
2858
|
-
/** What every eval runs in when it names no environment. */
|
|
2859
|
-
|
|
3111
|
+
/** What every eval runs in when it names no environment: one, or a list (a cell each). */
|
|
3112
|
+
environments?: Environment | Environment[];
|
|
2860
3113
|
/** Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more). */
|
|
2861
3114
|
runs?: Runs;
|
|
2862
3115
|
/** How each cell's attempts decide its outcome. */
|
|
@@ -2885,7 +3138,13 @@ type ProfileLimit = {
|
|
|
2885
3138
|
maxVariants: number;
|
|
2886
3139
|
};
|
|
2887
3140
|
/** Harness ids to allow; evals left with none are skipped. */
|
|
2888
|
-
|
|
3141
|
+
harnesses?: string[];
|
|
3142
|
+
/**
|
|
3143
|
+
* Cells to run, by their environment's traits (with `compare`, the control's): each key's trait
|
|
3144
|
+
* must equal the value, or one of a list's, or for a list trait contain it. Evals left with no
|
|
3145
|
+
* cell are skipped.
|
|
3146
|
+
*/
|
|
3147
|
+
traits?: Record<string, string | string[]>;
|
|
2889
3148
|
};
|
|
2890
3149
|
/** How a run is sized: chosen with `--profile`, `REEVAL_PROFILE` or `defaultProfile`. */
|
|
2891
3150
|
type Profile = {
|
|
@@ -3013,14 +3272,17 @@ type ReevalConfig = {
|
|
|
3013
3272
|
/** Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`. */
|
|
3014
3273
|
include?: string[];
|
|
3015
3274
|
/** The agents every eval runs on, unless a group or eval replaces them; default Cursor. */
|
|
3016
|
-
|
|
3275
|
+
harnesses?: HarnessDescriptor | HarnessDescriptor[];
|
|
3017
3276
|
/**
|
|
3018
3277
|
* The default machine when the resolved environment omits `sandbox`. Not an environment:
|
|
3019
3278
|
* it does not invent skills, files or a name. Implicit `docker()` when nobody sets one.
|
|
3020
3279
|
*/
|
|
3021
3280
|
sandbox?: SandboxDescriptor;
|
|
3022
|
-
/**
|
|
3023
|
-
|
|
3281
|
+
/**
|
|
3282
|
+
* What every eval runs in when it names no environment and has no compare: one, or a list (a
|
|
3283
|
+
* cell each).
|
|
3284
|
+
*/
|
|
3285
|
+
environments?: Environment | Environment[];
|
|
3024
3286
|
/**
|
|
3025
3287
|
* Attempts per cell: a number, or `{ bestOf: n }` (odd, 3 or more); unset, a compare with
|
|
3026
3288
|
* variants runs 4 a side and anything else once.
|
|
@@ -3158,9 +3420,8 @@ type PhaseTimings = {
|
|
|
3158
3420
|
type CellRole = 'control' | 'variant';
|
|
3159
3421
|
/**
|
|
3160
3422
|
* A cell of an evaluation combines a harness, an environment (`-` when none is set), and that
|
|
3161
|
-
* environment's sandbox id. With `compare`, the control is `
|
|
3162
|
-
*
|
|
3163
|
-
* including its sandbox.
|
|
3423
|
+
* environment's sandbox id. With `compare`, the control is `compare.control` and each variant is
|
|
3424
|
+
* that environment on its own, including its sandbox.
|
|
3164
3425
|
*/
|
|
3165
3426
|
type Cell = {
|
|
3166
3427
|
/** The harness id, the environment name, and the sandbox id, joined with `/`. */
|
|
@@ -3180,9 +3441,16 @@ type Cell = {
|
|
|
3180
3441
|
* eight-character hash.
|
|
3181
3442
|
*/
|
|
3182
3443
|
sandbox: string;
|
|
3444
|
+
/**
|
|
3445
|
+
* `variant` on a compare's variant cell, else `control`: the compare's control, or any cell of
|
|
3446
|
+
* an eval without `compare`, each environment of an `environments` grid included. A variant's
|
|
3447
|
+
* lift is against the `control` cell on its harness and sandbox.
|
|
3448
|
+
*/
|
|
3183
3449
|
role: CellRole;
|
|
3184
3450
|
/** The variant's name, for variant cells. */
|
|
3185
3451
|
variant?: string;
|
|
3452
|
+
/** The environment's merged traits; unset when it has none. Not part of the key. */
|
|
3453
|
+
traits?: Traits;
|
|
3186
3454
|
};
|
|
3187
3455
|
/** A plugin as an attempt placed it. */
|
|
3188
3456
|
type PluginRecord = {
|
|
@@ -3358,6 +3626,13 @@ type AttemptImage = {
|
|
|
3358
3626
|
digest: string | null;
|
|
3359
3627
|
};
|
|
3360
3628
|
};
|
|
3629
|
+
/** The image a `tart()` attempt's VM was cloned from. */
|
|
3630
|
+
type VmImage = {
|
|
3631
|
+
/** The image as `tart({ image })` names it. */
|
|
3632
|
+
ref: string;
|
|
3633
|
+
/** The OCI manifest digest (`sha256:…`) Tart had for it; null for a local VM, which has none. */
|
|
3634
|
+
digest: string | null;
|
|
3635
|
+
};
|
|
3361
3636
|
/** A harness session start that timed out and was started again. */
|
|
3362
3637
|
type StartRetry = {
|
|
3363
3638
|
/** `<harness kind>: session did not become ready within <startTimeout>`. */
|
|
@@ -3488,10 +3763,11 @@ type CaptureStep = {
|
|
|
3488
3763
|
};
|
|
3489
3764
|
/** Which environment ran as the control, so the studio can show it. */
|
|
3490
3765
|
type EvalCompareMeta = {
|
|
3766
|
+
/** The control's environment name. */
|
|
3491
3767
|
baseline: string;
|
|
3492
3768
|
/**
|
|
3493
|
-
*
|
|
3494
|
-
* an empty
|
|
3769
|
+
* Not written, since a compare names its control. A record that has it ran the eval's cascaded
|
|
3770
|
+
* environment (`'environment'`) or an empty one named `baseline` (`'empty'`) as the control.
|
|
3495
3771
|
*/
|
|
3496
3772
|
from?: 'environment' | 'empty';
|
|
3497
3773
|
};
|
|
@@ -3524,5 +3800,5 @@ type ResolvedGateway = {
|
|
|
3524
3800
|
why: string;
|
|
3525
3801
|
}>;
|
|
3526
3802
|
};
|
|
3527
|
-
export {
|
|
3528
|
-
//# sourceMappingURL=types-
|
|
3803
|
+
export { EvalFn as $, directional as $n, Secret as $t, CellRole as A, HistoryEntry as An, JudgeState as At, Diff as B, EFFORT_MIN_RUNS as Bn, PinnableKey as Bt, CaptureUseAction as C, FamilyLift as Cn, JSONValue as Ct, CaseFn as D, FamilyVerdict as Dn, JudgeInputs as Dt, CaptureWindow as E, FamilyRecord as En, JudgeFailure as Et, CompareCut as F, StackSummary as Fn, OpenRouterGatewayDescriptor as Ft, EgressPolicy as G, Spread as Gn, ProfileLimit as Gt, DirectoryOptions as H, EffortMetric as Hn, PluginSource as Ht, CompareOptions as I, UpstreamCount as In, Outcome as It, Environment as J, bothUnderATenth as Jn, QuestionOptions as Jt, EgressRecord as K, TOKENS_UNREPORTED as Kn, Question as Kt, ConfigHelpers as L, CellEffort as Ln, PairFactor as Lt, ClaudeCodeHarnessDescriptor as M, Lift as Mn, McpServer as Mt, CodexHarnessDescriptor as N, RunInvocation as Nn, Observation as Nt, CaseOptions as O, FileRef as On, JudgeModel as Ot, CompareAnswer as P, RunRecord as Pn, OpenCodeHarnessDescriptor as Pt, ErrorKind as Q, contrastEffort as Qn, SandboxWarning as Qt, ConfigJudgeOptions as R, DIRECTIONAL_MIN as Rn, PairVote as Rt, CaptureUse as S, EvalEntry as Sn, HostFile as St, CaptureWait as T, FamilyRate as Tn, JudgeCallOptions as Tt, DockerSandboxDescriptor as U, EffortSample as Un, Profile as Ut, DirectGatewayDescriptor as V, EffortContrast as Vn, PluginPart as Vt, Duration as W, EffortVerdict as Wn, ProfileDefaults as Wt, EnvironmentInput as X, cellEffort as Xn, ResolvedGateway as Xt, EnvironmentHelpers as Y, cellContrast as Yn, ReevalConfig as Yt, EnvironmentSpec as Z, cellSamples as Zn, SandboxDescriptor as Zt, CaptureInstruction as _, WebEventType as _n, HarnessData as _t, AttemptContext as a, TartSandboxDescriptor as an, effortVerdict as ar, FakeAnswer as at, CapturePage as b, CellRecord as bn, HarnessOption as bt, BlindPair as c, Turn as cn, formatRatio as cr, FakeSandboxDescriptor as ct, BoatSandboxDescriptor as d, VercelGatewayDescriptor as dn, FxHarnessDescriptor as dt, SecretTemplate as en, effortChangeLabel as er, EvalOptions as et, Budget as f, Verdict as fn, GatewayDescriptor as ft, CaptureFailedRequest as g, Wait as gn, GroupOptions as gt, CaptureConsoleMessage as h, Viewport as hn, GrokBuildHarnessDescriptor as ht, AttemptAnswer as i, StatusRange as in, effortSummary as ir, FakeAnalysisModel as it, ClaudeCodeEffort as j, JudgmentRecord as jn, JudgeSubject as jt, Cell as k, HistoryCell as kn, JudgeOptions as kt, BoatApiKey as l, TurnData as ln, formatSeconds as lr, FakeScriptStep as lt, CaptureCall as m, VertexPublisher as mn, GooseHarnessDescriptor as mt, AnalysisConfig as n, SetupStep as nn, effortOf as nr, ExecResult as nt, BackgroundHandle as o, TokenUsage as on, filesChanged as or, FakeHarnessDescriptor as ot, Capture as p, VertexGatewayDescriptor as pn, GatewayName as pt, EnvVars as q, VERDICT_TEXT as qn, QuestionInputs as qt, AntigravityHarnessDescriptor as r, SkillSource as rn, effortSample as rr, ExecRetry as rt, BlindAttempt as s, TranscriptEvent as sn, formatCount as sr, FakeJudgeModel as st, AgentHarnessDescriptor as t, ServeHandle as tn, effortLabel as tr, EvalOptionsData as tt, BoatMachineType as u, TurnStatus as un, tokenValues as ur, FakeVote as ut, CaptureKey as v, ArtifactRef as vn, HarnessDescriptor as vt, CaptureUseScript as w, FamilyLiftSummary as wn, JudgeArtifactRef as wt, CapturePageFile as x, CheckTally as xn, HistoryConfig as xt, CaptureOptions as y, AttemptRecord as yn, HarnessEdit as yt, CursorHarnessDescriptor as z, EFFORT_ALPHA as zn, PiHarnessDescriptor as zt };
|
|
3804
|
+
//# sourceMappingURL=types-DBwFsGem.d.ts.map
|