reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,3 +1,63 @@
|
|
|
1
|
+
//#region src/declaration.d.ts
|
|
2
|
+
/** How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`. */
|
|
3
|
+
type DeclaredMode = 'run' | 'skip' | 'only';
|
|
4
|
+
/** A `group`: options its evals and groups inherit, and its children in declaration order. */
|
|
5
|
+
type DeclaredGroup = {
|
|
6
|
+
type: 'group';
|
|
7
|
+
/** The group's name, as passed to `group`. */
|
|
8
|
+
name: string;
|
|
9
|
+
mode: DeclaredMode;
|
|
10
|
+
/** The group's own options, before the cascade. */
|
|
11
|
+
options: EvalOptionsData;
|
|
12
|
+
children: DeclaredNode[];
|
|
13
|
+
};
|
|
14
|
+
/** An `evaluate`: one eval, planned into a test per attempt. */
|
|
15
|
+
type DeclaredEval = {
|
|
16
|
+
type: 'eval';
|
|
17
|
+
/** The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`). */
|
|
18
|
+
name: string;
|
|
19
|
+
mode: DeclaredMode;
|
|
20
|
+
/** The eval's own options, before the cascade. */
|
|
21
|
+
options: EvalOptionsData;
|
|
22
|
+
/** Names the eval's body to whoever runs it. */
|
|
23
|
+
body: string;
|
|
24
|
+
source: DeclaredSource;
|
|
25
|
+
/** Set on each case of a pooled `evaluate.each`: its template, index and the list's length. */
|
|
26
|
+
family?: DeclaredFamily;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* An eval's own source, hashed by the SDK that declared it: sha256 (hex) of its body's source
|
|
30
|
+
* text, and of an `evaluate.each` case's data as canonical JSON (object keys sorted, no
|
|
31
|
+
* whitespace, non-ASCII characters as themselves).
|
|
32
|
+
*/
|
|
33
|
+
type DeclaredSource = {
|
|
34
|
+
/** sha256 of the body function's source text. */
|
|
35
|
+
body: string;
|
|
36
|
+
/** sha256 of the case's data; set on each case of an `evaluate.each`. */
|
|
37
|
+
case?: string;
|
|
38
|
+
};
|
|
39
|
+
/** Where a case of a pooled `evaluate.each` sits in its list. */
|
|
40
|
+
type DeclaredFamily = {
|
|
41
|
+
/** The name template passed to `evaluate.each`. */
|
|
42
|
+
name: string;
|
|
43
|
+
/** The case's index, from 0. */
|
|
44
|
+
case: number;
|
|
45
|
+
/** How many cases the list has. */
|
|
46
|
+
cases: number;
|
|
47
|
+
};
|
|
48
|
+
/** `evaluate.todo` or `group.todo`. */
|
|
49
|
+
type DeclaredTodo = {
|
|
50
|
+
type: 'todo';
|
|
51
|
+
/** The name passed to `.todo`. */
|
|
52
|
+
name: string;
|
|
53
|
+
};
|
|
54
|
+
/** One node of the tree: a group, an eval or a todo. */
|
|
55
|
+
type DeclaredNode = DeclaredGroup | DeclaredEval | DeclaredTodo;
|
|
56
|
+
/** An eval file's top-level groups, evals and todos, in declaration order. */
|
|
57
|
+
type DeclarationTree = {
|
|
58
|
+
children: DeclaredNode[];
|
|
59
|
+
};
|
|
60
|
+
//#endregion
|
|
1
61
|
//#region src/view/effort.d.ts
|
|
2
62
|
/** Below this two-sided p, an effort difference is an effect. */
|
|
3
63
|
declare const EFFORT_ALPHA = 0.05;
|
|
@@ -192,7 +252,7 @@ type JudgeRetry = {
|
|
|
192
252
|
error: string;
|
|
193
253
|
delayMs: number;
|
|
194
254
|
};
|
|
195
|
-
type
|
|
255
|
+
type JudgmentRecord = {
|
|
196
256
|
question: string;
|
|
197
257
|
verdict: 'pass' | 'fail';
|
|
198
258
|
score: number;
|
|
@@ -200,9 +260,17 @@ type JudgementRecord = {
|
|
|
200
260
|
model: string;
|
|
201
261
|
/** The model id the provider says served the call, when it reports one. */
|
|
202
262
|
resolvedModel?: string;
|
|
263
|
+
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
264
|
+
gateway?: string;
|
|
265
|
+
/** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
|
|
266
|
+
upstream?: string;
|
|
267
|
+
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
268
|
+
costUsd?: number;
|
|
203
269
|
/**
|
|
204
270
|
* `<index>:<sha256(question) prefix>`: the same question at the same place in the eval, so a
|
|
205
|
-
* verdict can be followed across runs (and an edited question starts a new series).
|
|
271
|
+
* verdict can be followed across runs (and an edited question starts a new series). `<index>`
|
|
272
|
+
* is the call's place among the attempt's `c.judge()` calls, counting calls that failed,
|
|
273
|
+
* whichever finishes first.
|
|
206
274
|
*/
|
|
207
275
|
key: string;
|
|
208
276
|
/** Transient provider errors retried before this verdict, with the backoff before each retry. */
|
|
@@ -214,7 +282,7 @@ type AttemptFiles = {
|
|
|
214
282
|
/** Each environment setup step's command, exit code, duration and output. */
|
|
215
283
|
setupLog?: FileRef;
|
|
216
284
|
transcript?: FileRef;
|
|
217
|
-
/**
|
|
285
|
+
/** Unnormalized harness stream parts, only with `REEVAL_RAW_TRANSCRIPT=1`. */
|
|
218
286
|
rawTranscript?: FileRef;
|
|
219
287
|
diff?: FileRef;
|
|
220
288
|
judgeInput?: FileRef;
|
|
@@ -225,6 +293,11 @@ type ContextFile = {
|
|
|
225
293
|
path: string;
|
|
226
294
|
sha256: string;
|
|
227
295
|
};
|
|
296
|
+
/** An upstream provider behind a gateway, and how many of the attempt's calls it served. */
|
|
297
|
+
type UpstreamCount = {
|
|
298
|
+
provider: string;
|
|
299
|
+
calls: number;
|
|
300
|
+
};
|
|
228
301
|
type AttemptRecord = {
|
|
229
302
|
/** `<eval key> # <cell key>#<n>`: unique in the run. */
|
|
230
303
|
key: string;
|
|
@@ -257,6 +330,23 @@ type AttemptRecord = {
|
|
|
257
330
|
* recorded them.
|
|
258
331
|
*/
|
|
259
332
|
servedModels: string[] | null;
|
|
333
|
+
/**
|
|
334
|
+
* The models the harness's successful API calls asked for, when nothing named the one that
|
|
335
|
+
* served them (fx on Vercel AI Gateway with a Gemini model): the agent's word, not the
|
|
336
|
+
* gateway's, so never part of `servedModels`. Absent when there's none.
|
|
337
|
+
*/
|
|
338
|
+
askedModels?: string[];
|
|
339
|
+
/**
|
|
340
|
+
* The upstream providers a gateway said served the harness's own API calls (OpenRouter's
|
|
341
|
+
* `provider`), each with how many it served, in the order first seen; more than one means the
|
|
342
|
+
* gateway spread or failed the calls over between them. Absent when the gateway names none.
|
|
343
|
+
*/
|
|
344
|
+
upstreams?: UpstreamCount[];
|
|
345
|
+
/**
|
|
346
|
+
* What a gateway said the harness's own API calls cost, in US dollars, summed (OpenRouter's
|
|
347
|
+
* `usage.cost`); absent when it names none.
|
|
348
|
+
*/
|
|
349
|
+
reportedCostUsd?: number;
|
|
260
350
|
/**
|
|
261
351
|
* The reasoning effort the harness factory asked for (`claudeCode({ effort: 'high' })`);
|
|
262
352
|
* null when the factory left it unset. Fast is not this.
|
|
@@ -317,14 +407,31 @@ type AttemptRecord = {
|
|
|
317
407
|
/** Longest tool `durationMs` in the transcript; null when none reported a duration. */
|
|
318
408
|
maxToolMs: number | null;
|
|
319
409
|
};
|
|
320
|
-
judgements:
|
|
410
|
+
judgements: JudgmentRecord[];
|
|
321
411
|
/** Harness session starts that didn't become ready within `startTimeout` and were started again (at most one). */
|
|
322
412
|
startRetries?: StartRetry[];
|
|
413
|
+
/**
|
|
414
|
+
* Processes the setup steps left running in a Docker sandbox with a `to: 'checks'` secret
|
|
415
|
+
* without `hosts` in their environment, so they hold its real value while the agent works and
|
|
416
|
+
* the agent, as the same user, can read it from them. Found by the secret's variable name, so a
|
|
417
|
+
* copy under another name or in a command's arguments isn't. At most 20, each
|
|
418
|
+
* `{ command, secrets }`: its command line, masked and cut at 200 characters, and those secrets'
|
|
419
|
+
* names. Absent when there were none.
|
|
420
|
+
*/
|
|
421
|
+
setupProcesses?: SetupProcess[];
|
|
422
|
+
/**
|
|
423
|
+
* Each foreground `c.exec`: its last try's exit code, and `durationMs` over all its tries. With
|
|
424
|
+
* `retries`, `tries` has each try's `{ exitCode, durationMs }` and the log each try's output.
|
|
425
|
+
*/
|
|
323
426
|
checks: Array<{
|
|
324
427
|
command: string;
|
|
325
428
|
exitCode: number;
|
|
326
429
|
durationMs: number;
|
|
327
430
|
log?: FileRef;
|
|
431
|
+
/** The `retries` it was given, when more than 0: it could run `retries + 1` times. */
|
|
432
|
+
retries?: number;
|
|
433
|
+
/** Every try, in order, when it was given `retries`; its log has each try's output. */
|
|
434
|
+
tries?: ExecTry[];
|
|
328
435
|
}>;
|
|
329
436
|
/** Every `expect` and `expect.soft` matcher call that settled, in call order; unset with none. */
|
|
330
437
|
assertions?: AssertionRecord[];
|
|
@@ -333,6 +440,13 @@ type AttemptRecord = {
|
|
|
333
440
|
count?: number;
|
|
334
441
|
}>;
|
|
335
442
|
files: AttemptFiles;
|
|
443
|
+
/**
|
|
444
|
+
* The eval's questions asked of this attempt on its own (those without `across`), in question
|
|
445
|
+
* order; unset without any. They annotate: `outcome` never depends on them. Asked once the
|
|
446
|
+
* attempt ends, when it ran to a pass or a fail. Each answer's `input` is `questions/<n>.json`
|
|
447
|
+
* in the attempt's directory, `<n>` the question's place in the eval's `questions`.
|
|
448
|
+
*/
|
|
449
|
+
answers?: AttemptAnswer[];
|
|
336
450
|
};
|
|
337
451
|
type EvalEntry = {
|
|
338
452
|
/** `file › group › … › name`, as shown. */
|
|
@@ -346,11 +460,21 @@ type EvalEntry = {
|
|
|
346
460
|
name: string;
|
|
347
461
|
/** Results are comparable only when fingerprints match. */
|
|
348
462
|
fingerprint: string;
|
|
349
|
-
/**
|
|
463
|
+
/**
|
|
464
|
+
* The eval file, by content hash: where the eval lives. Records from before `source` also list
|
|
465
|
+
* the local files it imported, and were fingerprinted over them.
|
|
466
|
+
*/
|
|
350
467
|
sources: Array<{
|
|
351
468
|
path: string;
|
|
352
469
|
sha256: string;
|
|
353
470
|
}>;
|
|
471
|
+
/**
|
|
472
|
+
* Hashes of what the `sources` fingerprint part covers but `revision`: the eval's body, its
|
|
473
|
+
* case data and its options with its groups'. Unset on records from before it.
|
|
474
|
+
*/
|
|
475
|
+
source?: EvalSource;
|
|
476
|
+
/** The eval's `revision`, part of the `sources` fingerprint part; unset when it sets none. */
|
|
477
|
+
revision?: Revision;
|
|
354
478
|
tags: string[];
|
|
355
479
|
/** The baseline environment, when the eval uses `compare`. */
|
|
356
480
|
compare?: EvalCompareMeta;
|
|
@@ -367,8 +491,8 @@ type EvalEntry = {
|
|
|
367
491
|
cells: CellRecord[];
|
|
368
492
|
attempts: AttemptRecord[];
|
|
369
493
|
/**
|
|
370
|
-
* The
|
|
371
|
-
* unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
494
|
+
* The eval's pairwise questions (those with `across`), answered after its attempts by votes,
|
|
495
|
+
* one per question; unset without any. They annotate: `outcome` and `passRate` never depend on them. Each vote
|
|
372
496
|
* may include a short `reason` (capped). `reeval analyze` includes each answer in the digest
|
|
373
497
|
* (rates by cut, votes, ties, a failed call, a sample of reasons). Each answer's `input` is
|
|
374
498
|
* `<eval-slug>/questions/<n>.json`: every call its model was sent, per pair and order, secrets
|
|
@@ -512,7 +636,7 @@ type Lift = {
|
|
|
512
636
|
};
|
|
513
637
|
/** Each input of a cell fingerprint, hashed on its own, so a change can be named. */
|
|
514
638
|
type FingerprintParts = {
|
|
515
|
-
/** Over `EvalEntry.sources`. */
|
|
639
|
+
/** Over `EvalEntry.source` and `EvalEntry.revision`; on records from before them, over `EvalEntry.sources`. */
|
|
516
640
|
sources: string;
|
|
517
641
|
/** Over the harness descriptor, less concurrency caps. */
|
|
518
642
|
harness: string;
|
|
@@ -523,15 +647,30 @@ type FingerprintParts = {
|
|
|
523
647
|
judge: string;
|
|
524
648
|
/** Over the attempt's context files. */
|
|
525
649
|
context: string;
|
|
650
|
+
/** The gateway its model calls went through; unset on `direct`, each provider's own key. */
|
|
651
|
+
gateway?: string;
|
|
652
|
+
/**
|
|
653
|
+
* The cell's gate (`none`, `passRate 0.5`); unset on the default `all`. Compare reads a change
|
|
654
|
+
* in it as a policy change, not a reason the runs can't be compared.
|
|
655
|
+
*/
|
|
656
|
+
gate?: string;
|
|
526
657
|
};
|
|
527
658
|
/** One harness and environment of an eval, and how its attempts went. */
|
|
528
659
|
type CellRecord = Cell & {
|
|
529
660
|
/** `sha256` of `fingerprintParts`; results are comparable only when it matches. */
|
|
530
661
|
fingerprint: string;
|
|
531
662
|
fingerprintParts: FingerprintParts;
|
|
532
|
-
/**
|
|
663
|
+
/**
|
|
664
|
+
* The environment's digest; null for none. With `environment`, the key into
|
|
665
|
+
* `RunRecord.environments`: environments with the same contents share a digest.
|
|
666
|
+
*/
|
|
533
667
|
environmentDigest: string | null;
|
|
534
668
|
harnessKind: string;
|
|
669
|
+
/**
|
|
670
|
+
* The gateway in the config's `gateway` list its harness resolved to (`direct` is each
|
|
671
|
+
* provider's own key); unset when no config, profile or pin named one.
|
|
672
|
+
*/
|
|
673
|
+
gateway?: string;
|
|
535
674
|
/** The configured harness model, when it has one. */
|
|
536
675
|
model: string | null;
|
|
537
676
|
/**
|
|
@@ -668,6 +807,7 @@ type RunRecord = {
|
|
|
668
807
|
config: {
|
|
669
808
|
sandbox: string;
|
|
670
809
|
environment: string | null;
|
|
810
|
+
/** The config's `runs`; 1 when it sets none, though a compare's cells then run 4 each. */
|
|
671
811
|
runs: number;
|
|
672
812
|
judgeModel: string | null;
|
|
673
813
|
threshold: number;
|
|
@@ -794,8 +934,15 @@ type HistoryCell = Pick<CellRecord, 'key' | 'role' | 'variant' | 'harness' | 'mo
|
|
|
794
934
|
};
|
|
795
935
|
/** Median tool calls, agent time and tokens over the counted attempts; null when none counted. */
|
|
796
936
|
effort: CellEffort | null;
|
|
797
|
-
/** Of `fail`, the attempts the run's interrupt
|
|
798
|
-
|
|
937
|
+
/** Of `fail`, the attempts the run's interrupt canceled (`error.kind: 'aborted'`); unset when none. */
|
|
938
|
+
canceled?: number;
|
|
939
|
+
};
|
|
940
|
+
/** A family's pooled lift as the history listing keeps it: enough to compare its gate. */
|
|
941
|
+
type FamilyLiftSummary = Pick<FamilyLift, 'variant' | 'vs' | 'points' | 'outcome'> & {
|
|
942
|
+
lo: number;
|
|
943
|
+
hi: number;
|
|
944
|
+
/** Unset when the compare has no `minLift`, so the lift can't fail. */
|
|
945
|
+
minLift?: number;
|
|
799
946
|
};
|
|
800
947
|
/**
|
|
801
948
|
* One line of the derived history listing: cached locally by the CLI, and the run header in the
|
|
@@ -830,599 +977,24 @@ type HistoryEntry = {
|
|
|
830
977
|
slug: string;
|
|
831
978
|
fingerprint: string;
|
|
832
979
|
sources: EvalEntry['sources'];
|
|
980
|
+
source?: EvalEntry['source'];
|
|
981
|
+
revision?: EvalEntry['revision'];
|
|
833
982
|
outcome: Outcome;
|
|
834
983
|
passRate: number | null;
|
|
835
984
|
cells: HistoryCell[];
|
|
836
985
|
}>;
|
|
837
986
|
/** The run's pooled families; unset when none ran. */
|
|
838
|
-
families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'
|
|
987
|
+
families?: Array<Pick<FamilyRecord, 'id' | 'name' | 'cases' | 'ran' | 'outcome'> & {
|
|
988
|
+
/** The eval ids of its cases in this run. */
|
|
989
|
+
caseIds: string[];
|
|
990
|
+
lifts: FamilyLiftSummary[];
|
|
991
|
+
}>;
|
|
839
992
|
/** The run's errors outside any attempt; unset when there were none. */
|
|
840
993
|
errors?: string[];
|
|
841
994
|
};
|
|
842
995
|
//#endregion
|
|
843
|
-
//#region src/view/suite.d.ts
|
|
844
|
-
/**
|
|
845
|
-
* A variant against its baseline: `better` or `worse` when the stored 95% interval on the
|
|
846
|
-
* difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
|
|
847
|
-
* including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
|
|
848
|
-
*/
|
|
849
|
-
type Effect = 'better' | 'worse' | 'same' | 'unclear';
|
|
850
|
-
/**
|
|
851
|
-
* What an eval's cells compare: `change` when it has variants (or several environments),
|
|
852
|
-
* `harness` when it has several harnesses, `both`, or `none`.
|
|
853
|
-
*/
|
|
854
|
-
type Across = 'none' | 'change' | 'harness' | 'both';
|
|
855
|
-
/**
|
|
856
|
-
* One finished run's facts. Every node has a `path`, a stable id to cite it by, and `paths`
|
|
857
|
-
* lists them all in the order the nodes appear.
|
|
858
|
-
*
|
|
859
|
-
* A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
|
|
860
|
-
*
|
|
861
|
-
* - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
|
|
862
|
-
* (`evals/a.eval.ts › fixes $id`);
|
|
863
|
-
* - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
|
|
864
|
-
* - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
|
|
865
|
-
* - `<cell>/checks/<label>`: a check, by its assertion label;
|
|
866
|
-
* - `<cell>/effort`: the cell's effort, only when it was tested;
|
|
867
|
-
* - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
|
|
868
|
-
*
|
|
869
|
-
* A case segment never has a raw `+` and a cell segment always has two, so the two can't be
|
|
870
|
-
* confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
|
|
871
|
-
* fact in every run that has it.
|
|
872
|
-
*/
|
|
873
|
-
type SuiteResult = {
|
|
874
|
-
runId: string;
|
|
875
|
-
status: RunRecord['status'];
|
|
876
|
-
/**
|
|
877
|
-
* The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
|
|
878
|
-
* an attempt, and `skipped` on an interrupted run.
|
|
879
|
-
*/
|
|
880
|
-
outcome: Outcome;
|
|
881
|
-
/**
|
|
882
|
-
* Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
|
|
883
|
-
* awaited, a worker that died. Any one fails the run.
|
|
884
|
-
*/
|
|
885
|
-
errors: string[];
|
|
886
|
-
/**
|
|
887
|
-
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
888
|
-
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`.
|
|
889
|
-
*/
|
|
890
|
-
tally: SuiteTally;
|
|
891
|
-
/** Sorted by path. */
|
|
892
|
-
files: ResultFile[];
|
|
893
|
-
/** Every node's `path`, in the order they appear. */
|
|
894
|
-
paths: string[];
|
|
895
|
-
};
|
|
896
|
-
type SuiteTally = {
|
|
897
|
-
errors: number;
|
|
898
|
-
pass: number;
|
|
899
|
-
fail: number;
|
|
900
|
-
infraError: number;
|
|
901
|
-
cancelled: number;
|
|
902
|
-
skipped: number;
|
|
903
|
-
};
|
|
904
|
-
/**
|
|
905
|
-
* An eval's or a family case's outcome: the record's, except `cancelled` where its only failed
|
|
906
|
-
* gated attempts are ones the run's interrupt cancelled, since the agent didn't fail those.
|
|
907
|
-
*/
|
|
908
|
-
type ResultOutcome = Outcome | 'cancelled';
|
|
909
|
-
type ResultFile = {
|
|
910
|
-
path: string;
|
|
911
|
-
evals: EvalResult[];
|
|
912
|
-
};
|
|
913
|
-
type EvalResult = {
|
|
914
|
-
path: string;
|
|
915
|
-
/** The eval's id, or the family's. */
|
|
916
|
-
id: string;
|
|
917
|
-
group: string[];
|
|
918
|
-
/** The eval's name; on a family, its template. */
|
|
919
|
-
name: string;
|
|
920
|
-
/** A family's name template (`fixes $id`); null on a single eval. */
|
|
921
|
-
template: string | null;
|
|
922
|
-
/** Source order in the file. */
|
|
923
|
-
order: number;
|
|
924
|
-
across: Across;
|
|
925
|
-
/**
|
|
926
|
-
* A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
|
|
927
|
-
* unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
|
|
928
|
-
*/
|
|
929
|
-
family: boolean;
|
|
930
|
-
/** A placeholder (`evaluate.todo`) that never ran. */
|
|
931
|
-
todo: boolean;
|
|
932
|
-
/** On a family: `fail` when a case failed or its pooled gate did. */
|
|
933
|
-
outcome: ResultOutcome;
|
|
934
|
-
cells: CellResult[] | null;
|
|
935
|
-
cases: CaseResult[] | null;
|
|
936
|
-
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
937
|
-
pooled: PooledArm[] | null;
|
|
938
|
-
/**
|
|
939
|
-
* Its compare questions' answers. On a family, each question asked of any case, in first-asked
|
|
940
|
-
* order, with its votes and wins summed over the cases that answered it.
|
|
941
|
-
*/
|
|
942
|
-
questions: QuestionResult[];
|
|
943
|
-
};
|
|
944
|
-
/** A compare question: each side's wins over the pairs the model decided. Never gates. */
|
|
945
|
-
type QuestionResult = {
|
|
946
|
-
question: string;
|
|
947
|
-
/** Pairs voted, each counted once per order it was asked in. */
|
|
948
|
-
votes: number;
|
|
949
|
-
rates: Array<{
|
|
950
|
-
cut: 'within' | 'across';
|
|
951
|
-
name: string;
|
|
952
|
-
wins: number;
|
|
953
|
-
n: number;
|
|
954
|
-
}>;
|
|
955
|
-
/** Why some or all of its votes are missing: the model call failed. */
|
|
956
|
-
error: string | null;
|
|
957
|
-
};
|
|
958
|
-
type CaseResult = {
|
|
959
|
-
path: string;
|
|
960
|
-
/** The case's own eval id. */
|
|
961
|
-
id: string;
|
|
962
|
-
/** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
|
|
963
|
-
label: string;
|
|
964
|
-
name: string;
|
|
965
|
-
/** Its place in the case list. */
|
|
966
|
-
order: number;
|
|
967
|
-
outcome: ResultOutcome;
|
|
968
|
-
cells: CellResult[];
|
|
969
|
-
};
|
|
970
|
-
type CellResult = {
|
|
971
|
-
path: string;
|
|
972
|
-
key: string;
|
|
973
|
-
harness: string;
|
|
974
|
-
/** `-` when no environment is set. */
|
|
975
|
-
environment: string;
|
|
976
|
-
sandbox: string;
|
|
977
|
-
/** The variant's name; null on a control. */
|
|
978
|
-
variant: string | null;
|
|
979
|
-
/** The control of an eval that has variants: the arm the variants are measured against. */
|
|
980
|
-
baseline: boolean;
|
|
981
|
-
/** On a variant, its baseline cell's key; else null. */
|
|
982
|
-
vs: string | null;
|
|
983
|
-
/** The cell's place in the eval's cells. */
|
|
984
|
-
order: number;
|
|
985
|
-
/** Its failures fail the eval (its gate isn't `'none'`). */
|
|
986
|
-
gated: boolean;
|
|
987
|
-
outcome: Outcome;
|
|
988
|
-
/** Every attempt: `passed + failed + infraError + skipped`. */
|
|
989
|
-
attempts: number;
|
|
990
|
-
passed: number;
|
|
991
|
-
failed: number;
|
|
992
|
-
/** Infrastructure errors, which are neither passes nor fails. */
|
|
993
|
-
infraError: number;
|
|
994
|
-
skipped: number;
|
|
995
|
-
fingerprint: string;
|
|
996
|
-
/** The skill sources and files the cell's environment placed, as written. */
|
|
997
|
-
context: string[];
|
|
998
|
-
/**
|
|
999
|
-
* Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
|
|
1000
|
-
* attempt (every one an infra error or skipped) lists each as `not run`.
|
|
1001
|
-
*/
|
|
1002
|
-
checks: CheckResult[];
|
|
1003
|
-
/** On a variant, against its baseline; null on a control. */
|
|
1004
|
-
effect: Effect | null;
|
|
1005
|
-
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1006
|
-
interval: Interval | null;
|
|
1007
|
-
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1008
|
-
minLift: number | null;
|
|
1009
|
-
/** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
|
|
1010
|
-
effort: EffortCounters | null;
|
|
1011
|
-
/** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
|
|
1012
|
-
effortDiffers: boolean;
|
|
1013
|
-
/** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
|
|
1014
|
-
errors: CellError[];
|
|
1015
|
-
/** Web searches and fetches per counted attempt; null where the harness can't report them. */
|
|
1016
|
-
searches: Array<number | null>;
|
|
1017
|
-
fetches: Array<number | null>;
|
|
1018
|
-
};
|
|
1019
|
-
/** A cell's attempts the run's interrupt cancelled: recorded as fails, but not the agent's. */
|
|
1020
|
-
declare const cancelledOf: (cell: CellResult) => number;
|
|
1021
|
-
/** A cell's counted tries: its passes and fails, less the ones the interrupt cancelled. */
|
|
1022
|
-
declare const countedOf: (cell: CellResult) => number;
|
|
1023
|
-
type CheckResult = {
|
|
1024
|
-
path: string;
|
|
1025
|
-
label: string;
|
|
1026
|
-
/** First-seen order across the eval's attempts. */
|
|
1027
|
-
order: number;
|
|
1028
|
-
outcome: CheckOutcome;
|
|
1029
|
-
/**
|
|
1030
|
-
* On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
|
|
1031
|
-
* null on a control or without one.
|
|
1032
|
-
*/
|
|
1033
|
-
effect: Effect | null;
|
|
1034
|
-
/** Set when the check is a judge verdict. */
|
|
1035
|
-
judge: JudgeResult | null;
|
|
1036
|
-
};
|
|
1037
|
-
/**
|
|
1038
|
-
* `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
|
|
1039
|
-
* none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
|
|
1040
|
-
* reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
|
|
1041
|
-
* first failure's message.
|
|
1042
|
-
*/
|
|
1043
|
-
type CheckOutcome = {
|
|
1044
|
-
status: 'not run';
|
|
1045
|
-
} | {
|
|
1046
|
-
status: 'ran';
|
|
1047
|
-
passed: number;
|
|
1048
|
-
failed: number;
|
|
1049
|
-
errored: number;
|
|
1050
|
-
notRun: number;
|
|
1051
|
-
assertion: string | null;
|
|
1052
|
-
};
|
|
1053
|
-
/**
|
|
1054
|
-
* One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
|
|
1055
|
-
* an error), its message's first line, and how often.
|
|
1056
|
-
*/
|
|
1057
|
-
type CellError = {
|
|
1058
|
-
kind: ErrorKind | 'turn';
|
|
1059
|
-
message: string;
|
|
1060
|
-
attempts: number;
|
|
1061
|
-
};
|
|
1062
|
-
type JudgeResult = {
|
|
1063
|
-
question: string;
|
|
1064
|
-
threshold: number;
|
|
1065
|
-
/** One per counted attempt that judged it. */
|
|
1066
|
-
scores: number[];
|
|
1067
|
-
};
|
|
1068
|
-
type Interval = {
|
|
1069
|
-
low: number;
|
|
1070
|
-
high: number;
|
|
1071
|
-
};
|
|
1072
|
-
/** A null counter is one the harness didn't record; a recorded zero stays zero. */
|
|
1073
|
-
type EffortCounters = {
|
|
1074
|
-
tools: number | null;
|
|
1075
|
-
agentMs: number | null;
|
|
1076
|
-
tokens: number | null;
|
|
1077
|
-
};
|
|
1078
|
-
type PooledArm = {
|
|
1079
|
-
path: string;
|
|
1080
|
-
/** The variant cell's key, the same in every case. */
|
|
1081
|
-
key: string;
|
|
1082
|
-
vs: string;
|
|
1083
|
-
effect: Effect;
|
|
1084
|
-
/** Cases with counted attempts on both sides. */
|
|
1085
|
-
pairs: number;
|
|
1086
|
-
interval: Interval;
|
|
1087
|
-
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
1088
|
-
minLift: number | null;
|
|
1089
|
-
outcome: 'pass' | 'fail';
|
|
1090
|
-
};
|
|
1091
|
-
type Side = {
|
|
1092
|
-
pass: number;
|
|
1093
|
-
counted: number;
|
|
1094
|
-
};
|
|
1095
|
-
/**
|
|
1096
|
-
* The effect of `variant` against `control`, from the interval on their difference. A side
|
|
1097
|
-
* with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
|
|
1098
|
-
*/
|
|
1099
|
-
declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
|
|
1100
|
-
/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
|
|
1101
|
-
declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
|
|
1102
|
-
/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
|
|
1103
|
-
declare const factSegment: (s: string) => string;
|
|
1104
|
-
declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
|
|
1105
|
-
/**
|
|
1106
|
-
* An error's one line: its first line with any letter or digit in it, joined with the next one
|
|
1107
|
-
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1108
|
-
*/
|
|
1109
|
-
declare function messageLine(text: string): string;
|
|
1110
|
-
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1111
|
-
declare function suiteResult(record: RunRecord): SuiteResult;
|
|
1112
|
-
//#endregion
|
|
1113
|
-
//#region src/view/insights.d.ts
|
|
1114
|
-
/** `<history>/<run>/insights.json`, beside the record. */
|
|
1115
|
-
declare const INSIGHTS_FILE = "insights.json";
|
|
1116
|
-
/** `RunInsights.model` when no analysis model wrote the file: it holds deterministic remarks only. */
|
|
1117
|
-
declare const NO_MODEL = "none";
|
|
1118
|
-
/** Whether an analysis model wrote these insights, rather than reeval alone for its remarks. */
|
|
1119
|
-
declare const fromModel: (insights: RunInsights) => boolean;
|
|
1120
|
-
/** What one attempt tests. Only an analysis model can tell; without one every eval is `other`. */
|
|
1121
|
-
type Job = 'choose' | 'finish' | 'other';
|
|
1122
|
-
/**
|
|
1123
|
-
* A run's insights, `<history>/<run>/insights.json`. Written create-only; the record is untouched.
|
|
1124
|
-
* They're for the facts they were written from: a `factsHash` that isn't the run's marks them stale.
|
|
1125
|
-
*/
|
|
1126
|
-
type RunInsights = {
|
|
1127
|
-
kind: 'reeval-insights';
|
|
1128
|
-
schemaVersion: 1;
|
|
1129
|
-
runId: string;
|
|
1130
|
-
/**
|
|
1131
|
-
* The analysis model, as configured (`google/gemini-3.8-flash`, or `fake`); `NO_MODEL` when
|
|
1132
|
-
* reeval wrote the file without one, to hold the run's deterministic remarks only.
|
|
1133
|
-
*/
|
|
1134
|
-
model: string;
|
|
1135
|
-
/** The prompts' version: insights from another version were asked differently. */
|
|
1136
|
-
promptVersion: string;
|
|
1137
|
-
/** sha256 of the run's `SuiteResult` as canonical JSON: the facts the model saw. */
|
|
1138
|
-
factsHash: string;
|
|
1139
|
-
createdAt: string;
|
|
1140
|
-
/**
|
|
1141
|
-
* Over every call; `outputTokens` includes `reasoningTokens`, the model's thinking, when it
|
|
1142
|
-
* reports them. `costUsd` is null when reeval doesn't know the model's price.
|
|
1143
|
-
*/
|
|
1144
|
-
usage: {
|
|
1145
|
-
inputTokens: number;
|
|
1146
|
-
outputTokens: number;
|
|
1147
|
-
reasoningTokens?: number;
|
|
1148
|
-
costUsd: number | null;
|
|
1149
|
-
};
|
|
1150
|
-
/**
|
|
1151
|
-
* `analysis.maxTokens`, and the evals (`SuiteResult` paths) it ran out before: they print
|
|
1152
|
-
* without insights. Absent when no budget applied.
|
|
1153
|
-
*/
|
|
1154
|
-
budget?: {
|
|
1155
|
-
maxTokens: number;
|
|
1156
|
-
skipped: string[];
|
|
1157
|
-
};
|
|
1158
|
-
/**
|
|
1159
|
-
* Evals whose calls failed twice (an error, a reply cut off at the output cap, or a call past its
|
|
1160
|
-
* timeout), with the last error: they print without insights. Absent when none failed.
|
|
1161
|
-
*/
|
|
1162
|
-
failed?: Array<{
|
|
1163
|
-
evalId: string;
|
|
1164
|
-
error: string;
|
|
1165
|
-
}>;
|
|
1166
|
-
/**
|
|
1167
|
-
* Set when the run was stopped (Ctrl-C, or SIGTERM) before every eval's insights had finished:
|
|
1168
|
-
* the evals that hadn't, which print without insights. Absent otherwise, even when the stop came
|
|
1169
|
-
* during the headline call, which leaves `headline` null.
|
|
1170
|
-
*/
|
|
1171
|
-
stopped?: {
|
|
1172
|
-
signal: 'SIGINT' | 'SIGTERM';
|
|
1173
|
-
skipped: string[];
|
|
1174
|
-
};
|
|
1175
|
-
/** One sentence for the whole run, citing facts; null when the model's was dropped. */
|
|
1176
|
-
headline: Sentence | null;
|
|
1177
|
-
/** One per eval (a family is one), in the order the output prints them. */
|
|
1178
|
-
evals: EvalInsight[];
|
|
1179
|
-
/**
|
|
1180
|
-
* Remarks about the evals' own quality, ranked: every one this run raised, including those an
|
|
1181
|
-
* earlier run printed first (`firstSeenRun`), which don't print again.
|
|
1182
|
-
*/
|
|
1183
|
-
remarks: Remark[];
|
|
1184
|
-
/**
|
|
1185
|
-
* What the checks dropped, and how many sentences were asked for again, so the output can say
|
|
1186
|
-
* so. `drops` is each dropped sentence: its slot (`<eval> › answer`, `› row <path>`, `headline`),
|
|
1187
|
-
* its text and why.
|
|
1188
|
-
*/
|
|
1189
|
-
validation: {
|
|
1190
|
-
dropped: number;
|
|
1191
|
-
regenerated: number;
|
|
1192
|
-
drops?: Array<{
|
|
1193
|
-
slot: string;
|
|
1194
|
-
text: string;
|
|
1195
|
-
reason: string;
|
|
1196
|
-
}>;
|
|
1197
|
-
};
|
|
1198
|
-
/** Set when insights failed; the facts still print. */
|
|
1199
|
-
error?: string;
|
|
1200
|
-
};
|
|
1201
|
-
type EvalInsight = {
|
|
1202
|
-
/** The eval's `SuiteResult` path; for a family, the family's. */
|
|
1203
|
-
evalId: string;
|
|
1204
|
-
intent: {
|
|
1205
|
-
/** What one attempt is testing. Can't be read from the cell shape. */
|
|
1206
|
-
job: Job;
|
|
1207
|
-
/** What the cells compare. Deterministic, copied from the facts. */
|
|
1208
|
-
across: Across;
|
|
1209
|
-
/** The eval's question in plain words: "Do agents pick Genkit for a production AI feature?" */
|
|
1210
|
-
question: string;
|
|
1211
|
-
/** The tool a choose eval asks about, and the change a measure eval makes. */
|
|
1212
|
-
subject: {
|
|
1213
|
-
tool?: string;
|
|
1214
|
-
change?: string;
|
|
1215
|
-
};
|
|
1216
|
-
/** How the eval name relates to what the eval checks. */
|
|
1217
|
-
name: 'states-intent' | 'no-intent' | 'contradicts';
|
|
1218
|
-
/**
|
|
1219
|
-
* sha256 of the eval's id, sources, first attempt's prompts, judge questions, check labels,
|
|
1220
|
-
* the model and the prompt version: the classification's cache key.
|
|
1221
|
-
*/
|
|
1222
|
-
key: string;
|
|
1223
|
-
};
|
|
1224
|
-
/** The answer to `question`, at most two sentences; null when it was dropped. */
|
|
1225
|
-
answer: Sentence | null;
|
|
1226
|
-
/** Up to three supporting points: what moved, what broke, what the agent did. */
|
|
1227
|
-
findings: Sentence[];
|
|
1228
|
-
/**
|
|
1229
|
-
* At most one sentence per row (a case, an arm or a harness), keyed by the row's fact path.
|
|
1230
|
-
* Only rows the eval's layout opens a slot on are kept.
|
|
1231
|
-
*/
|
|
1232
|
-
rows?: Array<Sentence & {
|
|
1233
|
-
row: string;
|
|
1234
|
-
}>;
|
|
1235
|
-
/**
|
|
1236
|
-
* Per attempt (`<cell path>#<n>`), the tools the reply recommended or used, each found
|
|
1237
|
-
* verbatim in that attempt's final message. Only on `choose` evals.
|
|
1238
|
-
*/
|
|
1239
|
-
namedTools?: Array<{
|
|
1240
|
-
attempt: string;
|
|
1241
|
-
tools: string[];
|
|
1242
|
-
}>;
|
|
1243
|
-
};
|
|
1244
|
-
/**
|
|
1245
|
-
* A plain sentence citing the facts it states: `SuiteResult` paths
|
|
1246
|
-
* (`evals%2Fbilling.eval.ts › bills in pence/fake+-+fake/checks/amount`) or a transcript line
|
|
1247
|
-
* (`transcript:<cell path>#<n>#L<line>`). A sentence citing nothing is dropped.
|
|
1248
|
-
*/
|
|
1249
|
-
type Sentence = {
|
|
1250
|
-
text: string;
|
|
1251
|
-
cites: string[];
|
|
1252
|
-
};
|
|
1253
|
-
/**
|
|
1254
|
-
* Each remark kind's origin and severity: an analysis model raises the `model` kinds, through the
|
|
1255
|
-
* gates (a known kind, a quote verbatim on the cited line); reeval's rules raise the others. The
|
|
1256
|
-
* severity is the kind's, never the model's.
|
|
1257
|
-
*/
|
|
1258
|
-
declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
1259
|
-
origin: Remark['origin'];
|
|
1260
|
-
severity: 'concern' | 'suggestion';
|
|
1261
|
-
}>;
|
|
1262
|
-
/** The kinds an analysis model may raise; a remark of any other kind from it is dropped. */
|
|
1263
|
-
declare const MODEL_REMARK_KINDS: ("check-never-fails" | "compound-judge" | "flaky-check" | "judge-leaks-answer" | "judge-too-loose" | "name-contradicts" | "name-no-intent" | "no-baseline" | "prompt-names-tool" | "slow-mcp-start" | "too-few-tries")[];
|
|
1264
|
-
/** A remark about the eval's quality, not about the agent. Anything that fails a gate is dropped. */
|
|
1265
|
-
type Remark = {
|
|
1266
|
-
/**
|
|
1267
|
-
* sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
|
|
1268
|
-
* per kind (and server) per eval version.
|
|
1269
|
-
*/
|
|
1270
|
-
id: string;
|
|
1271
|
-
evalId: string;
|
|
1272
|
-
/** Closed set; an unknown kind is dropped. There is no general or free-form kind. */
|
|
1273
|
-
kind: 'name-contradicts' | 'name-no-intent' | 'judge-too-loose' | 'judge-leaks-answer' | 'prompt-names-tool' | 'compound-judge' | 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'no-baseline' | 'slow-mcp-start';
|
|
1274
|
-
/** Fixed by kind (`REMARK_KINDS`), never chosen by the model. */
|
|
1275
|
-
severity: 'concern' | 'suggestion';
|
|
1276
|
-
/** One plain sentence: what's wrong. */
|
|
1277
|
-
text: string;
|
|
1278
|
-
/**
|
|
1279
|
-
* Where the problem is: `source:<file>#L<n>` with `quote`, found verbatim on that line of the
|
|
1280
|
-
* file at the run's source hash. Every model remark has one.
|
|
1281
|
-
*/
|
|
1282
|
-
source?: {
|
|
1283
|
-
at: string;
|
|
1284
|
-
quote: string;
|
|
1285
|
-
};
|
|
1286
|
-
/** Facts from this run (or history facts) showing it mattered now; a model's, as it cited them. */
|
|
1287
|
-
evidence: string[];
|
|
1288
|
-
/**
|
|
1289
|
-
* An edit at a source location (`source:<file>#L<n>`) in the eval. A model remark's `edit` is
|
|
1290
|
-
* empty when the model gave none, and its `at` is the quoted line when it named no line.
|
|
1291
|
-
*/
|
|
1292
|
-
fix: {
|
|
1293
|
-
at: string;
|
|
1294
|
-
edit: string;
|
|
1295
|
-
};
|
|
1296
|
-
origin: 'model' | 'deterministic';
|
|
1297
|
-
/**
|
|
1298
|
-
* The first run that printed this `id`, or this run when none has: it prints only in that run,
|
|
1299
|
-
* except `slow-mcp-start`, which is about the environment, not the eval's source, and prints in
|
|
1300
|
-
* every run it happens.
|
|
1301
|
-
*/
|
|
1302
|
-
firstSeenRun: string;
|
|
1303
|
-
/**
|
|
1304
|
-
* Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
|
|
1305
|
-
* when the cap or a run without `-v` left it out, so a later run prints it. Missing: printed in
|
|
1306
|
-
* `firstSeenRun` only.
|
|
1307
|
-
*/
|
|
1308
|
-
printed?: boolean;
|
|
1309
|
-
/** Set on later runs once the cited span changed: the remark was acted on. */
|
|
1310
|
-
actedOn?: {
|
|
1311
|
-
run: string;
|
|
1312
|
-
};
|
|
1313
|
-
};
|
|
1314
|
-
//#endregion
|
|
1315
|
-
//#region src/insights/input.d.ts
|
|
1316
|
-
/** One fact the model may cite, by its path. */
|
|
1317
|
-
type InsightFact = {
|
|
1318
|
-
path: string;
|
|
1319
|
-
kind: string;
|
|
1320
|
-
} & Record<string, unknown>;
|
|
1321
|
-
type InsightAttempt = {
|
|
1322
|
-
/** `<cell path>#<n>`. */
|
|
1323
|
-
id: string;
|
|
1324
|
-
cell: string;
|
|
1325
|
-
outcome: string;
|
|
1326
|
-
/** The agent's final message, clipped. */
|
|
1327
|
-
lastText: string;
|
|
1328
|
-
/** A failed attempt's transcript lines, `L<n> …`, each citable as `transcript:<id>#L<n>`. */
|
|
1329
|
-
lines: string[];
|
|
1330
|
-
};
|
|
1331
|
-
/** Where an insight sentence can go, as the eval's layout opens them. */
|
|
1332
|
-
type InsightSlots = {
|
|
1333
|
-
/** Row paths that take one sentence each. */
|
|
1334
|
-
rows: string[];
|
|
1335
|
-
findings: boolean;
|
|
1336
|
-
namedTools: boolean;
|
|
1337
|
-
};
|
|
1338
|
-
/** One eval as the classifier and the insight model see it. */
|
|
1339
|
-
type InsightInput = {
|
|
1340
|
-
call: 'classify' | 'insights';
|
|
1341
|
-
eval: {
|
|
1342
|
-
path: string;
|
|
1343
|
-
/** The eval's name, or a family's template: the author's statement of what it asks. */
|
|
1344
|
-
name: string;
|
|
1345
|
-
group: string[];
|
|
1346
|
-
file: string;
|
|
1347
|
-
across: Across;
|
|
1348
|
-
family: boolean;
|
|
1349
|
-
verdict: string;
|
|
1350
|
-
};
|
|
1351
|
-
/** The first attempt's prompts, in order. */
|
|
1352
|
-
prompts: string[];
|
|
1353
|
-
judgeQuestions: string[];
|
|
1354
|
-
checks: string[];
|
|
1355
|
-
/** Set on the insights call: the classification. */
|
|
1356
|
-
intent?: {
|
|
1357
|
-
job: Job;
|
|
1358
|
-
question: string;
|
|
1359
|
-
subject: {
|
|
1360
|
-
tool?: string;
|
|
1361
|
-
change?: string;
|
|
1362
|
-
};
|
|
1363
|
-
};
|
|
1364
|
-
/** Set on the insights call. */
|
|
1365
|
-
slots?: InsightSlots;
|
|
1366
|
-
facts: InsightFact[];
|
|
1367
|
-
attempts: InsightAttempt[];
|
|
1368
|
-
/**
|
|
1369
|
-
* Set on the insights call: the eval's declaration in its file, each line `L<n> …` as the file
|
|
1370
|
-
* numbers it, which a remark's `source` and `fix` cite.
|
|
1371
|
-
*/
|
|
1372
|
-
source?: {
|
|
1373
|
-
file: string;
|
|
1374
|
-
lines: string[];
|
|
1375
|
-
};
|
|
1376
|
-
/** What was left out to fit the eval's size cap (`trim.ts`), when anything was. */
|
|
1377
|
-
trimmed?: string[];
|
|
1378
|
-
/** On a regeneration: why the earlier sentences were refused. */
|
|
1379
|
-
retry?: string[];
|
|
1380
|
-
};
|
|
1381
|
-
/** The headline call's input: each eval's verdict and answer, and the run's tally with its total. */
|
|
1382
|
-
type HeadlineInput = {
|
|
1383
|
-
call: 'headline';
|
|
1384
|
-
/**
|
|
1385
|
-
* Counts results, not evals: each case of a family counts one, as the tally line does, and
|
|
1386
|
-
* `total` is their sum, so no count is the model's arithmetic; `errors` (outside any eval) aren't
|
|
1387
|
-
* results and stay out of it. The eval count is `evals.length`.
|
|
1388
|
-
*/
|
|
1389
|
-
tally: SuiteResult['tally'] & {
|
|
1390
|
-
total: number;
|
|
1391
|
-
unit: 'results';
|
|
1392
|
-
};
|
|
1393
|
-
evals: Array<{
|
|
1394
|
-
path: string;
|
|
1395
|
-
name: string;
|
|
1396
|
-
verdict: string;
|
|
1397
|
-
answer: string | null;
|
|
1398
|
-
}>;
|
|
1399
|
-
retry?: string[];
|
|
1400
|
-
};
|
|
1401
|
-
/** A fact as the model sees it: its short id (`F1`), which is what it cites. */
|
|
1402
|
-
type ModelFact = InsightFact & {
|
|
1403
|
-
id: string;
|
|
1404
|
-
};
|
|
1405
|
-
/** An attempt as the model sees it: `A1`, its cell's fact id, and its lines citable as `A1#L<n>`. */
|
|
1406
|
-
type ModelAttempt = Omit<InsightAttempt, 'id' | 'cell'> & {
|
|
1407
|
-
id: string;
|
|
1408
|
-
cell: string;
|
|
1409
|
-
};
|
|
1410
|
-
/** One eval's call as the model sees it: facts, attempts and row slots by short id. */
|
|
1411
|
-
type ModelInsightInput = Omit<InsightInput, 'facts' | 'attempts' | 'slots'> & {
|
|
1412
|
-
facts: ModelFact[];
|
|
1413
|
-
attempts: ModelAttempt[];
|
|
1414
|
-
slots?: Omit<InsightSlots, 'rows'> & {
|
|
1415
|
-
rows: string[];
|
|
1416
|
-
};
|
|
1417
|
-
};
|
|
1418
|
-
/** The headline call as the model sees it: each eval by its short id. */
|
|
1419
|
-
type ModelHeadlineInput = Omit<HeadlineInput, 'evals'> & {
|
|
1420
|
-
evals: Array<HeadlineInput['evals'][number] & {
|
|
1421
|
-
id: string;
|
|
1422
|
-
}>;
|
|
1423
|
-
};
|
|
1424
|
-
//#endregion
|
|
1425
996
|
//#region src/types.d.ts
|
|
997
|
+
/** More than zero and at most 596h: `90s`, `15m`, `1h`, or a number of milliseconds. */
|
|
1426
998
|
type Duration = number | `${number}${'ms' | 's' | 'm' | 'h'}`;
|
|
1427
999
|
/** When an event started: wall clock, and ms since its turn started. */
|
|
1428
1000
|
type EventTime = {
|
|
@@ -1433,13 +1005,17 @@ type FileEditEvent = {
|
|
|
1433
1005
|
type: 'file-edit';
|
|
1434
1006
|
toolCallId: string;
|
|
1435
1007
|
path: string;
|
|
1436
|
-
|
|
1008
|
+
/**
|
|
1009
|
+
* `write` is a whole-file write whose result doesn't say whether the file existed before
|
|
1010
|
+
* (Antigravity's, fx's, Goose's, Grok Build's and pi's write tools, for example).
|
|
1011
|
+
*/
|
|
1012
|
+
kind: 'create' | 'modify' | 'write' | 'delete';
|
|
1437
1013
|
/** Line counts, or null when the harness didn't send the content (e.g. deletes). */
|
|
1438
1014
|
added: number | null;
|
|
1439
1015
|
removed: number | null;
|
|
1440
1016
|
};
|
|
1441
1017
|
/**
|
|
1442
|
-
*
|
|
1018
|
+
* Normalized agent events. Each tool call is one event, merged with its result; `durationMs`
|
|
1443
1019
|
* is wall time until the result arrived (Cursor's own `completed_at_ms` can be earlier). Null
|
|
1444
1020
|
* when the harness never reported the end. Long shell or edit calls set `stalled: true`.
|
|
1445
1021
|
*/
|
|
@@ -1496,6 +1072,18 @@ type ExecResult = {
|
|
|
1496
1072
|
stdout: string;
|
|
1497
1073
|
stderr: string;
|
|
1498
1074
|
};
|
|
1075
|
+
/** How `c.exec` reruns a command that exits non-zero, for a check that can fail on a transient. */
|
|
1076
|
+
type ExecRetry = {
|
|
1077
|
+
/** Extra tries after a non-zero exit, a whole number from 0. Default 0. */
|
|
1078
|
+
retries?: number;
|
|
1079
|
+
/** Milliseconds to wait before each extra try. Default 1000. */
|
|
1080
|
+
interval?: number;
|
|
1081
|
+
};
|
|
1082
|
+
/** One run of a `c.exec` command that had `retries`. */
|
|
1083
|
+
type ExecTry = {
|
|
1084
|
+
exitCode: number;
|
|
1085
|
+
durationMs: number;
|
|
1086
|
+
};
|
|
1499
1087
|
/** Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`. */
|
|
1500
1088
|
type CaptureWindow = {
|
|
1501
1089
|
/** In CSS pixels. */
|
|
@@ -1569,8 +1157,8 @@ type CaptureOptions = {
|
|
|
1569
1157
|
/** `false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws. */
|
|
1570
1158
|
browser?: boolean;
|
|
1571
1159
|
/**
|
|
1572
|
-
* Vision model for `instructions`. Cascades like `question`. Default is
|
|
1573
|
-
*
|
|
1160
|
+
* Vision model for `instructions`. Cascades like `question`. Default is `analysis.model`, else
|
|
1161
|
+
* the attempt's judge model, whichever is first a language model; never Jev.
|
|
1574
1162
|
*/
|
|
1575
1163
|
model?: string;
|
|
1576
1164
|
};
|
|
@@ -1604,6 +1192,7 @@ type Capture = {
|
|
|
1604
1192
|
* default `name` is derived from the path. `wait` and `instructions` are call-only.
|
|
1605
1193
|
*/
|
|
1606
1194
|
type CaptureCall = {
|
|
1195
|
+
/** The `artifacts/<name>.png` file name: 1 to 100 of `a-z`, `0-9`, `_` and `-`, starting with a letter or digit. Unique within the attempt. */
|
|
1607
1196
|
name?: string;
|
|
1608
1197
|
path?: string;
|
|
1609
1198
|
viewport?: Viewport;
|
|
@@ -1665,7 +1254,7 @@ type TurnData = {
|
|
|
1665
1254
|
};
|
|
1666
1255
|
type Turn = TurnData & {
|
|
1667
1256
|
/** Runs in the eval's sandbox. Same as `c.exec`. */
|
|
1668
|
-
exec(command: string): Promise<ExecResult>;
|
|
1257
|
+
exec(command: string, options?: ExecRetry): Promise<ExecResult>;
|
|
1669
1258
|
readFile(path: string): Promise<string | null>;
|
|
1670
1259
|
};
|
|
1671
1260
|
/** Any JSON value. */
|
|
@@ -1730,6 +1319,11 @@ type JudgeOptions = {
|
|
|
1730
1319
|
* attempt's state. Merged key by key down the cascade.
|
|
1731
1320
|
*/
|
|
1732
1321
|
state?: JudgeStateOptions;
|
|
1322
|
+
/**
|
|
1323
|
+
* The gateway the judge's calls go through, by name or by its helper. It skips the config's
|
|
1324
|
+
* `gateway` fallback: the plan refuses a gateway that isn't listed or can't serve the model.
|
|
1325
|
+
*/
|
|
1326
|
+
gateway?: GatewayName | GatewayDescriptor;
|
|
1733
1327
|
};
|
|
1734
1328
|
/**
|
|
1735
1329
|
* What the judge is given: each key is a key of the state in `judge-input.json`. An unset key
|
|
@@ -1798,6 +1392,12 @@ type Verdict = {
|
|
|
1798
1392
|
model: string;
|
|
1799
1393
|
/** The model id the provider says served the call, when it reports one. */
|
|
1800
1394
|
resolvedModel?: string;
|
|
1395
|
+
/** The gateway from the config's `gateway` list the call went through; unset without one. */
|
|
1396
|
+
gateway?: GatewayName;
|
|
1397
|
+
/** The upstream provider the gateway says served the call (OpenRouter's); unset when it names none. */
|
|
1398
|
+
upstream?: string;
|
|
1399
|
+
/** What the gateway says the call cost, in US dollars (OpenRouter's `usage.cost`); unset when it names none. */
|
|
1400
|
+
costUsd?: number;
|
|
1801
1401
|
/** Captures this call sent the judge as file parts, when any. */
|
|
1802
1402
|
artifacts?: JudgeArtifactRef[];
|
|
1803
1403
|
};
|
|
@@ -1805,7 +1405,7 @@ type Verdict = {
|
|
|
1805
1405
|
* What `c.judge()` returns when the call failed (the provider's error, after any retries): the attempt fails as a judge error once the eval's body has finished,
|
|
1806
1406
|
* so the judges and checks after it still run.
|
|
1807
1407
|
*/
|
|
1808
|
-
type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel'> & {
|
|
1408
|
+
type JudgeFailure = Omit<Verdict, 'verdict' | 'score' | 'resolvedModel' | 'upstream' | 'costUsd'> & {
|
|
1809
1409
|
verdict: 'error';
|
|
1810
1410
|
score: null;
|
|
1811
1411
|
error: string;
|
|
@@ -1822,11 +1422,17 @@ type AttemptContext = {
|
|
|
1822
1422
|
* `options.artifacts` is a list of `c.capture()` handles or names, or `true` for all of them.
|
|
1823
1423
|
*/
|
|
1824
1424
|
judge(question: string, options?: JudgeCallOptions): Promise<Verdict | JudgeFailure>;
|
|
1825
|
-
|
|
1425
|
+
/**
|
|
1426
|
+
* Runs `command` in the sandbox. With `retries`, a non-zero exit runs it again, up to that
|
|
1427
|
+
* many extra times, `interval` ms apart; it resolves with the last try.
|
|
1428
|
+
*/
|
|
1429
|
+
exec(command: string, options?: ExecRetry & {
|
|
1430
|
+
background?: false;
|
|
1431
|
+
}): Promise<ExecResult>;
|
|
1826
1432
|
exec(command: string, options: {
|
|
1827
1433
|
background: true;
|
|
1828
1434
|
}): Promise<BackgroundHandle>;
|
|
1829
|
-
exec(command: string, options?: {
|
|
1435
|
+
exec(command: string, options?: ExecRetry & {
|
|
1830
1436
|
background?: boolean;
|
|
1831
1437
|
}): Promise<ExecResult | BackgroundHandle>;
|
|
1832
1438
|
/**
|
|
@@ -1863,13 +1469,18 @@ type EvalFn = (c: AttemptContext) => Promise<void>;
|
|
|
1863
1469
|
/** A canned agent for tests and dry runs, made by `fakeHarness()` in `reeval/harness`. */
|
|
1864
1470
|
type FakeHarnessDescriptor = {
|
|
1865
1471
|
kind: 'fake';
|
|
1472
|
+
/**
|
|
1473
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1474
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1475
|
+
*/
|
|
1476
|
+
gateway?: GatewayName;
|
|
1866
1477
|
/** Cell id; defaults to `fake`. */
|
|
1867
1478
|
id?: string;
|
|
1868
1479
|
/** Files the fake agent writes into the sandbox on every turn. */
|
|
1869
1480
|
files?: Record<string, string>;
|
|
1870
1481
|
/** The fake agent's final message on every turn; default `Done.`. */
|
|
1871
1482
|
text?: string;
|
|
1872
|
-
/** Simulated turn length,
|
|
1483
|
+
/** Simulated turn length, honors the abort signal. */
|
|
1873
1484
|
delayMs?: number;
|
|
1874
1485
|
/** Sits out `delayMs` even after an abort, like an agent that doesn't stop when it's told to. */
|
|
1875
1486
|
ignoreAbort?: boolean;
|
|
@@ -1949,6 +1560,11 @@ type FakeScriptStep = {
|
|
|
1949
1560
|
*/
|
|
1950
1561
|
type CursorHarnessDescriptor = {
|
|
1951
1562
|
kind: 'cursor';
|
|
1563
|
+
/**
|
|
1564
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1565
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1566
|
+
*/
|
|
1567
|
+
gateway?: GatewayName;
|
|
1952
1568
|
/** A model id `Cursor.models.list()` returns for the key, e.g. `composer-2.5`. */
|
|
1953
1569
|
model: string;
|
|
1954
1570
|
/**
|
|
@@ -1976,6 +1592,11 @@ type ClaudeCodeEffort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
|
1976
1592
|
/** Claude Code through `@ai-sdk/harness-claude-code`, made by `claudeCode()`. */
|
|
1977
1593
|
type ClaudeCodeHarnessDescriptor = {
|
|
1978
1594
|
kind: 'claude-code';
|
|
1595
|
+
/**
|
|
1596
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1597
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1598
|
+
*/
|
|
1599
|
+
gateway?: GatewayName;
|
|
1979
1600
|
/** A model id or alias the `claude` CLI accepts, e.g. `claude-sonnet-5` or `sonnet`. */
|
|
1980
1601
|
model: string;
|
|
1981
1602
|
/**
|
|
@@ -1996,6 +1617,11 @@ type ClaudeCodeHarnessDescriptor = {
|
|
|
1996
1617
|
/** The Antigravity CLI through `@reeval/harness-antigravity`, made by `antigravity()`. */
|
|
1997
1618
|
type AntigravityHarnessDescriptor = {
|
|
1998
1619
|
kind: 'antigravity';
|
|
1620
|
+
/**
|
|
1621
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1622
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1623
|
+
*/
|
|
1624
|
+
gateway?: GatewayName;
|
|
1999
1625
|
/** A model slug `agy models` lists for a Gemini API key, e.g. `gemini-3.8-flash-medium`. */
|
|
2000
1626
|
model: string;
|
|
2001
1627
|
/**
|
|
@@ -2020,6 +1646,11 @@ type AntigravityHarnessDescriptor = {
|
|
|
2020
1646
|
*/
|
|
2021
1647
|
type OpenCodeHarnessDescriptor = {
|
|
2022
1648
|
kind: 'opencode';
|
|
1649
|
+
/**
|
|
1650
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1651
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1652
|
+
*/
|
|
1653
|
+
gateway?: GatewayName;
|
|
2023
1654
|
/**
|
|
2024
1655
|
* OpenCode's model id, `<provider>/<model>`, e.g. `google/gemini-3.8-flash` or
|
|
2025
1656
|
* `anthropic/claude-haiku-4-5`; the provider picks the key.
|
|
@@ -2046,6 +1677,11 @@ type OpenCodeHarnessDescriptor = {
|
|
|
2046
1677
|
*/
|
|
2047
1678
|
type GooseHarnessDescriptor = {
|
|
2048
1679
|
kind: 'goose';
|
|
1680
|
+
/**
|
|
1681
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1682
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1683
|
+
*/
|
|
1684
|
+
gateway?: GatewayName;
|
|
2049
1685
|
/** A Gemini model id, run on Goose's `google` provider, e.g. `gemini-3.8-flash`. */
|
|
2050
1686
|
model: string;
|
|
2051
1687
|
/**
|
|
@@ -2069,6 +1705,11 @@ type GooseHarnessDescriptor = {
|
|
|
2069
1705
|
*/
|
|
2070
1706
|
type PiHarnessDescriptor = {
|
|
2071
1707
|
kind: 'pi';
|
|
1708
|
+
/**
|
|
1709
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1710
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1711
|
+
*/
|
|
1712
|
+
gateway?: GatewayName;
|
|
2072
1713
|
/** A Gemini model id, run on pi's `google` provider, e.g. `gemini-3.8-flash`. */
|
|
2073
1714
|
model: string;
|
|
2074
1715
|
/**
|
|
@@ -2092,6 +1733,11 @@ type PiHarnessDescriptor = {
|
|
|
2092
1733
|
*/
|
|
2093
1734
|
type CodexHarnessDescriptor = {
|
|
2094
1735
|
kind: 'codex';
|
|
1736
|
+
/**
|
|
1737
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1738
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1739
|
+
*/
|
|
1740
|
+
gateway?: GatewayName;
|
|
2095
1741
|
/** An OpenAI model id Codex runs on the Responses API, e.g. `gpt-5.6-luna`. */
|
|
2096
1742
|
model: string;
|
|
2097
1743
|
/**
|
|
@@ -2111,17 +1757,30 @@ type CodexHarnessDescriptor = {
|
|
|
2111
1757
|
};
|
|
2112
1758
|
/**
|
|
2113
1759
|
* fx through `@reeval/harness-fx` (`fx acp`), made by `fx()`. It runs on Vercel AI Gateway or
|
|
2114
|
-
* OpenAI's API by the model id, and reports each turn's tokens.
|
|
1760
|
+
* OpenAI's API by the model id, or on OpenRouter or Vertex AI, and reports each turn's tokens.
|
|
2115
1761
|
*/
|
|
2116
1762
|
type FxHarnessDescriptor = {
|
|
2117
1763
|
kind: 'fx';
|
|
1764
|
+
/**
|
|
1765
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1766
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1767
|
+
*/
|
|
1768
|
+
gateway?: GatewayName;
|
|
2118
1769
|
/**
|
|
2119
1770
|
* A Vercel AI Gateway model id, `<provider>/<model>`, run on fx's `gateway` provider with
|
|
2120
1771
|
* `AI_GATEWAY_API_KEY`, e.g. `anthropic/claude-sonnet-5`; or a bare OpenAI model id, run on
|
|
2121
|
-
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-
|
|
1772
|
+
* OpenAI's Chat Completions API with `OPENAI_API_KEY`, e.g. `gpt-5.5`. On `openrouter()`, either
|
|
1773
|
+
* runs on OpenRouter's Chat Completions API with `OPENROUTER_API_KEY`, a bare id as `openai/<id>`.
|
|
1774
|
+
* On `vertex()`, a Gemini model id, `google/<id>` or a bare `gemini-…` id, run on Vertex AI's
|
|
1775
|
+
* Chat Completions endpoint, e.g. `google/gemini-3.8-flash`.
|
|
2122
1776
|
*/
|
|
2123
1777
|
model: string;
|
|
2124
|
-
/**
|
|
1778
|
+
/**
|
|
1779
|
+
* Reasoning effort, sent as `reasoning_effort` on each model call; only on `vertex()`, since fx
|
|
1780
|
+
* sends none itself. Values are what the model takes there, e.g. `low`, `medium` or `high`.
|
|
1781
|
+
*/
|
|
1782
|
+
effort?: string;
|
|
1783
|
+
/** Cell id; defaults to `fx:<model>`, or `fx:<model>[effort=<value>]` when `effort` is set. */
|
|
2125
1784
|
id?: string;
|
|
2126
1785
|
/** The most fx attempts at once, across the run, e.g. for the key's rate limits. */
|
|
2127
1786
|
maxConcurrent?: number;
|
|
@@ -2137,6 +1796,11 @@ type FxHarnessDescriptor = {
|
|
|
2137
1796
|
*/
|
|
2138
1797
|
type GrokBuildHarnessDescriptor = {
|
|
2139
1798
|
kind: 'grok-build';
|
|
1799
|
+
/**
|
|
1800
|
+
* The gateway this harness's model calls go through, by name. It skips the config's `gateway`
|
|
1801
|
+
* fallback: the plan refuses a gateway that isn't listed or can't serve this harness.
|
|
1802
|
+
*/
|
|
1803
|
+
gateway?: GatewayName;
|
|
2140
1804
|
/** An xAI model id Grok Build runs on the xAI API, e.g. `grok-4.6`. */
|
|
2141
1805
|
model: string;
|
|
2142
1806
|
/**
|
|
@@ -2154,6 +1818,62 @@ type GrokBuildHarnessDescriptor = {
|
|
|
2154
1818
|
*/
|
|
2155
1819
|
startTimeout?: Duration;
|
|
2156
1820
|
};
|
|
1821
|
+
/**
|
|
1822
|
+
* A gateway the config lists by name: `direct` is each provider's own key, as without `gateway`;
|
|
1823
|
+
* `vercel` is Vercel AI Gateway; `openrouter` is OpenRouter; `vertex` is
|
|
1824
|
+
* Google Cloud's Vertex AI.
|
|
1825
|
+
*/
|
|
1826
|
+
type GatewayName = 'direct' | 'vercel' | 'openrouter' | 'vertex';
|
|
1827
|
+
/**
|
|
1828
|
+
* Each provider's own key: made by `direct()` in `reeval/gateway`. Every `gateway` list ends with
|
|
1829
|
+
* it unless it's listed elsewhere, where it's tried in that place.
|
|
1830
|
+
*/
|
|
1831
|
+
type DirectGatewayDescriptor = {
|
|
1832
|
+
kind: 'direct';
|
|
1833
|
+
/**
|
|
1834
|
+
* `false` turns off the fallback to each provider's own key, so a cell no other gateway in the
|
|
1835
|
+
* list serves is refused at plan time. Default `true`.
|
|
1836
|
+
*/
|
|
1837
|
+
enabled?: boolean;
|
|
1838
|
+
};
|
|
1839
|
+
/**
|
|
1840
|
+
* Vercel AI Gateway, with `AI_GATEWAY_API_KEY` (else `VERCEL_OIDC_TOKEN`): made by `vercel()` in
|
|
1841
|
+
* `reeval/gateway`. It serves Claude Code, Codex, Goose, pi, OpenCode, fx and Grok Build.
|
|
1842
|
+
*/
|
|
1843
|
+
type VercelGatewayDescriptor = {
|
|
1844
|
+
kind: 'vercel';
|
|
1845
|
+
};
|
|
1846
|
+
/**
|
|
1847
|
+
* OpenRouter, with `OPENROUTER_API_KEY`: made by `openrouter()` in `reeval/gateway`. It serves
|
|
1848
|
+
* Claude Code, Codex, Goose, OpenCode, fx, Grok Build and pi.
|
|
1849
|
+
*/
|
|
1850
|
+
type OpenRouterGatewayDescriptor = {
|
|
1851
|
+
kind: 'openrouter';
|
|
1852
|
+
/**
|
|
1853
|
+
* Let OpenRouter fail over to another upstream provider when the one it picked can't serve a
|
|
1854
|
+
* call. Off by default, so reeval's own calls (the judge, analysis, compare questions and
|
|
1855
|
+
* captures) send `allow_fallbacks: false`. The harnesses' calls take OpenRouter's own routing
|
|
1856
|
+
* either way.
|
|
1857
|
+
*/
|
|
1858
|
+
fallbacks?: boolean;
|
|
1859
|
+
};
|
|
1860
|
+
/** A model publisher on Vertex AI, which may serve its models in a region of its own. */
|
|
1861
|
+
type VertexPublisher = 'anthropic' | 'google' | 'xai';
|
|
1862
|
+
/**
|
|
1863
|
+
* Google Cloud's Vertex AI, on Application Default Credentials: made by `vertex()` in
|
|
1864
|
+
* `reeval/gateway`.
|
|
1865
|
+
*/
|
|
1866
|
+
type VertexGatewayDescriptor = {
|
|
1867
|
+
kind: 'vertex';
|
|
1868
|
+
/** The Google Cloud project the calls are billed to; defaults to `GOOGLE_CLOUD_PROJECT`. */
|
|
1869
|
+
project?: string;
|
|
1870
|
+
/** The region the models are called in, e.g. `us-east5`; defaults to `global`. */
|
|
1871
|
+
location?: string;
|
|
1872
|
+
/** A region per publisher, where it differs from `location`, e.g. `{ anthropic: 'us-east5' }`. */
|
|
1873
|
+
regions?: Partial<Record<VertexPublisher, string>>;
|
|
1874
|
+
};
|
|
1875
|
+
/** One entry of the config's `gateway` list. */
|
|
1876
|
+
type GatewayDescriptor = DirectGatewayDescriptor | VercelGatewayDescriptor | OpenRouterGatewayDescriptor | VertexGatewayDescriptor;
|
|
2157
1877
|
/** A real coding agent, driven through its AI SDK harness adapter. */
|
|
2158
1878
|
type AgentHarnessDescriptor = CursorHarnessDescriptor | ClaudeCodeHarnessDescriptor | AntigravityHarnessDescriptor | OpenCodeHarnessDescriptor | GooseHarnessDescriptor | PiHarnessDescriptor | CodexHarnessDescriptor | FxHarnessDescriptor | GrokBuildHarnessDescriptor;
|
|
2159
1879
|
/** The agent an attempt runs: a real harness, or the fake one. */
|
|
@@ -2359,12 +2079,13 @@ type SetupStep = string | {
|
|
|
2359
2079
|
cwd?: string;
|
|
2360
2080
|
};
|
|
2361
2081
|
/**
|
|
2362
|
-
* Where the sandbox may connect:
|
|
2363
|
-
*
|
|
2364
|
-
*
|
|
2365
|
-
*
|
|
2082
|
+
* Where the sandbox may connect: any host over HTTP(S) through the egress gateway (`'proxy'`, the
|
|
2083
|
+
* default), anywhere including raw TCP and UDP around the gateway (`'allow-all'`), nowhere
|
|
2084
|
+
* (`'deny-all'`), or only the listed hosts (`'*.example.com'` matches subdomains). Under a
|
|
2085
|
+
* restricted policy the harness's own hosts, brokered secrets' `hosts` and MCP servers' hosts are
|
|
2086
|
+
* allowed too; any other request gets a refusal, and the attempt goes on.
|
|
2366
2087
|
*/
|
|
2367
|
-
type EgressPolicy = 'allow-all' | 'deny-all' | {
|
|
2088
|
+
type EgressPolicy = 'proxy' | 'allow-all' | 'deny-all' | {
|
|
2368
2089
|
allow: readonly string[];
|
|
2369
2090
|
};
|
|
2370
2091
|
/**
|
|
@@ -2651,45 +2372,40 @@ type CompareOptions<E = Environment> = {
|
|
|
2651
2372
|
minLift?: number;
|
|
2652
2373
|
/** Harnesses (ids or descriptors) to compare on; the others run the control only. */
|
|
2653
2374
|
harness?: Array<string | HarnessDescriptor>;
|
|
2654
|
-
/**
|
|
2655
|
-
* Asked of pairs of attempts after the run, always a list. They annotate the compare with
|
|
2656
|
-
* win rates and never change pass, fail or the exit code.
|
|
2657
|
-
*/
|
|
2658
|
-
questions?: CompareQuestion[];
|
|
2659
|
-
/**
|
|
2660
|
-
* The factor every question varies when pairing, when a question does not name its own
|
|
2661
|
-
* `across`. `'environment'` holds the harness; `'harness'` holds the environment. A list of
|
|
2662
|
-
* both, or omitted, asks both pair kinds.
|
|
2663
|
-
*/
|
|
2664
|
-
across?: PairFactor | PairFactor[];
|
|
2665
2375
|
};
|
|
2666
2376
|
/**
|
|
2667
|
-
* A question
|
|
2668
|
-
*
|
|
2669
|
-
*
|
|
2670
|
-
* `
|
|
2671
|
-
* `
|
|
2377
|
+
* A question a language model answers after the attempts: its text, or the text and its options.
|
|
2378
|
+
* With `across`, it picks between two attempts that differ only in that factor, in both orders;
|
|
2379
|
+
* without it, it's asked of each attempt on its own. Without a model, the cascaded
|
|
2380
|
+
* `question.model` answers, else `analysis.model`, else the judge's when it's a language model.
|
|
2381
|
+
* `state` and `artifacts` choose what it's shown of each attempt.
|
|
2672
2382
|
*/
|
|
2673
|
-
type
|
|
2383
|
+
type Question = string | {
|
|
2674
2384
|
question: string;
|
|
2675
2385
|
/** `'provider/model'` that answers it, over the cascaded `question.model`. */
|
|
2676
2386
|
model?: string;
|
|
2677
|
-
/**
|
|
2678
|
-
|
|
2679
|
-
|
|
2387
|
+
/**
|
|
2388
|
+
* The factor it varies when pairing, or both: `'environment'` holds the harness,
|
|
2389
|
+
* `'harness'` holds the environment. Unset, it's asked of each attempt on its own.
|
|
2390
|
+
*/
|
|
2391
|
+
across?: PairFactor | PairFactor[];
|
|
2392
|
+
/** The answers a question asked of each attempt picks from, with a short reason. */
|
|
2393
|
+
choices?: string[];
|
|
2394
|
+
/** What it's shown of each attempt, over the cascaded `question.state`, key by key. */
|
|
2680
2395
|
state?: QuestionStateOptions;
|
|
2681
|
-
/** Each
|
|
2396
|
+
/** Each attempt's own captures, by name; `true` sends all of them. Replaces `question.artifacts`. */
|
|
2682
2397
|
artifacts?: true | string[];
|
|
2683
2398
|
};
|
|
2684
2399
|
/**
|
|
2685
|
-
* What
|
|
2686
|
-
* attempts' workspaces are gone when questions are asked). An unset key keeps its default
|
|
2687
|
-
*
|
|
2400
|
+
* What a question is shown of each attempt: the keys of `judge.state` but `contents` (the
|
|
2401
|
+
* attempts' workspaces are gone when questions are asked). An unset key keeps its default: a
|
|
2402
|
+
* pairwise question shows the task and the diff; one asked of each attempt also shows the final
|
|
2403
|
+
* message and the transcript.
|
|
2688
2404
|
*/
|
|
2689
2405
|
type QuestionStateOptions = {
|
|
2690
2406
|
/** The prompts the attempt was sent, once for both sides when they match. Default true. */
|
|
2691
2407
|
task?: boolean;
|
|
2692
|
-
/** The final message: the agent's own account of its work. Default false. */
|
|
2408
|
+
/** The final message: the agent's own account of its work. Default false in a pair, true per attempt. */
|
|
2693
2409
|
output?: boolean;
|
|
2694
2410
|
/** `status path` lines for the changed files, under `diff`'s patterns. Default false. */
|
|
2695
2411
|
files?: boolean;
|
|
@@ -2701,7 +2417,8 @@ type QuestionStateOptions = {
|
|
|
2701
2417
|
/**
|
|
2702
2418
|
* The transcript as text lines: `true` is messages and tool calls with tool output clipped,
|
|
2703
2419
|
* `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each
|
|
2704
|
-
* harness's tools, so a pair of two harnesses isn't blind. Default false
|
|
2420
|
+
* harness's tools, so a pair of two harnesses isn't blind. Default false in a pair, true per
|
|
2421
|
+
* attempt.
|
|
2705
2422
|
*/
|
|
2706
2423
|
transcript?: boolean | 'tools' | 'full';
|
|
2707
2424
|
/**
|
|
@@ -2722,20 +2439,20 @@ type ResolvedQuestionStateOptions = {
|
|
|
2722
2439
|
turns: 'last' | 'all';
|
|
2723
2440
|
checks: boolean | string[];
|
|
2724
2441
|
};
|
|
2725
|
-
/** Defaults for every
|
|
2442
|
+
/** Defaults for every question: `model` and `artifacts` replaced, `state` merged key by key. */
|
|
2726
2443
|
type QuestionOptions = {
|
|
2727
2444
|
/** `'provider/model'` that answers every question below it without its own `model`. */
|
|
2728
2445
|
model?: string;
|
|
2729
|
-
/** What each
|
|
2446
|
+
/** What it's shown of each attempt. */
|
|
2730
2447
|
state?: QuestionStateOptions;
|
|
2731
|
-
/** Each
|
|
2448
|
+
/** Each attempt's own captures, by name; `true` sends all of them. Default none. */
|
|
2732
2449
|
artifacts?: true | string[];
|
|
2733
2450
|
};
|
|
2734
|
-
/** A matrix factor a
|
|
2451
|
+
/** A matrix factor a pairwise question varies; the others are held. */
|
|
2735
2452
|
type PairFactor = 'environment' | 'harness';
|
|
2736
2453
|
/** Which pairs a vote is over: two environments on one harness, or two harnesses on one environment. */
|
|
2737
2454
|
type CompareCut = 'within' | 'across';
|
|
2738
|
-
/** One
|
|
2455
|
+
/** One judgment of one pair of attempts, in one order. */
|
|
2739
2456
|
type PairVote = {
|
|
2740
2457
|
question: string;
|
|
2741
2458
|
cut: CompareCut;
|
|
@@ -2760,7 +2477,7 @@ type PairVote = {
|
|
|
2760
2477
|
b: string[];
|
|
2761
2478
|
};
|
|
2762
2479
|
};
|
|
2763
|
-
/** A
|
|
2480
|
+
/** A pairwise question's answer: every vote, and each side's wins over decided votes per cut. */
|
|
2764
2481
|
type CompareAnswer = {
|
|
2765
2482
|
question: string;
|
|
2766
2483
|
/** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
|
|
@@ -2784,6 +2501,28 @@ type CompareAnswer = {
|
|
|
2784
2501
|
/** `questions/<n>.json` under the eval's directory: what the model was sent, per pair and order. */
|
|
2785
2502
|
input?: FileRef;
|
|
2786
2503
|
};
|
|
2504
|
+
/** A question asked of one attempt on its own, and the model's answer. Never gates. */
|
|
2505
|
+
type AttemptAnswer = {
|
|
2506
|
+
question: string;
|
|
2507
|
+
/** `'provider/model'`, or `fake` for `fakeAnalysis()`. */
|
|
2508
|
+
model: string;
|
|
2509
|
+
/** The model id the provider says served the call, when it reports one. */
|
|
2510
|
+
resolvedModel?: string;
|
|
2511
|
+
/** What the attempt showed, every key resolved. */
|
|
2512
|
+
stateOptions: ResolvedQuestionStateOptions;
|
|
2513
|
+
/** The captures it sent, as asked: `true` for all, or their names. */
|
|
2514
|
+
artifacts?: true | string[];
|
|
2515
|
+
/** The answers it picked from; unset on a free-text question. */
|
|
2516
|
+
choices?: string[];
|
|
2517
|
+
/** One of `choices`; unset on a free-text question and when the call failed. */
|
|
2518
|
+
choice?: string;
|
|
2519
|
+
/** The model's answer, or its reason for `choice`; capped, and empty when the call failed. */
|
|
2520
|
+
text: string;
|
|
2521
|
+
/** Why there's no answer: the model call failed. Outcomes never change. */
|
|
2522
|
+
error?: string;
|
|
2523
|
+
/** `questions/<n>.json` under the attempt's directory: what the model was sent. */
|
|
2524
|
+
input?: FileRef;
|
|
2525
|
+
};
|
|
2787
2526
|
/** An option a profile's `limit` can narrow, and `pin` can protect. */
|
|
2788
2527
|
type PinnableKey = 'harness' | 'runs' | 'compare';
|
|
2789
2528
|
/**
|
|
@@ -2807,7 +2546,11 @@ type EvalOptionFields = {
|
|
|
2807
2546
|
* Not an environment: it does not invent skills, files or a name.
|
|
2808
2547
|
*/
|
|
2809
2548
|
sandbox?: SandboxDescriptor;
|
|
2810
|
-
/**
|
|
2549
|
+
/**
|
|
2550
|
+
* Attempts per cell. Unset at every level, a compare with variants runs 4 a side, the fewest
|
|
2551
|
+
* whose lift can reach a verdict, and anything else runs once, a pooled `evaluate.each` case
|
|
2552
|
+
* included.
|
|
2553
|
+
*/
|
|
2811
2554
|
runs?: number;
|
|
2812
2555
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
2813
2556
|
gate?: GatePolicy;
|
|
@@ -2822,15 +2565,38 @@ type EvalOptionFields = {
|
|
|
2822
2565
|
judge?: JudgeOptions;
|
|
2823
2566
|
/** Screenshot defaults for `c.capture()`. Shallow-merged; `viewport` and `model` replace the value. */
|
|
2824
2567
|
capture?: CaptureOptions;
|
|
2825
|
-
/** Defaults for every
|
|
2568
|
+
/** Defaults for every question: `model`, and `state` merged key by key, down the cascade. */
|
|
2826
2569
|
question?: QuestionOptions;
|
|
2570
|
+
/**
|
|
2571
|
+
* What a language model is asked about the attempts, always a list; they never change pass,
|
|
2572
|
+
* fail or the exit code. A question with `across` picks between pairs of attempts once they all
|
|
2573
|
+
* end; one without is asked of each attempt as it ends. Replaces the inherited list, so `[]`
|
|
2574
|
+
* asks none.
|
|
2575
|
+
*/
|
|
2576
|
+
questions?: Question[];
|
|
2827
2577
|
/** Per attempt. */
|
|
2828
2578
|
timeout?: Duration;
|
|
2829
2579
|
/** Labels for `--tag` and profiles to select by; a group's are added to its evals'. */
|
|
2830
2580
|
tags?: string[];
|
|
2581
|
+
/**
|
|
2582
|
+
* The eval's version, part of its fingerprint: bump it when a change the eval file doesn't show
|
|
2583
|
+
* (a helper, a fixture, data it reads) changes what it measures. A group's applies to the evals
|
|
2584
|
+
* inside it that set none.
|
|
2585
|
+
*/
|
|
2586
|
+
revision?: Revision;
|
|
2831
2587
|
/** Keys a profile's `limit` may not narrow for this group or eval. */
|
|
2832
2588
|
pin?: PinnableKey[];
|
|
2833
2589
|
};
|
|
2590
|
+
/** An eval's `revision`: a whole number or a non-empty string, compared exactly. */
|
|
2591
|
+
type Revision = number | string;
|
|
2592
|
+
/**
|
|
2593
|
+
* An eval's own source as hashes: its body and case data as declared, and its options and its
|
|
2594
|
+
* enclosing groups' in canonical form (`tags`, `revision` and `gate` aside).
|
|
2595
|
+
*/
|
|
2596
|
+
type EvalSource = DeclaredSource & {
|
|
2597
|
+
/** sha256 of the canonical options. */
|
|
2598
|
+
options: string;
|
|
2599
|
+
};
|
|
2834
2600
|
/**
|
|
2835
2601
|
* `compare: false` is not an option here: a profile's `limit.compare: false` strips variants.
|
|
2836
2602
|
* A compare with a `baseline` names every arm, so the cascaded `environment` is not applied, and
|
|
@@ -2879,7 +2645,7 @@ type ConfigJudgeOptions = Omit<JudgeOptions, 'model'> & {
|
|
|
2879
2645
|
type HistoryConfig = {
|
|
2880
2646
|
/** Committed history folder (default: `.history` in the directory the evals live in). */
|
|
2881
2647
|
dir?: string;
|
|
2882
|
-
/** Artifacts recorded next to each run: every
|
|
2648
|
+
/** Artifacts recorded next to each run: every attempt's, failed attempts' only, or none. */
|
|
2883
2649
|
artifacts?: ArtifactLevel;
|
|
2884
2650
|
};
|
|
2885
2651
|
/** Run-level caps. `attempts` is checked against the plan; `duration` while running. */
|
|
@@ -2911,7 +2677,7 @@ type ProfileDefaults = {
|
|
|
2911
2677
|
timeout?: Duration;
|
|
2912
2678
|
/** Screenshot defaults for `c.capture()`, shallow-merged over the config's. */
|
|
2913
2679
|
capture?: CaptureOptions;
|
|
2914
|
-
/** Defaults for every
|
|
2680
|
+
/** Defaults for every question, merged over the config's. */
|
|
2915
2681
|
question?: QuestionOptions;
|
|
2916
2682
|
};
|
|
2917
2683
|
/** Over every eval, and it can only narrow; `pin` on a group or eval skips it. */
|
|
@@ -2939,6 +2705,8 @@ type Profile = {
|
|
|
2939
2705
|
record?: RecordMode;
|
|
2940
2706
|
/** Tag filters for this profile, like `--tag` (`'!slow'` excludes). */
|
|
2941
2707
|
tags?: string[];
|
|
2708
|
+
/** Replaces the config's `gateway` list for this profile's runs. */
|
|
2709
|
+
gateway?: GatewayDescriptor[];
|
|
2942
2710
|
};
|
|
2943
2711
|
/** A canned analysis for tests; see `fakeAnalysis()` in `reeval/judge`. */
|
|
2944
2712
|
type FakeAnalysisModel = {
|
|
@@ -2946,95 +2714,47 @@ type FakeAnalysisModel = {
|
|
|
2946
2714
|
/** The model's structured answer, or a function of the digest it was given. */
|
|
2947
2715
|
output?: unknown | ((digest: unknown) => unknown);
|
|
2948
2716
|
/**
|
|
2949
|
-
* How it answers
|
|
2717
|
+
* How it answers pairwise questions: a fixed pick, a function of the blinded pair, or
|
|
2950
2718
|
* `{ error }` to fail every call. A pick may be `{ winner, reason? }`. Default `'tie'`.
|
|
2951
2719
|
*/
|
|
2952
2720
|
vote?: FakePick | ((pair: BlindPair) => FakePick) | {
|
|
2953
2721
|
error: string;
|
|
2954
2722
|
};
|
|
2955
2723
|
/**
|
|
2956
|
-
*
|
|
2957
|
-
*
|
|
2958
|
-
*
|
|
2959
|
-
* token per 4 characters of each call's input and answer, so `analysis.maxTokens` applies.
|
|
2724
|
+
* How it answers questions asked of each attempt: a fixed answer, a function of the attempt as
|
|
2725
|
+
* the model sees it, or `{ error }` to fail every call. An answer may be `{ choice?, text? }`.
|
|
2726
|
+
* Default the first of the question's `choices`, with the text `fake answer`.
|
|
2960
2727
|
*/
|
|
2961
|
-
|
|
2728
|
+
answer?: FakeAnswer | ((attempt: BlindAttempt) => FakeAnswer) | {
|
|
2962
2729
|
error: string;
|
|
2963
2730
|
};
|
|
2964
2731
|
};
|
|
2965
|
-
/** What a fake insights function is given: one eval's scoped facts, or the run's evals for the headline. */
|
|
2966
|
-
type FakeInsightsInput = ModelInsightInput | ModelHeadlineInput;
|
|
2967
|
-
/**
|
|
2968
|
-
* A fake insight answer. Each call takes its part: `intent` the classification, `answer`,
|
|
2969
|
-
* `findings`, `rows`, `namedTools` and `remarks` the eval's, `headline` the run's. Whatever's
|
|
2970
|
-
* left out is empty, and the intent defaults to `job: 'other'`.
|
|
2971
|
-
*/
|
|
2972
|
-
type FakeInsights = {
|
|
2973
|
-
intent?: {
|
|
2974
|
-
job: 'choose' | 'finish' | 'other';
|
|
2975
|
-
question?: string;
|
|
2976
|
-
subject?: {
|
|
2977
|
-
tool?: string;
|
|
2978
|
-
change?: string;
|
|
2979
|
-
};
|
|
2980
|
-
name?: 'states-intent' | 'no-intent' | 'contradicts';
|
|
2981
|
-
/** `low` falls back to `job: 'other'`, as a real classification does. */
|
|
2982
|
-
confidence?: 'high' | 'medium' | 'low';
|
|
2983
|
-
};
|
|
2984
|
-
answer?: {
|
|
2985
|
-
text: string;
|
|
2986
|
-
cites: string[];
|
|
2987
|
-
} | null;
|
|
2988
|
-
findings?: Array<{
|
|
2989
|
-
text: string;
|
|
2990
|
-
cites: string[];
|
|
2991
|
-
}>;
|
|
2992
|
-
rows?: Array<{
|
|
2993
|
-
row: string;
|
|
2994
|
-
text: string;
|
|
2995
|
-
cites: string[];
|
|
2996
|
-
}>;
|
|
2997
|
-
namedTools?: Array<{
|
|
2998
|
-
attempt: string;
|
|
2999
|
-
tools: string[];
|
|
3000
|
-
}>;
|
|
3001
|
-
/**
|
|
3002
|
-
* The eval's remarks, as the model's JSON would have them: `source.at` and `fix.at` are
|
|
3003
|
-
* `L<n>` lines of the eval's source, `evidence` fact ids (`F1`) or attempt lines (`A1#L3`).
|
|
3004
|
-
* They go through the same gates as a model's, so one missing a part is dropped.
|
|
3005
|
-
*/
|
|
3006
|
-
remarks?: Array<{
|
|
3007
|
-
kind: string;
|
|
3008
|
-
text: string;
|
|
3009
|
-
source?: {
|
|
3010
|
-
at: string;
|
|
3011
|
-
quote: string;
|
|
3012
|
-
} | null;
|
|
3013
|
-
evidence?: string[];
|
|
3014
|
-
fix?: {
|
|
3015
|
-
at: string;
|
|
3016
|
-
edit: string;
|
|
3017
|
-
} | null;
|
|
3018
|
-
}>;
|
|
3019
|
-
headline?: {
|
|
3020
|
-
text: string;
|
|
3021
|
-
cites: string[];
|
|
3022
|
-
} | null;
|
|
3023
|
-
};
|
|
3024
2732
|
/** A blinded label's pick: the attempt shown first (`A`), second (`B`), or neither. */
|
|
3025
2733
|
type FakeVote = 'A' | 'B' | 'tie';
|
|
3026
|
-
/** A fake
|
|
2734
|
+
/** A fake pairwise pick, optionally with a reason. */
|
|
3027
2735
|
type FakePick = FakeVote | {
|
|
3028
2736
|
winner: FakeVote;
|
|
3029
2737
|
reason?: string;
|
|
3030
2738
|
};
|
|
3031
|
-
/**
|
|
2739
|
+
/** A fake per-attempt answer: its text, or a choice and its reason. */
|
|
2740
|
+
type FakeAnswer = string | {
|
|
2741
|
+
choice?: string;
|
|
2742
|
+
text?: string;
|
|
2743
|
+
};
|
|
2744
|
+
/** What a pairwise question's model is shown: the prompt, and the two sides in slot order. */
|
|
3032
2745
|
type BlindPair = {
|
|
3033
2746
|
question: string;
|
|
3034
2747
|
prompt: string;
|
|
3035
2748
|
a: BlindSide;
|
|
3036
2749
|
b: BlindSide;
|
|
3037
2750
|
};
|
|
2751
|
+
/** What a per-attempt question's model is shown: the prompt, its choices, and the attempt. */
|
|
2752
|
+
type BlindAttempt = {
|
|
2753
|
+
question: string;
|
|
2754
|
+
prompt: string;
|
|
2755
|
+
choices?: string[];
|
|
2756
|
+
attempt: BlindSide;
|
|
2757
|
+
};
|
|
3038
2758
|
/** A turn of one side, with the keys its question's `state` turned on. */
|
|
3039
2759
|
type BlindTurn = {
|
|
3040
2760
|
task?: string;
|
|
@@ -3042,9 +2762,9 @@ type BlindTurn = {
|
|
|
3042
2762
|
transcript?: string;
|
|
3043
2763
|
};
|
|
3044
2764
|
/**
|
|
3045
|
-
* One side of a blinded pair
|
|
3046
|
-
* `turns: 'all'`, `task`, `output` and `transcript` are
|
|
3047
|
-
* the pair's prompt's, not either side's.
|
|
2765
|
+
* One side of a blinded pair, or the attempt a per-attempt question is asked of: only the keys
|
|
2766
|
+
* its question's `state` turned on. With `turns: 'all'`, `task`, `output` and `transcript` are
|
|
2767
|
+
* under `turns`; a task both sides of a pair share is the pair's prompt's, not either side's.
|
|
3048
2768
|
*/
|
|
3049
2769
|
type BlindSide = BlindTurn & {
|
|
3050
2770
|
turns?: BlindTurn[];
|
|
@@ -3061,7 +2781,7 @@ type BlindSide = BlindTurn & {
|
|
|
3061
2781
|
command: string;
|
|
3062
2782
|
notRun: true;
|
|
3063
2783
|
}>;
|
|
3064
|
-
/** Its captures, in the order sent,
|
|
2784
|
+
/** Its captures, in the order sent, labeled by position; a name it lacks is `missing`. */
|
|
3065
2785
|
artifacts?: Array<{
|
|
3066
2786
|
label: string;
|
|
3067
2787
|
mediaType: 'image/png';
|
|
@@ -3071,24 +2791,13 @@ type BlindSide = BlindTurn & {
|
|
|
3071
2791
|
missing: true;
|
|
3072
2792
|
}>;
|
|
3073
2793
|
};
|
|
3074
|
-
/**
|
|
2794
|
+
/** `reeval analyze`: a language model (not an evaluation model) and what matters in this project. */
|
|
3075
2795
|
type AnalysisConfig = {
|
|
3076
2796
|
/**
|
|
3077
|
-
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`.
|
|
3078
|
-
*
|
|
2797
|
+
* `'provider/model'`, e.g. `'google/gemini-3.8-flash'`. Unset, `reeval analyze` and questions
|
|
2798
|
+
* fall back to the judge's when it's a language model.
|
|
3079
2799
|
*/
|
|
3080
2800
|
model?: string | FakeAnalysisModel;
|
|
3081
|
-
/**
|
|
3082
|
-
* `reeval run` classifies each eval and explains its result with `model` once the attempts
|
|
3083
|
-
* finish (default true when `model` is set; `--no-insights` turns it off for one run).
|
|
3084
|
-
*/
|
|
3085
|
-
insights?: boolean;
|
|
3086
|
-
/**
|
|
3087
|
-
* Tokens (in and out) that one run's insights may use, over every classification, insight and
|
|
3088
|
-
* headline call (default 250000). Past it, the evals left print without insights; the run's
|
|
3089
|
-
* outcome and exit code never change.
|
|
3090
|
-
*/
|
|
3091
|
-
maxTokens?: number;
|
|
3092
2801
|
/** Added to the prompt, e.g. "Fewer tool calls count as a win. The skill under test is reeval-good." */
|
|
3093
2802
|
instructions?: string;
|
|
3094
2803
|
/** The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000). */
|
|
@@ -3108,13 +2817,13 @@ type ReevalConfig = {
|
|
|
3108
2817
|
sandbox?: SandboxDescriptor;
|
|
3109
2818
|
/** What every eval runs in when it names no environment, and a baseline-less compare's control. */
|
|
3110
2819
|
environment?: Environment;
|
|
3111
|
-
/** Attempts per cell;
|
|
2820
|
+
/** Attempts per cell; unset, a compare with variants runs 4 a side and anything else once. */
|
|
3112
2821
|
runs?: number;
|
|
3113
2822
|
/** How each cell's attempts decide its outcome; default `'all'`. */
|
|
3114
2823
|
gate?: GatePolicy;
|
|
3115
2824
|
/** Variables and secrets for the sandbox, merged by name down the cascade. */
|
|
3116
2825
|
env?: EnvVars;
|
|
3117
|
-
/** Where the sandbox may connect; default `'
|
|
2826
|
+
/** Where the sandbox may connect; default `'proxy'`. */
|
|
3118
2827
|
egress?: EgressPolicy;
|
|
3119
2828
|
/** Defaults for `c.judge()`, merged field by field down the cascade. */
|
|
3120
2829
|
judge?: ConfigJudgeOptions;
|
|
@@ -3125,7 +2834,7 @@ type ReevalConfig = {
|
|
|
3125
2834
|
* binary when `c.capture()` runs; it does not change this object.
|
|
3126
2835
|
*/
|
|
3127
2836
|
capture?: CaptureOptions;
|
|
3128
|
-
/** Defaults for every
|
|
2837
|
+
/** Defaults for every question: the model that answers it and what it's shown of each attempt. Merged down the cascade. */
|
|
3129
2838
|
question?: QuestionOptions;
|
|
3130
2839
|
/** How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it. */
|
|
3131
2840
|
concurrency?: number;
|
|
@@ -3142,7 +2851,7 @@ type ReevalConfig = {
|
|
|
3142
2851
|
ci?: string;
|
|
3143
2852
|
local?: string;
|
|
3144
2853
|
};
|
|
3145
|
-
/** `unbrokered: 'error'` refuses Docker secrets without `hosts
|
|
2854
|
+
/** `unbrokered: 'error'` refuses Docker secrets without `hosts`, and credential-like MCP headers written as plain strings, instead of warning (default `'warn'`). */
|
|
3146
2855
|
secrets?: {
|
|
3147
2856
|
unbrokered?: 'warn' | 'error';
|
|
3148
2857
|
};
|
|
@@ -3154,8 +2863,14 @@ type ReevalConfig = {
|
|
|
3154
2863
|
keep?: number;
|
|
3155
2864
|
traceBufferMb?: number;
|
|
3156
2865
|
};
|
|
3157
|
-
/**
|
|
2866
|
+
/** `reeval analyze`, and questions without a model of their own. */
|
|
3158
2867
|
analysis?: AnalysisConfig;
|
|
2868
|
+
/**
|
|
2869
|
+
* The gateways model calls may go through, in the order you prefer them: each harness takes
|
|
2870
|
+
* the first that can serve it. `direct()`, each provider's own key, ends the list unless it's listed
|
|
2871
|
+
* in another place or turned off with `direct({ enabled: false })`. Default `[direct()]`.
|
|
2872
|
+
*/
|
|
2873
|
+
gateway?: GatewayDescriptor[];
|
|
3159
2874
|
};
|
|
3160
2875
|
/** What a `defineConfig` callback gets: the environment helpers plus the run's context. */
|
|
3161
2876
|
type ConfigHelpers = EnvironmentHelpers & {
|
|
@@ -3307,12 +3022,20 @@ type EnvironmentRecord = {
|
|
|
3307
3022
|
sha256: string;
|
|
3308
3023
|
}>;
|
|
3309
3024
|
vars: string[];
|
|
3025
|
+
/**
|
|
3026
|
+
* Every secret, by the variable it's set as in the sandbox; an MCP header's (`secret.fromEnv`,
|
|
3027
|
+
* `secret.template`, a plugin's `${VAR}`) by each host variable it reads.
|
|
3028
|
+
*/
|
|
3310
3029
|
secrets: string[];
|
|
3311
|
-
/**
|
|
3030
|
+
/**
|
|
3031
|
+
* Secrets the sandbox only had placeholders for, swapped in by the egress gateway, named as in
|
|
3032
|
+
* `secrets`. A variable it also got as its real value somewhere (an `http://` header, an env
|
|
3033
|
+
* secret without `hosts`) isn't listed.
|
|
3034
|
+
*/
|
|
3312
3035
|
brokered: string[];
|
|
3313
3036
|
/** Optional secrets whose host variable was unset, so they weren't injected. */
|
|
3314
3037
|
unset: string[];
|
|
3315
|
-
/** The policy the attempt ran under, from the options and the environment; in the digest unless `
|
|
3038
|
+
/** The policy the attempt ran under, from the options and the environment; in the digest unless `proxy`. */
|
|
3316
3039
|
egress: EgressPolicy;
|
|
3317
3040
|
/**
|
|
3318
3041
|
* The git repository the workspace started as: the repo as written (a local one relative to the
|
|
@@ -3331,15 +3054,22 @@ type EnvironmentRecord = {
|
|
|
3331
3054
|
};
|
|
3332
3055
|
/**
|
|
3333
3056
|
* An attempt's network traffic. `allowed` is every host a restricted policy let through, its own
|
|
3334
|
-
* and the automatic ones; null under `allow-all`. `recorded` is false when the sandbox
|
|
3335
|
-
* the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3336
|
-
* sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3337
|
-
* always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
|
|
3057
|
+
* and the automatic ones; null under `proxy` and `allow-all`. `recorded` is false when the sandbox
|
|
3058
|
+
* can't see the traffic, and then `contacted` and `blocked` are empty. Bytes are counted between
|
|
3059
|
+
* the sandbox and the egress gateway, TLS included. `expectedBlocked` are refused hosts the
|
|
3060
|
+
* harness always tries (its CLI's telemetry, say), kept out of `blocked` so a real block stands
|
|
3061
|
+
* out.
|
|
3338
3062
|
*/
|
|
3339
3063
|
type EgressRecord = {
|
|
3340
3064
|
policy: EgressPolicy;
|
|
3341
3065
|
allowed: string[] | null;
|
|
3342
3066
|
recorded: boolean;
|
|
3067
|
+
/**
|
|
3068
|
+
* True when traffic could leave around the gateway (`allow-all` in a container that kept its
|
|
3069
|
+
* network): `contacted` lists only what went through the gateway, and anything that ignored the
|
|
3070
|
+
* proxy (raw TCP and UDP, SSH, database drivers) went out unrecorded. Unset otherwise.
|
|
3071
|
+
*/
|
|
3072
|
+
partial?: true;
|
|
3343
3073
|
contacted: Array<{
|
|
3344
3074
|
host: string;
|
|
3345
3075
|
port: number;
|
|
@@ -3391,6 +3121,16 @@ type StartRetry = {
|
|
|
3391
3121
|
/** How long it waited before giving up on that start. */
|
|
3392
3122
|
waitedMs: number;
|
|
3393
3123
|
};
|
|
3124
|
+
/**
|
|
3125
|
+
* A process the setup steps left running in a Docker sandbox with a `to: 'checks'` secret without
|
|
3126
|
+
* `hosts` in its environment: it holds the real value while the agent works.
|
|
3127
|
+
*/
|
|
3128
|
+
type SetupProcess = {
|
|
3129
|
+
/** Its command line, cut at 200 characters. */
|
|
3130
|
+
command: string;
|
|
3131
|
+
/** The `to: 'checks'` secrets without `hosts` in its environment, by name. */
|
|
3132
|
+
secrets: string[];
|
|
3133
|
+
};
|
|
3394
3134
|
/** One `expect` or `expect.soft` matcher call an attempt made. */
|
|
3395
3135
|
type AssertionRecord = {
|
|
3396
3136
|
/**
|
|
@@ -3436,6 +3176,14 @@ type SandboxWarning = {
|
|
|
3436
3176
|
/** The line reeval prints, after `WARNING: ` (root) or `warning: ` (capabilities). */
|
|
3437
3177
|
message: string;
|
|
3438
3178
|
};
|
|
3179
|
+
/** Where a cell's harness resolved in the `gateway` list, and the entries before it that couldn't serve it. */
|
|
3180
|
+
type ResolvedGateway = {
|
|
3181
|
+
name: GatewayName;
|
|
3182
|
+
skipped: Array<{
|
|
3183
|
+
gateway: GatewayName;
|
|
3184
|
+
why: string;
|
|
3185
|
+
}>;
|
|
3186
|
+
};
|
|
3439
3187
|
//#endregion
|
|
3440
|
-
export {
|
|
3441
|
-
//# sourceMappingURL=types-
|
|
3188
|
+
export { FakeVote as $, VertexPublisher as $t, Diff as A, VERDICT_TEXT as An, Profile as At, EnvironmentInput as B, effortSample as Bn, Secret as Bt, CodexHarnessDescriptor as C, EFFORT_MIN_RUNS as Cn, Outcome as Ct, ConfigHelpers as D, EffortVerdict as Dn, PinnableKey as Dt, CompareOptions as E, EffortSample as En, PiHarnessDescriptor as Et, EgressPolicy as F, contrastEffort as Fn, QuestionStateOptions as Ft, ExecResult as G, formatSeconds as Gn, TokenUsage as Gt, ErrorKind as H, effortVerdict as Hn, ServeHandle as Ht, EgressRecord as I, directional as In, ReevalConfig as It, FakeAnswer as J, DeclaredNode as Jn, TurnData as Jt, ExecRetry as K, tokenValues as Kn, TranscriptEvent as Kt, EnvVars as L, effortChangeLabel as Ln, ResolvedGateway as Lt, DirectoryOptions as M, cellContrast as Mn, ProfileLimit as Mt, DockerSandboxDescriptor as N, cellEffort as Nn, Question as Nt, ConfigJudgeOptions as O, Spread as On, PluginPart as Ot, Duration as P, cellSamples as Pn, QuestionOptions as Pt, FakeScriptStep as Q, VertexGatewayDescriptor as Qt, Environment as R, effortLabel as Rn, SandboxDescriptor as Rt, ClaudeCodeHarnessDescriptor as S, EFFORT_ALPHA as Sn, OpenRouterGatewayDescriptor as St, CompareCut as T, EffortMetric as Tn, PairVote as Tt, EvalFn as U, formatCount as Un, SetupStep as Ut, EnvironmentSpec as V, effortSummary as Vn, SecretTemplate as Vt, EvalOptions as W, formatRatio as Wn, SkillSource as Wt, FakeJudgeModel as X, VercelGatewayDescriptor as Xt, FakeHarnessDescriptor as Y, TurnStatus as Yt, FakeSandboxDescriptor as Z, Verdict as Zt, CaptureWait as _, RunRecord as _n, JudgeOptions as _t, AttemptContext as a, CellRecord as an, GroupOptions as at, CellRole as b, CellEffort as bn, McpServer as bt, BlindPair as c, FamilyLift as cn, HarnessEdit as ct, CaptureCall as d, FamilyRecord as dn, HostFile as dt, Viewport as en, FxHarnessDescriptor as et, CaptureInstruction as f, FileRef as fn, JSONValue as ft, CaptureUseScript as g, Lift as gn, JudgeModel as gt, CaptureUseAction as h, JudgmentRecord as hn, JudgeFailure as ht, AttemptAnswer as i, AttemptRecord as in, GrokBuildHarnessDescriptor as it, DirectGatewayDescriptor as j, bothUnderATenth as jn, ProfileDefaults as jt, CursorHarnessDescriptor as k, TOKENS_UNREPORTED as kn, PluginSource as kt, Budget as l, FamilyLiftSummary as ln, HarnessOption as lt, CaptureUse as m, HistoryEntry as mn, JudgeCallOptions as mt, AnalysisConfig as n, WebEventType as nn, GatewayName as nt, BackgroundHandle as o, CheckTally as on, HarnessData as ot, CaptureOptions as p, HistoryCell as pn, JudgeArtifactRef as pt, FakeAnalysisModel as q, DeclarationTree as qn, Turn as qt, AntigravityHarnessDescriptor as r, ArtifactRef as rn, GooseHarnessDescriptor as rt, BlindAttempt as s, EvalEntry as sn, HarnessDescriptor as st, AgentHarnessDescriptor as t, Wait as tn, GatewayDescriptor as tt, Capture as u, FamilyRate as un, HistoryConfig as ut, CaptureWindow as v, StackSummary as vn, JudgeState as vt, CompareAnswer as w, EffortContrast as wn, PairFactor as wt, ClaudeCodeEffort as x, DIRECTIONAL_MIN as xn, OpenCodeHarnessDescriptor as xt, Cell as y, UpstreamCount as yn, JudgeStateOptions as yt, EnvironmentHelpers as z, effortOf as zn, SandboxWarning as zt };
|
|
3189
|
+
//# sourceMappingURL=types-coMWHaU2.d.ts.map
|