reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
package/dist/view/index.d.ts
CHANGED
|
@@ -1,95 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
//#region src/stats.d.ts
|
|
3
|
-
/**
|
|
4
|
-
* Counted attempts a side before a pass-rate lift can be called: below it, every view says
|
|
5
|
-
* `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
|
|
6
|
-
* `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
|
|
7
|
-
* makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
|
|
8
|
-
* the time when nothing changed.
|
|
9
|
-
*/
|
|
10
|
-
export declare const EFFECT_MIN_COUNTED = 4;
|
|
11
|
-
//#endregion
|
|
12
|
-
//#region src/view/debug.d.ts
|
|
13
|
-
/**
|
|
14
|
-
* An attempt's local debug data, as `reeval run` writes it to `.reeval/debug/<runId>/<eval>/<cell>/<n>/`
|
|
15
|
-
* and `reeval studio` serves it. Browser-safe. Never part of a record or the history folder.
|
|
16
|
-
*
|
|
17
|
-
* debug.log the `-v` trace, one line per event, always written
|
|
18
|
-
* requests.ndjson one line per egress gateway request, always written
|
|
19
|
-
* trace.ndjson raw payloads (`-vv`), written when the attempt fails or with `--trace`
|
|
20
|
-
*/
|
|
21
|
-
export declare const DEBUG_LOG = "debug.log";
|
|
22
|
-
export declare const REQUESTS_FILE = "requests.ndjson";
|
|
23
|
-
export declare const TRACE_FILE = "trace.ndjson";
|
|
24
|
-
/** What part of the attempt a line is about, in the order an attempt goes through them. */
|
|
25
|
-
export declare const DEBUG_PHASES: readonly ['plan', 'image', 'container', 'env', 'broker', 'harness', 'check', 'judge', 'gateway', 'teardown', 'docker'];
|
|
26
|
-
export type DebugPhase = (typeof DEBUG_PHASES)[number];
|
|
27
|
-
/** One debug line: level 1 is `-v`, level 2 is `-vv` only. */
|
|
28
|
-
export type DebugLine = {
|
|
29
|
-
at: number;
|
|
30
|
-
level: 1 | 2;
|
|
31
|
-
phase: DebugPhase;
|
|
32
|
-
text: string;
|
|
33
|
-
/** More lines under it, e.g. a docker command's output or a raw payload. */
|
|
34
|
-
detail?: string;
|
|
35
|
-
};
|
|
36
|
-
/** One raw trace entry: `ms` is since the attempt started. */
|
|
37
|
-
export type TraceEntry = {
|
|
38
|
-
at: string;
|
|
39
|
-
ms: number;
|
|
40
|
-
phase: DebugPhase;
|
|
41
|
-
kind: string;
|
|
42
|
-
data: unknown;
|
|
43
|
-
};
|
|
44
|
-
/** One request through the egress gateway, or one tunnel it passed through without reading. */
|
|
45
|
-
export type DebugRequest = {
|
|
46
|
-
at: string;
|
|
47
|
-
method: string;
|
|
48
|
-
url: string;
|
|
49
|
-
host: string;
|
|
50
|
-
port: number;
|
|
51
|
-
via: 'tunnel' | 'decrypted' | 'http';
|
|
52
|
-
allowed: boolean;
|
|
53
|
-
status?: number;
|
|
54
|
-
reason?: string;
|
|
55
|
-
sent: number;
|
|
56
|
-
received: number;
|
|
57
|
-
/** Brokered secrets and the harness credential swapped in, by name. */
|
|
58
|
-
substituted: string[];
|
|
59
|
-
ms: number;
|
|
60
|
-
};
|
|
61
|
-
/** What the studio's Debug tab shows for one attempt. */
|
|
62
|
-
export type DebugView = {
|
|
63
|
-
lines: Array<{
|
|
64
|
-
at: string;
|
|
65
|
-
phase: string;
|
|
66
|
-
text: string;
|
|
67
|
-
}>;
|
|
68
|
-
requests: DebugRequest[];
|
|
69
|
-
/** Whether `trace.ndjson` was kept. */
|
|
70
|
-
trace: boolean;
|
|
71
|
-
};
|
|
72
|
-
/**
|
|
73
|
-
* The studio server's debug API (`reeval studio` only): an attempt's DebugView (null when it has no
|
|
74
|
-
* local data), and its raw trace.
|
|
75
|
-
*/
|
|
76
|
-
export declare const debugApiPath: (runId: string, attemptKey: string) => string;
|
|
77
|
-
export declare const traceApiPath: (runId: string, attemptKey: string) => string;
|
|
78
|
-
/** `12:50:01`, or `12:50:01.234` with milliseconds, in UTC like every reeval timestamp. */
|
|
79
|
-
export declare const debugTime: (at: number, ms: boolean) => string;
|
|
80
|
-
/** A line of debug.log: `2026-09-26T12:50:01.234Z plan text`. */
|
|
81
|
-
export declare const debugLogLine: (line: Pick<DebugLine, 'at' | 'phase' | 'text'>) => string;
|
|
82
|
-
/** debug.log's lines; a line that isn't one (a wrapped message) continues the one before. */
|
|
83
|
-
export declare function parseDebugLog(text: string): DebugView['lines'];
|
|
84
|
-
export declare function parseNdjson<T>(text: string): T[];
|
|
85
|
-
/** `512 B`, `1.2 KB`, `3.4 MB`. */
|
|
86
|
-
export declare function formatBytes(n: number): string;
|
|
87
|
-
/**
|
|
88
|
-
* The gateway line for a request: method, URL, status, allowed or blocked and why, bytes each way,
|
|
89
|
-
* time, and which secrets were swapped in (by name).
|
|
90
|
-
*/
|
|
91
|
-
export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
|
|
92
|
-
//#endregion
|
|
1
|
+
import { An as VERDICT_TEXT, Bn as effortSample, Cn as EFFORT_MIN_RUNS, Ct as Outcome, Dn as EffortVerdict, En as EffortSample, Fn as contrastEffort, Gn as formatSeconds, Gt as TokenUsage, H as ErrorKind, Hn as effortVerdict, In as directional, Kn as tokenValues, Kt as TranscriptEvent, Ln as effortChangeLabel, Mn as cellContrast, Nn as cellEffort, On as Spread, Pn as cellSamples, Rn as effortLabel, Sn as EFFORT_ALPHA, Tn as EffortMetric, Un as formatCount, Vn as effortSummary, Wn as formatRatio, Yt as TurnStatus, _n as RunRecord, an as CellRecord, bn as CellEffort, cn as FamilyLift, dn as FamilyRecord, fn as FileRef, gn as Lift, hn as JudgmentRecord, in as AttemptRecord, jn as bothUnderATenth, kn as TOKENS_UNREPORTED, ln as FamilyLiftSummary, mn as HistoryEntry, on as CheckTally, pn as HistoryCell, rn as ArtifactRef, sn as EvalEntry, un as FamilyRate, vn as StackSummary, wn as EffortContrast, xn as DIRECTIONAL_MIN, yn as UpstreamCount, zn as effortOf, zt as SandboxWarning } from "../types-coMWHaU2.js";
|
|
93
2
|
//#region src/view/analysis.d.ts
|
|
94
3
|
/** Bumped when the system prompt or output schema changes: an older analysis is superseded. */
|
|
95
4
|
export declare const ANALYSIS_PROMPT_VERSION = "analysis-v11";
|
|
@@ -155,7 +64,7 @@ export type AnalysisValidation = {
|
|
|
155
64
|
export type AnalysisScope = {
|
|
156
65
|
kind: 'run';
|
|
157
66
|
head: string;
|
|
158
|
-
/** Resolved once, so a nearer ancestor recorded later doesn't change what this
|
|
67
|
+
/** Resolved once, so a nearer ancestor recorded later doesn't change what this analyzed. */
|
|
159
68
|
base: string | null;
|
|
160
69
|
/** An eval filter (substring of the eval id), or null for every eval. */
|
|
161
70
|
evals: string | null;
|
|
@@ -177,7 +86,7 @@ export type AnalysisDoc = {
|
|
|
177
86
|
promptVersion: string;
|
|
178
87
|
model: string;
|
|
179
88
|
resolvedModel: string | null;
|
|
180
|
-
route: 'direct' | 'gateway' | 'fake';
|
|
89
|
+
route: 'direct' | 'gateway' | 'openrouter' | 'vertex' | 'fake';
|
|
181
90
|
usage: {
|
|
182
91
|
inputTokens: number | null;
|
|
183
92
|
outputTokens: number | null;
|
|
@@ -193,7 +102,7 @@ export type AnalysisDoc = {
|
|
|
193
102
|
evalSlug: string;
|
|
194
103
|
attempt: string;
|
|
195
104
|
}>;
|
|
196
|
-
/** Each cited fact or evidence id,
|
|
105
|
+
/** Each cited fact or evidence id, summarized in one line. */
|
|
197
106
|
cited: Record<string, string>;
|
|
198
107
|
/**
|
|
199
108
|
* Why the analysis failed; its `analysis` is then empty. `reeval analyze` treats a cached one as
|
|
@@ -247,14 +156,27 @@ export declare function sortAnalyses(entries: AnalysisEntry[]): AnalysisEntry[];
|
|
|
247
156
|
export declare const isSuperseded: (doc: AnalysisDoc) => boolean;
|
|
248
157
|
//#endregion
|
|
249
158
|
//#region src/view/compare.d.ts
|
|
250
|
-
export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'drift' | 'added' | 'removed' | 'infra';
|
|
159
|
+
export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'gate' | 'drift' | 'added' | 'removed' | 'infra';
|
|
251
160
|
export type CompareRow = {
|
|
252
161
|
id: string;
|
|
253
162
|
status: CompareStatus;
|
|
254
163
|
base: Outcome | null;
|
|
255
164
|
head: Outcome | null;
|
|
256
|
-
/**
|
|
165
|
+
/**
|
|
166
|
+
* For `drift`: what changed, e.g. `body changed`, `revision 2 → 3` or `harness config changed`. On
|
|
167
|
+
* any other row, `gate changed` when the cells' gate did.
|
|
168
|
+
*/
|
|
257
169
|
changes?: string[];
|
|
170
|
+
/**
|
|
171
|
+
* When the cells' gate changed but nothing else did: each side's gate, and base's outcome with
|
|
172
|
+
* its recorded attempts scored under head's gate. `status` compares that with head, so a flip
|
|
173
|
+
* the gate alone explains is `gate`, not `regressed` or `fixed`.
|
|
174
|
+
*/
|
|
175
|
+
gate?: {
|
|
176
|
+
base: string;
|
|
177
|
+
head: string;
|
|
178
|
+
rescored: Outcome;
|
|
179
|
+
};
|
|
258
180
|
/**
|
|
259
181
|
* When the runs ran different cells of the eval: the cells only head or only base has. Then
|
|
260
182
|
* `status`, `base` and `head` are over the cells both have.
|
|
@@ -266,10 +188,24 @@ export type CompareRow = {
|
|
|
266
188
|
/** Cells in both runs, with each side's lift (variants) and effort. */
|
|
267
189
|
perCell?: CompareCell[];
|
|
268
190
|
/**
|
|
269
|
-
* The sides whose `fail` is only attempts the run's interrupt
|
|
270
|
-
* reads as `
|
|
191
|
+
* The sides whose `fail` is only attempts the run's interrupt canceled, which `reeval show`
|
|
192
|
+
* reads as `canceled`. Such an eval is never `regressed` or `fixed`: it's `drift`.
|
|
271
193
|
*/
|
|
272
|
-
|
|
194
|
+
canceled?: Array<'base' | 'head'>;
|
|
195
|
+
/**
|
|
196
|
+
* `family`: a pooled `evaluate.each`'s gate, its `id` the family's. `base` and `head` are its
|
|
197
|
+
* pooled outcome (`fail` when a lift is below its `minLift`); its cases have rows of their own.
|
|
198
|
+
* For such a row, `cells` are the variants whose pooled lift only one run has.
|
|
199
|
+
*/
|
|
200
|
+
kind?: 'family';
|
|
201
|
+
/** On a family row: each variant's pooled lift on each side, null where that run has none. */
|
|
202
|
+
lifts?: CompareFamilyLift[];
|
|
203
|
+
};
|
|
204
|
+
export type CompareFamilyLift = {
|
|
205
|
+
key: string;
|
|
206
|
+
vs: string;
|
|
207
|
+
base: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
|
|
208
|
+
head: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
|
|
273
209
|
};
|
|
274
210
|
export type CompareCell = {
|
|
275
211
|
key: string;
|
|
@@ -303,17 +239,114 @@ export declare function stackChanges(base: StackSource, head: StackSource): Stac
|
|
|
303
239
|
* A cell `seen` doesn't have yet is new coverage, not a change.
|
|
304
240
|
*/
|
|
305
241
|
export declare function cellsChanged(seen: Map<string, string>, evalId: string, cells: ReadonlyArray<Pick<HistoryCell, 'key' | 'fingerprint'>>): boolean;
|
|
306
|
-
type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'cells'>;
|
|
307
|
-
type Comparable = Pick<HistoryEntry, 'stack'> & {
|
|
242
|
+
type ComparableEval = Pick<HistoryEntry['evals'][number], 'id' | 'fingerprint' | 'outcome' | 'sources' | 'source' | 'revision' | 'cells'>;
|
|
243
|
+
type Comparable = Pick<HistoryEntry, 'stack' | 'families'> & {
|
|
308
244
|
evals: ComparableEval[];
|
|
309
245
|
};
|
|
310
246
|
/** Which source files were added, removed or changed. */
|
|
311
247
|
export declare function sourceChanges(before: ComparableEval['sources'], after: ComparableEval['sources']): string[];
|
|
312
|
-
/**
|
|
248
|
+
/**
|
|
249
|
+
* Why two runs of an eval aren't comparable: what in its source differs (`body`, `options`,
|
|
250
|
+
* `case data`, `revision 2 → 3`; a record from before `source`, its files), then the fingerprint
|
|
251
|
+
* parts that differ. Each reason but the revision ends in ` changed`.
|
|
252
|
+
*/
|
|
313
253
|
export declare function whyChanged(base: ComparableEval, head: ComparableEval): string[];
|
|
314
254
|
/** An eval only head has. */
|
|
315
255
|
export declare const addedRow: (h: ComparableEval) => CompareRow;
|
|
256
|
+
/** The rows of a run with nothing to compare it with: every eval and gated family is new. */
|
|
257
|
+
export declare const newRows: (head: Comparable) => CompareRow[];
|
|
258
|
+
/** Each eval's row, then each family's that either run gated. */
|
|
316
259
|
export declare function compare(base: Comparable, head: Comparable): CompareRow[];
|
|
260
|
+
/** The key that tells a family's row from an eval's, whose ids can coincide. */
|
|
261
|
+
export declare const rowKey: (row: Pick<CompareRow, 'id' | 'kind'>) => string;
|
|
262
|
+
//#endregion
|
|
263
|
+
//#region src/view/debug.d.ts
|
|
264
|
+
/**
|
|
265
|
+
* An attempt's local debug data, as `reeval run` writes it to `.reeval/debug/<runId>/<eval>/<cell>/<n>/`
|
|
266
|
+
* and `reeval studio` serves it. Browser-safe. Never part of a record or the history folder.
|
|
267
|
+
*
|
|
268
|
+
* debug.log the `-v` trace, one line per event, always written
|
|
269
|
+
* requests.ndjson one line per egress gateway request, always written
|
|
270
|
+
* trace.ndjson raw payloads (`-vv`), written when the attempt fails or with `--trace`
|
|
271
|
+
*/
|
|
272
|
+
export declare const DEBUG_LOG = "debug.log";
|
|
273
|
+
export declare const REQUESTS_FILE = "requests.ndjson";
|
|
274
|
+
export declare const TRACE_FILE = "trace.ndjson";
|
|
275
|
+
/** What part of the attempt a line is about, in the order an attempt goes through them. */
|
|
276
|
+
export declare const DEBUG_PHASES: readonly ['plan', 'image', 'container', 'env', 'broker', 'harness', 'check', 'judge', 'gateway', 'teardown', 'docker'];
|
|
277
|
+
export type DebugPhase = (typeof DEBUG_PHASES)[number];
|
|
278
|
+
/** One debug line: level 1 is `-v`, level 2 is `-vv` only. */
|
|
279
|
+
export type DebugLine = {
|
|
280
|
+
at: number;
|
|
281
|
+
level: 1 | 2;
|
|
282
|
+
phase: DebugPhase;
|
|
283
|
+
text: string;
|
|
284
|
+
/** More lines under it, e.g. a docker command's output or a raw payload. */
|
|
285
|
+
detail?: string;
|
|
286
|
+
};
|
|
287
|
+
/** One raw trace entry: `ms` is since the attempt started. */
|
|
288
|
+
export type TraceEntry = {
|
|
289
|
+
at: string;
|
|
290
|
+
ms: number;
|
|
291
|
+
phase: DebugPhase;
|
|
292
|
+
kind: string;
|
|
293
|
+
data: unknown;
|
|
294
|
+
};
|
|
295
|
+
/** One request through the egress gateway, or one tunnel it passed through without reading. */
|
|
296
|
+
export type DebugRequest = {
|
|
297
|
+
at: string;
|
|
298
|
+
method: string;
|
|
299
|
+
url: string;
|
|
300
|
+
host: string;
|
|
301
|
+
port: number;
|
|
302
|
+
via: 'tunnel' | 'decrypted' | 'http';
|
|
303
|
+
allowed: boolean;
|
|
304
|
+
status?: number;
|
|
305
|
+
reason?: string;
|
|
306
|
+
sent: number;
|
|
307
|
+
received: number;
|
|
308
|
+
/** Brokered secrets and the harness credential swapped in, by name. */
|
|
309
|
+
substituted: string[];
|
|
310
|
+
ms: number;
|
|
311
|
+
/**
|
|
312
|
+
* On a request the gateway sent the harness's key on, or a model call whose response it reads,
|
|
313
|
+
* the reasoning settings its JSON body carried, as
|
|
314
|
+
* `<path>=<value>` joined by spaces: `reasoning_effort=high`,
|
|
315
|
+
* `generationConfig.thinkingConfig.thinkingLevel=LOW`.
|
|
316
|
+
*/
|
|
317
|
+
requestEffort?: string;
|
|
318
|
+
};
|
|
319
|
+
/** What the studio's Debug tab shows for one attempt. */
|
|
320
|
+
export type DebugView = {
|
|
321
|
+
lines: Array<{
|
|
322
|
+
at: string;
|
|
323
|
+
phase: string;
|
|
324
|
+
text: string;
|
|
325
|
+
}>;
|
|
326
|
+
requests: DebugRequest[];
|
|
327
|
+
/** Whether `trace.ndjson` was kept. */
|
|
328
|
+
trace: boolean;
|
|
329
|
+
};
|
|
330
|
+
/**
|
|
331
|
+
* The studio server's debug API (`reeval studio` only): an attempt's DebugView (null when it has no
|
|
332
|
+
* local data), and its raw trace.
|
|
333
|
+
*/
|
|
334
|
+
export declare const debugApiPath: (runId: string, attemptKey: string) => string;
|
|
335
|
+
export declare const traceApiPath: (runId: string, attemptKey: string) => string;
|
|
336
|
+
/** `12:50:01`, or `12:50:01.234` with milliseconds, in UTC like every reeval timestamp. */
|
|
337
|
+
export declare const debugTime: (at: number, ms: boolean) => string;
|
|
338
|
+
/** A line of debug.log: `2026-09-26T12:50:01.234Z plan text`. */
|
|
339
|
+
export declare const debugLogLine: (line: Pick<DebugLine, 'at' | 'phase' | 'text'>) => string;
|
|
340
|
+
/** debug.log's lines; a line that isn't one (a wrapped message) continues the one before. */
|
|
341
|
+
export declare function parseDebugLog(text: string): DebugView['lines'];
|
|
342
|
+
export declare function parseNdjson<T>(text: string): T[];
|
|
343
|
+
/** `512 B`, `1.2 KB`, `3.4 MB`. */
|
|
344
|
+
export declare function formatBytes(n: number): string;
|
|
345
|
+
/**
|
|
346
|
+
* The gateway line for a request: method, URL, status, allowed or blocked and why, bytes each way,
|
|
347
|
+
* time, and which secrets were swapped in (by name).
|
|
348
|
+
*/
|
|
349
|
+
export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
|
|
317
350
|
//#endregion
|
|
318
351
|
//#region src/view/family.d.ts
|
|
319
352
|
/** `pass@1 2/3 · 67% (95% 21…94)`, or `pass@1 67% over 3 cases (95% 30…100)` at several runs. */
|
|
@@ -342,7 +375,7 @@ export declare const familyCases: (family: Pick<FamilyRecord, 'id'>, evals: read
|
|
|
342
375
|
/** The derived header of one run: what listings, indexes and comparisons need without the record. */
|
|
343
376
|
export declare function runHeader(record: RunRecord, recorded: boolean): HistoryEntry;
|
|
344
377
|
//#endregion
|
|
345
|
-
//#region src/view/
|
|
378
|
+
//#region src/view/judgments.d.ts
|
|
346
379
|
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
347
380
|
export declare function judgeLabel(question: string): string;
|
|
348
381
|
/**
|
|
@@ -369,7 +402,7 @@ export type JudgeKeyRate = {
|
|
|
369
402
|
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides qualify. */
|
|
370
403
|
export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
|
|
371
404
|
/**
|
|
372
|
-
* The judge rates a cell's summary line adds to its checks: a
|
|
405
|
+
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
373
406
|
* of the cell's checks against its control is already there.
|
|
374
407
|
*/
|
|
375
408
|
export declare function summaryJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cell: Pick<CellRecord, 'key' | 'checks'>): JudgeKeyRate[];
|
|
@@ -383,20 +416,20 @@ export type CollapsedJudge = {
|
|
|
383
416
|
key: string;
|
|
384
417
|
};
|
|
385
418
|
/** Each key, or the lowest score plus how many judges there were. */
|
|
386
|
-
export declare function collapsedJudges(
|
|
419
|
+
export declare function collapsedJudges(judgments: readonly JudgmentRecord[]): {
|
|
387
420
|
shown: CollapsedJudge[];
|
|
388
421
|
extra: number;
|
|
389
422
|
};
|
|
390
423
|
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
391
|
-
export declare function collapsedJudgeLabel(
|
|
424
|
+
export declare function collapsedJudgeLabel(judgments: readonly JudgmentRecord[]): string | null;
|
|
392
425
|
/**
|
|
393
|
-
* Each
|
|
394
|
-
* failed judge call, which recorded no
|
|
395
|
-
* its
|
|
426
|
+
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
427
|
+
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
428
|
+
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
396
429
|
* reaches is named by its question.
|
|
397
430
|
*/
|
|
398
|
-
export declare function
|
|
399
|
-
|
|
431
|
+
export declare function labeledJudgments(a: Pick<AttemptRecord, 'judgements' | 'assertions'>): Array<{
|
|
432
|
+
judgment: JudgmentRecord;
|
|
400
433
|
label: string;
|
|
401
434
|
labelled: boolean;
|
|
402
435
|
check?: string;
|
|
@@ -457,7 +490,7 @@ export type LiveCounts = {
|
|
|
457
490
|
skipped: number;
|
|
458
491
|
running: number;
|
|
459
492
|
};
|
|
460
|
-
/** What the run is doing: running attempts,
|
|
493
|
+
/** What the run is doing: running attempts, canceling them, or assembling its record. */
|
|
461
494
|
export type LivePhase = 'running' | 'cancelling' | 'finishing';
|
|
462
495
|
export type LiveProgress = {
|
|
463
496
|
schemaVersion: 1;
|
|
@@ -741,11 +774,15 @@ export type CompareViewRow = {
|
|
|
741
774
|
/** See `CompareRow`: then pass@1 is over the cells both runs have too. */
|
|
742
775
|
cells?: CompareRow['cells'];
|
|
743
776
|
changes?: string[];
|
|
744
|
-
|
|
777
|
+
gate?: CompareRow['gate'];
|
|
778
|
+
/** Mean judge score per run; null without judgments or without the record. */
|
|
745
779
|
baseScore: number | null;
|
|
746
780
|
headScore: number | null;
|
|
747
781
|
/** Cells in both runs, with lift and effort on each side. */
|
|
748
782
|
perCell: CompareCell[];
|
|
783
|
+
/** See `CompareRow`: a pooled family's gate, with no slug, pass@1 or score of its own. */
|
|
784
|
+
kind?: 'family';
|
|
785
|
+
lifts?: CompareRow['lifts'];
|
|
749
786
|
};
|
|
750
787
|
export declare const COMPARE_ORDER: CompareStatus[];
|
|
751
788
|
/**
|
|
@@ -814,6 +851,12 @@ export type TranscriptItem = {
|
|
|
814
851
|
/** Parses `transcript.jsonl`, skipping lines that aren't JSON (a truncated tail, say). */
|
|
815
852
|
export declare function parseTranscript(text: string): TranscriptLine[];
|
|
816
853
|
export declare function transcriptItems(lines: TranscriptLine[]): TranscriptItem[];
|
|
854
|
+
export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
|
|
855
|
+
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
856
|
+
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
857
|
+
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
858
|
+
//#endregion
|
|
859
|
+
//#region src/view/patch.d.ts
|
|
817
860
|
export type PatchFile = {
|
|
818
861
|
path: string;
|
|
819
862
|
status: 'added' | 'deleted' | 'modified' | 'renamed' | 'binary';
|
|
@@ -821,12 +864,13 @@ export type PatchFile = {
|
|
|
821
864
|
removed: number;
|
|
822
865
|
text: string;
|
|
823
866
|
};
|
|
867
|
+
/**
|
|
868
|
+
* The path one file's part of a git patch changes: the new path, or the old one for a deletion.
|
|
869
|
+
* Read from its header lines, before the first `@@`, where git C-quotes an awkward name.
|
|
870
|
+
*/
|
|
871
|
+
export declare function patchPath(part: string): string | null;
|
|
824
872
|
/** Splits a git patch into one entry per file, with line counts. */
|
|
825
873
|
export declare function splitPatch(patch: string): PatchFile[];
|
|
826
|
-
export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
|
|
827
|
-
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
828
|
-
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
829
|
-
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
830
874
|
//#endregion
|
|
831
875
|
//#region src/view/paths.d.ts
|
|
832
876
|
/**
|
|
@@ -875,6 +919,57 @@ export declare function parseDataPath(path: string): DataTarget | null;
|
|
|
875
919
|
/** Artifacts are untrusted agent output: JSON stays JSON, PNGs are images, else plain text. */
|
|
876
920
|
export declare const artifactType: (name: string) => string;
|
|
877
921
|
//#endregion
|
|
922
|
+
//#region src/view/remarks.d.ts
|
|
923
|
+
/** `<history>/<run>/remarks.json`, beside the record. */
|
|
924
|
+
export declare const REMARKS_FILE = "remarks.json";
|
|
925
|
+
/** A run's remarks, `<history>/<run>/remarks.json`. Written only when the run raised one. */
|
|
926
|
+
export type RunRemarks = {
|
|
927
|
+
kind: 'reeval-remarks';
|
|
928
|
+
schemaVersion: 1;
|
|
929
|
+
runId: string;
|
|
930
|
+
createdAt: string;
|
|
931
|
+
/**
|
|
932
|
+
* Every remark this run raised, ranked, including those an earlier run printed first
|
|
933
|
+
* (`firstSeenRun`), which don't print again.
|
|
934
|
+
*/
|
|
935
|
+
remarks: Remark[];
|
|
936
|
+
};
|
|
937
|
+
/** Each remark kind's severity. */
|
|
938
|
+
export declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
939
|
+
severity: 'concern' | 'suggestion';
|
|
940
|
+
}>;
|
|
941
|
+
/** A remark about the eval's quality, not about the agent. */
|
|
942
|
+
export type Remark = {
|
|
943
|
+
/**
|
|
944
|
+
* sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
|
|
945
|
+
* per kind (and server) per eval version.
|
|
946
|
+
*/
|
|
947
|
+
id: string;
|
|
948
|
+
evalId: string;
|
|
949
|
+
kind: 'too-few-tries' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret';
|
|
950
|
+
/** Fixed by kind (`REMARK_KINDS`). */
|
|
951
|
+
severity: 'concern' | 'suggestion';
|
|
952
|
+
/** One plain sentence: what's wrong. */
|
|
953
|
+
text: string;
|
|
954
|
+
/** Facts from this run (or history facts) showing it mattered now. */
|
|
955
|
+
evidence: string[];
|
|
956
|
+
/** An edit at a source location (`source:<file>#L<n>`) in the eval. */
|
|
957
|
+
fix: {
|
|
958
|
+
at: string;
|
|
959
|
+
edit: string;
|
|
960
|
+
};
|
|
961
|
+
/**
|
|
962
|
+
* The first run that printed this `id`, or this run when none has: it prints only in that run,
|
|
963
|
+
* except the kinds about the environment, which print in every run they happen.
|
|
964
|
+
*/
|
|
965
|
+
firstSeenRun: string;
|
|
966
|
+
/**
|
|
967
|
+
* Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
|
|
968
|
+
* when the cap or a run without `-v` left it out, so a later run prints it.
|
|
969
|
+
*/
|
|
970
|
+
printed?: boolean;
|
|
971
|
+
};
|
|
972
|
+
//#endregion
|
|
878
973
|
//#region src/view/record.d.ts
|
|
879
974
|
/** A file from a newer reeval, whose schemaVersion this reeval doesn't know. */
|
|
880
975
|
export declare class NewerSchemaError extends Error {
|
|
@@ -893,8 +988,328 @@ export declare function readableRecord(record: RunRecord, what?: string): RunRec
|
|
|
893
988
|
export declare const readableTranscriptLine: (line: TranscriptLine) => TranscriptLine;
|
|
894
989
|
/** An analysis document as every reader sees it; one from a newer reeval throws. */
|
|
895
990
|
export declare const readableAnalysis: (doc: AnalysisDoc, what?: string) => AnalysisDoc;
|
|
896
|
-
/** A run's
|
|
897
|
-
export declare const
|
|
991
|
+
/** A run's remarks as every reader sees them; a file from a newer reeval throws. */
|
|
992
|
+
export declare const readableRemarks: (doc: RunRemarks, what?: string) => RunRemarks;
|
|
993
|
+
//#endregion
|
|
994
|
+
//#region src/stats.d.ts
|
|
995
|
+
/**
|
|
996
|
+
* Counted attempts a side before a pass-rate lift can be called: below it, every view says
|
|
997
|
+
* `insufficient` (`can't tell yet`, `unclear`), whatever the interval. A product floor that keeps
|
|
998
|
+
* `reeval run`, `reeval query`, the judge-key lift and `reeval analyze` agreeing, not one that
|
|
999
|
+
* makes the stated 95% exact: at 4 a side, a 4/4 against 0/4 split happens by chance about 7% of
|
|
1000
|
+
* the time when nothing changed.
|
|
1001
|
+
*/
|
|
1002
|
+
export declare const EFFECT_MIN_COUNTED = 4;
|
|
1003
|
+
//#endregion
|
|
1004
|
+
//#region src/view/suite.d.ts
|
|
1005
|
+
/**
|
|
1006
|
+
* A variant against its baseline: `better` or `worse` when the stored 95% interval on the
|
|
1007
|
+
* difference excludes 0, `same` when both pass at the same rate, and `unclear` otherwise,
|
|
1008
|
+
* including whenever either side has fewer than `EFFECT_MIN_COUNTED` counted attempts.
|
|
1009
|
+
*/
|
|
1010
|
+
export type Effect = 'better' | 'worse' | 'same' | 'unclear';
|
|
1011
|
+
/**
|
|
1012
|
+
* What an eval's cells compare: `change` when it has variants (or several environments),
|
|
1013
|
+
* `harness` when it has several harnesses, `both`, or `none`.
|
|
1014
|
+
*/
|
|
1015
|
+
export type Across = 'none' | 'change' | 'harness' | 'both';
|
|
1016
|
+
/**
|
|
1017
|
+
* One finished run's facts. Every node has a `path`, a stable id that remarks cite it by.
|
|
1018
|
+
*
|
|
1019
|
+
* A path is `/`-separated segments, each with `%`, `/` and `+` percent-encoded:
|
|
1020
|
+
*
|
|
1021
|
+
* - `<eval>`: the eval's id (`evals/a.eval.ts › group › name`) or a family's
|
|
1022
|
+
* (`evals/a.eval.ts › fixes $id`);
|
|
1023
|
+
* - `<eval>/<case>`: a family case, by its label (`GH-101` for `fixes GH-101`);
|
|
1024
|
+
* - `<eval>[/<case>]/<harness>+<environment>+<sandbox>`: a cell;
|
|
1025
|
+
* - `<cell>/checks/<label>`: a check, by its assertion label;
|
|
1026
|
+
* - `<eval>/<harness>+<environment>+<sandbox>`: on a family, a variant's pooled effect.
|
|
1027
|
+
*
|
|
1028
|
+
* A case segment never has a raw `+` and a cell segment always has two, so the two can't be
|
|
1029
|
+
* confused. Ids, labels and cell keys are what runs are matched on, so a path names the same
|
|
1030
|
+
* fact in every run that has it.
|
|
1031
|
+
*/
|
|
1032
|
+
export type SuiteResult = {
|
|
1033
|
+
runId: string;
|
|
1034
|
+
status: RunRecord['status'];
|
|
1035
|
+
/**
|
|
1036
|
+
* The run's outcome, which `reeval run`'s exit code is read from: `fail` on any error outside
|
|
1037
|
+
* an attempt, and `skipped` on an interrupted run.
|
|
1038
|
+
*/
|
|
1039
|
+
outcome: Outcome;
|
|
1040
|
+
/**
|
|
1041
|
+
* Errors outside any attempt: an eval file that didn't load, a rejection or throw no attempt
|
|
1042
|
+
* awaited, a worker that died. Any one fails the run.
|
|
1043
|
+
*/
|
|
1044
|
+
errors: string[];
|
|
1045
|
+
/**
|
|
1046
|
+
* Outcomes counted per block: each family case is one, each eval with no cases is one, and a
|
|
1047
|
+
* family whose pooled gate failed is one more fail. `errors` counts the run's `errors`, and
|
|
1048
|
+
* `notGated` the passes whose cells are all `gate: 'none'` (studies), which `pass` leaves out.
|
|
1049
|
+
*/
|
|
1050
|
+
tally: SuiteTally;
|
|
1051
|
+
/** Sorted by path. */
|
|
1052
|
+
files: ResultFile[];
|
|
1053
|
+
};
|
|
1054
|
+
export type SuiteTally = {
|
|
1055
|
+
errors: number;
|
|
1056
|
+
pass: number;
|
|
1057
|
+
notGated: number;
|
|
1058
|
+
fail: number;
|
|
1059
|
+
infraError: number;
|
|
1060
|
+
canceled: number;
|
|
1061
|
+
skipped: number;
|
|
1062
|
+
};
|
|
1063
|
+
/**
|
|
1064
|
+
* An eval's or a family case's outcome: the record's, except `canceled` where its only failed
|
|
1065
|
+
* gated attempts are ones the run's interrupt canceled, since the agent didn't fail those.
|
|
1066
|
+
*/
|
|
1067
|
+
export type ResultOutcome = Outcome | 'canceled';
|
|
1068
|
+
export type ResultFile = {
|
|
1069
|
+
path: string;
|
|
1070
|
+
evals: EvalResult[];
|
|
1071
|
+
};
|
|
1072
|
+
export type EvalResult = {
|
|
1073
|
+
path: string;
|
|
1074
|
+
/** The eval's id, or the family's. */
|
|
1075
|
+
id: string;
|
|
1076
|
+
group: string[];
|
|
1077
|
+
/** The eval's name; on a family, its template. */
|
|
1078
|
+
name: string;
|
|
1079
|
+
/** A family's name template (`fixes $id`); null on a single eval. */
|
|
1080
|
+
template: string | null;
|
|
1081
|
+
/** Source order in the file. */
|
|
1082
|
+
order: number;
|
|
1083
|
+
across: Across;
|
|
1084
|
+
/**
|
|
1085
|
+
* A pooled `evaluate.each` with two or more cases: `cases` is set and `cells` is null. An
|
|
1086
|
+
* unpooled one (`{ pool: false }`) is recorded as separate evals, each a single eval.
|
|
1087
|
+
*/
|
|
1088
|
+
family: boolean;
|
|
1089
|
+
/** A placeholder (`evaluate.todo`) that never ran. */
|
|
1090
|
+
todo: boolean;
|
|
1091
|
+
/** On a family: `fail` when a case failed or its pooled gate did. */
|
|
1092
|
+
outcome: ResultOutcome;
|
|
1093
|
+
cells: CellResult[] | null;
|
|
1094
|
+
cases: CaseResult[] | null;
|
|
1095
|
+
/** A family's variants against their baseline, pooled over the cases; null on a single eval. */
|
|
1096
|
+
pooled: PooledArm[] | null;
|
|
1097
|
+
/**
|
|
1098
|
+
* Its questions' answers: the pairwise ones, then those asked of each attempt. On a family,
|
|
1099
|
+
* each question asked of any case, in first-asked order, summed over the cases that answered it.
|
|
1100
|
+
*/
|
|
1101
|
+
questions: Array<QuestionResult | AttemptQuestionResult>;
|
|
1102
|
+
};
|
|
1103
|
+
/** A pairwise question: each side's wins over the pairs the model decided. Never gates. */
|
|
1104
|
+
export type QuestionResult = {
|
|
1105
|
+
kind: 'pairs';
|
|
1106
|
+
question: string;
|
|
1107
|
+
/** Pairs voted, each counted once per order it was asked in. */
|
|
1108
|
+
votes: number;
|
|
1109
|
+
rates: Array<{
|
|
1110
|
+
cut: 'within' | 'across';
|
|
1111
|
+
name: string;
|
|
1112
|
+
wins: number;
|
|
1113
|
+
n: number;
|
|
1114
|
+
}>;
|
|
1115
|
+
/** Why some or all of its votes are missing: the model call failed. */
|
|
1116
|
+
error: string | null;
|
|
1117
|
+
};
|
|
1118
|
+
/** A question asked of each attempt on its own: what each one answered. Never gates. */
|
|
1119
|
+
export type AttemptQuestionResult = {
|
|
1120
|
+
kind: 'attempts';
|
|
1121
|
+
question: string;
|
|
1122
|
+
/** Attempts it was asked of, failed calls included. */
|
|
1123
|
+
asked: number;
|
|
1124
|
+
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
1125
|
+
choices: Array<{
|
|
1126
|
+
choice: string;
|
|
1127
|
+
n: number;
|
|
1128
|
+
}> | null;
|
|
1129
|
+
/** Each attempt's answer, in attempt order; failed calls left out. */
|
|
1130
|
+
answers: Array<{
|
|
1131
|
+
attempt: string;
|
|
1132
|
+
choice: string | null;
|
|
1133
|
+
text: string;
|
|
1134
|
+
}>;
|
|
1135
|
+
/** Calls that failed, and the first one's error. */
|
|
1136
|
+
failed: number;
|
|
1137
|
+
error: string | null;
|
|
1138
|
+
};
|
|
1139
|
+
export type CaseResult = {
|
|
1140
|
+
path: string;
|
|
1141
|
+
/** The case's own eval id. */
|
|
1142
|
+
id: string;
|
|
1143
|
+
/** Its name less the template's literal text: `GH-101` for `fixes GH-101`. */
|
|
1144
|
+
label: string;
|
|
1145
|
+
name: string;
|
|
1146
|
+
/** Its place in the case list. */
|
|
1147
|
+
order: number;
|
|
1148
|
+
outcome: ResultOutcome;
|
|
1149
|
+
cells: CellResult[];
|
|
1150
|
+
};
|
|
1151
|
+
export type CellResult = {
|
|
1152
|
+
path: string;
|
|
1153
|
+
key: string;
|
|
1154
|
+
harness: string;
|
|
1155
|
+
/** `-` when no environment is set. */
|
|
1156
|
+
environment: string;
|
|
1157
|
+
sandbox: string;
|
|
1158
|
+
/** The variant's name; null on a control. */
|
|
1159
|
+
variant: string | null;
|
|
1160
|
+
/** The control of an eval that has variants: the arm the variants are measured against. */
|
|
1161
|
+
baseline: boolean;
|
|
1162
|
+
/** On a variant, its baseline cell's key; else null. */
|
|
1163
|
+
vs: string | null;
|
|
1164
|
+
/** The cell's place in the eval's cells. */
|
|
1165
|
+
order: number;
|
|
1166
|
+
/** Its failures fail the eval (its gate isn't `'none'`). */
|
|
1167
|
+
gated: boolean;
|
|
1168
|
+
outcome: Outcome;
|
|
1169
|
+
/** Every attempt: `passed + failed + infraError + skipped`. */
|
|
1170
|
+
attempts: number;
|
|
1171
|
+
passed: number;
|
|
1172
|
+
failed: number;
|
|
1173
|
+
/** Infrastructure errors, which are neither passes nor fails. */
|
|
1174
|
+
infraError: number;
|
|
1175
|
+
skipped: number;
|
|
1176
|
+
fingerprint: string;
|
|
1177
|
+
/** The skill sources and files the cell's environment placed, as written. */
|
|
1178
|
+
context: string[];
|
|
1179
|
+
/**
|
|
1180
|
+
* Every check the eval's attempts recorded (on a family, any case's). A cell with no counted
|
|
1181
|
+
* attempt (every one an infra error or skipped) lists each as `not run`.
|
|
1182
|
+
*/
|
|
1183
|
+
checks: CheckResult[];
|
|
1184
|
+
/** On a variant, against its baseline; null on a control. */
|
|
1185
|
+
effect: Effect | null;
|
|
1186
|
+
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1187
|
+
interval: Interval | null;
|
|
1188
|
+
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1189
|
+
minLift: number | null;
|
|
1190
|
+
/** Medians over the counted attempts; null below `EFFECT_MIN_COUNTED` of them. */
|
|
1191
|
+
effort: EffortCounters | null;
|
|
1192
|
+
/** On a variant, its tool calls, agent time or tokens differ from its baseline's by the rank test. */
|
|
1193
|
+
effortDiffers: boolean;
|
|
1194
|
+
/** Why attempts failed or errored where no check says: a timeout, an infra error, a throw. */
|
|
1195
|
+
errors: CellError[];
|
|
1196
|
+
/** Web searches and fetches per counted attempt; null where the harness can't report them. */
|
|
1197
|
+
searches: Array<number | null>;
|
|
1198
|
+
fetches: Array<number | null>;
|
|
1199
|
+
/**
|
|
1200
|
+
* The upstream providers a gateway (OpenRouter) said served the cell's agent calls, with how
|
|
1201
|
+
* many, summed over its attempts in first-seen order; absent when it named none.
|
|
1202
|
+
*/
|
|
1203
|
+
upstreams?: UpstreamCount[];
|
|
1204
|
+
/** The same for the cell's judge calls; absent when the gateway named none. */
|
|
1205
|
+
judgeUpstreams?: UpstreamCount[];
|
|
1206
|
+
};
|
|
1207
|
+
/** A cell's attempts the run's interrupt canceled: recorded as fails, but not the agent's. */
|
|
1208
|
+
export declare const canceledOf: (cell: CellResult) => number;
|
|
1209
|
+
/** A cell's counted tries: its passes and fails, less the ones the interrupt canceled. */
|
|
1210
|
+
export declare const countedOf: (cell: CellResult) => number;
|
|
1211
|
+
export type CheckResult = {
|
|
1212
|
+
path: string;
|
|
1213
|
+
label: string;
|
|
1214
|
+
/** First-seen order across the eval's attempts. */
|
|
1215
|
+
order: number;
|
|
1216
|
+
outcome: CheckOutcome;
|
|
1217
|
+
/**
|
|
1218
|
+
* On a variant, this check against the baseline's (`unclear` when the baseline never ran it);
|
|
1219
|
+
* null on a control or without one.
|
|
1220
|
+
*/
|
|
1221
|
+
effect: Effect | null;
|
|
1222
|
+
/** Set when the check is a judge verdict. */
|
|
1223
|
+
judge: JudgeResult | null;
|
|
1224
|
+
/**
|
|
1225
|
+
* A `toSucceed` check whose `c.exec` had `retries` and needed them: on one counted attempt how
|
|
1226
|
+
* it ended (`passed on try 2 of 3`, `failed all 3 tries`), on several how many retried. Unset
|
|
1227
|
+
* when none did.
|
|
1228
|
+
*/
|
|
1229
|
+
retried?: string;
|
|
1230
|
+
};
|
|
1231
|
+
/**
|
|
1232
|
+
* `not run`: no counted attempt reached the check, because a hard `expect` before it failed or
|
|
1233
|
+
* none counted. That is not a failure, and is never `0 of N`. `ran`: counts over the counted attempts that
|
|
1234
|
+
* reached it (`errored` ones couldn't answer), with `notRun` for those that didn't, and the
|
|
1235
|
+
* first failure's message.
|
|
1236
|
+
*/
|
|
1237
|
+
export type CheckOutcome = {
|
|
1238
|
+
status: 'not run';
|
|
1239
|
+
} | {
|
|
1240
|
+
status: 'ran';
|
|
1241
|
+
passed: number;
|
|
1242
|
+
failed: number;
|
|
1243
|
+
errored: number;
|
|
1244
|
+
notRun: number;
|
|
1245
|
+
assertion: string | null;
|
|
1246
|
+
};
|
|
1247
|
+
/**
|
|
1248
|
+
* One distinct error over a cell's attempts: its kind (`turn` for a turn the harness ended with
|
|
1249
|
+
* an error), its message's first line, and how often.
|
|
1250
|
+
*/
|
|
1251
|
+
export type CellError = {
|
|
1252
|
+
kind: ErrorKind | 'turn';
|
|
1253
|
+
message: string;
|
|
1254
|
+
attempts: number;
|
|
1255
|
+
};
|
|
1256
|
+
export type JudgeResult = {
|
|
1257
|
+
question: string;
|
|
1258
|
+
threshold: number;
|
|
1259
|
+
/** One per counted attempt that judged it. */
|
|
1260
|
+
scores: number[];
|
|
1261
|
+
};
|
|
1262
|
+
export type Interval = {
|
|
1263
|
+
low: number;
|
|
1264
|
+
high: number;
|
|
1265
|
+
};
|
|
1266
|
+
/** A null counter is one the harness didn't record; a recorded zero stays zero. */
|
|
1267
|
+
export type EffortCounters = {
|
|
1268
|
+
tools: number | null;
|
|
1269
|
+
agentMs: number | null;
|
|
1270
|
+
tokens: number | null;
|
|
1271
|
+
};
|
|
1272
|
+
export type PooledArm = {
|
|
1273
|
+
path: string;
|
|
1274
|
+
/** The variant cell's key, the same in every case. */
|
|
1275
|
+
key: string;
|
|
1276
|
+
vs: string;
|
|
1277
|
+
effect: Effect;
|
|
1278
|
+
/** Cases with counted attempts on both sides. */
|
|
1279
|
+
pairs: number;
|
|
1280
|
+
interval: Interval;
|
|
1281
|
+
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
1282
|
+
minLift: number | null;
|
|
1283
|
+
outcome: 'pass' | 'fail';
|
|
1284
|
+
};
|
|
1285
|
+
type Side = {
|
|
1286
|
+
pass: number;
|
|
1287
|
+
counted: number;
|
|
1288
|
+
};
|
|
1289
|
+
/**
|
|
1290
|
+
* The effect of `variant` against `control`, from the interval on their difference. A side
|
|
1291
|
+
* with fewer than `EFFECT_MIN_COUNTED` counted attempts is `unclear`, whatever the interval.
|
|
1292
|
+
*/
|
|
1293
|
+
export declare function effectOf(variant: Side, control: Side, interval: Interval | null): Effect;
|
|
1294
|
+
/** The check's effect, on the same Newcombe interval the record stores for a cell's lift. */
|
|
1295
|
+
export declare function checkEffect(variant: CheckTally, control: CheckTally): Effect;
|
|
1296
|
+
/** One path segment: `%`, `/` and `+` percent-encoded, so segments split back unambiguously. */
|
|
1297
|
+
export declare const factSegment: (s: string) => string;
|
|
1298
|
+
export declare function evalAcross(cells: readonly Pick<CellRecord, 'harness' | 'environment' | 'role'>[]): Across;
|
|
1299
|
+
/**
|
|
1300
|
+
* An error's one line: its first line with any letter or digit in it, joined with the next one
|
|
1301
|
+
* when it ends with `:`, since `Docker failed:` alone says nothing. A log line's `| ` is dropped.
|
|
1302
|
+
*/
|
|
1303
|
+
export declare function messageLine(text: string): string;
|
|
1304
|
+
/** A study: every cell `gate: 'none'`, so it passes whatever it saw and never reads `pass`. */
|
|
1305
|
+
export declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => boolean;
|
|
1306
|
+
/**
|
|
1307
|
+
* A study's pass: every cell `gate: 'none'`, or on a family every case's with no pooled `minLift`
|
|
1308
|
+
* gating it, so it reads `not gated`, never `pass`.
|
|
1309
|
+
*/
|
|
1310
|
+
export declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled'>) => boolean;
|
|
1311
|
+
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1312
|
+
export declare function suiteResult(record: RunRecord): SuiteResult;
|
|
898
1313
|
//#endregion
|
|
899
|
-
export {
|
|
1314
|
+
export { type ArtifactRef, type AttemptRecord, CellEffort, type CellRecord, DIRECTIONAL_MIN, EFFORT_ALPHA, EFFORT_MIN_RUNS, EffortContrast, EffortMetric, EffortSample, EffortVerdict, type EvalEntry, type FamilyLift, type FamilyRate, type FamilyRecord, type FileRef, type HistoryCell, type HistoryEntry, type JudgmentRecord, type Outcome, type RunRecord, type SandboxWarning, Spread, type StackSummary, TOKENS_UNREPORTED, type TranscriptEvent, VERDICT_TEXT, bothUnderATenth, cellContrast, cellEffort, cellSamples, contrastEffort, directional, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, formatCount, formatRatio, formatSeconds, tokenValues };
|
|
900
1315
|
//# sourceMappingURL=index.d.ts.map
|