reeval 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CbqbMR7y.js} +37 -8
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-Zgn-kiqq.js} +57 -15
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-ChbvRx3P.js} +4 -4
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-SKQGW_QF.js} +136 -116
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-BOp8JklD.js} +96 -5
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-BM_3100g.js} +1218 -1207
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-PdYrcoKM.js} +106 -170
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-D0qyrPZi.js} +8 -20
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-Ct6s6OnF.js} +2 -2
- package/dist/{client-D5hNjpEx.js.map → client-Ct6s6OnF.js.map} +1 -1
- package/dist/{command-BXLUfw4h.js → command-4pJYVCT4.js} +33 -20
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-BhwqrVsm.js → command-9xm5ImqI.js} +387 -59
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-BqBxLgTc.js} +31 -30
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-DbSMcxZL.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-DbSMcxZL.js.map} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BU6YJCz9.js} +175 -72
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/{config-BrdHA9bJ.js → config-k6m7XbAz.js} +499 -271
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-DLZF4dGF.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-DLZF4dGF.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-B4-rx4dA.js} +26 -8
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-BuQnfz4e.js} +275 -37
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-BOFgzFDg.js} +115 -296
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-BPP9s3s6.js} +367 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-B-CJeeEh.js → files-Bi461Be2.js} +197 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-kUCrswI6.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +51 -10
- package/dist/index.js +5 -5
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-SRalM5xF.js → init-cOk3D44b.js} +10 -11
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-CKdgEXN7.js} +236 -46
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-D4eUqx4j.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-1ZLr9Dcv.js → model-Bq3G0OG1.js} +29 -36
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-CTnyvh7H.js} +292 -68
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/{model-B45lDUV1.js → model-vRiUycWh.js} +22 -13
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-BSq5qHnQ.js → node-BVbbnFtv.js} +72 -50
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-CHfZNRT6.js → pairs-puYi-zj2.js} +11 -4
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-DdWkM6SW.js} +64 -29
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-D-CP6mMd.js} +253 -17
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-DJCD_BDn.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-D9y-HvZQ.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-CIxCGG0M.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-CemclveB.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-BRhU8MzB.js} +100 -25
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-C7i2LMRb.js} +296 -66
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-dA8LyoPc.js} +9 -5
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DFIzcx_6.js} +39 -99
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-DWSJpKHX.js} +46 -30
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-CMgZEANy.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-d56Gg3J5.js} +30 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-TGW4WyAz.js → server-BOoH10J8.js} +51 -33
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-B1WvBIQt.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-CyaDNG9r.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-D91Hd6hT.js} +546 -1060
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DjRI-IeE.js → src-DgHuStEi.js} +134 -34
- package/dist/{src-DjRI-IeE.js.map → src-DgHuStEi.js.map} +1 -1
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-B1t143wG.js → state-DrJ687ut.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-Co8cwelb.js} +48 -63
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-DLFHVoux.js} +80 -270
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +417 -142
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-coMWHaU2.d.ts} +253 -733
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +536 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-BFO1BLKu.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-BFO1BLKu.js.map} +1 -1
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-CJtdsKBP.js} +19 -12
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +35 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -1,21 +1,24 @@
|
|
|
1
|
+
import { t as RUN_ID } from "./paths-BmR-7tx3.js";
|
|
1
2
|
import { l as sha256 } from "./ids-Bp6ut7ZD.js";
|
|
2
|
-
import "./
|
|
3
|
-
import "./
|
|
4
|
-
import "./
|
|
3
|
+
import { n as clip } from "./text-DTigcA64.js";
|
|
4
|
+
import { t as inCI } from "./env-CARj5Axq.js";
|
|
5
|
+
import "./stats-hfpuYsAM.js";
|
|
6
|
+
import { t as git } from "./git-CsVZkNNL.js";
|
|
7
|
+
import { u as suiteResult } from "./suite-Co8cwelb.js";
|
|
8
|
+
import { n as REMARK_KINDS } from "./remarks-D_3qhBWU.js";
|
|
9
|
+
import { p as scratchDir, u as recordNames } from "./files-Bi461Be2.js";
|
|
10
|
+
import { n as remarksFile, t as readRemarks } from "./files-Clipx4OW.js";
|
|
11
|
+
import { i as comparableFingerprint, s as openHistoryDb } from "./db-CpsifVln.js";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { readFile } from "node:fs/promises";
|
|
5
14
|
/** `KIND_ORDER` ranks remarks of one severity: the ones most worth acting on first. */
|
|
6
15
|
const KIND_ORDER = [
|
|
7
16
|
"flaky-check",
|
|
8
|
-
"judge-too-loose",
|
|
9
|
-
"judge-leaks-answer",
|
|
10
|
-
"name-contradicts",
|
|
11
|
-
"prompt-names-tool",
|
|
12
|
-
"no-baseline",
|
|
13
17
|
"slow-mcp-start",
|
|
14
18
|
"proxy-only-egress",
|
|
19
|
+
"setup-process-secret",
|
|
15
20
|
"too-few-tries",
|
|
16
|
-
"
|
|
17
|
-
"check-never-fails",
|
|
18
|
-
"name-no-intent"
|
|
21
|
+
"check-never-fails"
|
|
19
22
|
];
|
|
20
23
|
/** Remarks ranked: concerns first, then by kind, then in the order they came. */
|
|
21
24
|
function rankRemarks(remarks) {
|
|
@@ -25,9 +28,12 @@ function rankRemarks(remarks) {
|
|
|
25
28
|
i
|
|
26
29
|
})).sort((a, b) => order(a.r) - order(b.r) || a.i - b.i).map(({ r }) => r);
|
|
27
30
|
}
|
|
28
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* sha256(kind, eval, the eval's version[, what it's about]): one remark per kind per eval version.
|
|
33
|
+
* The version is its `source` and `revision`, or its files on a record from before them.
|
|
34
|
+
*/
|
|
29
35
|
function remarkId(kind, evalId, entry, extra = "") {
|
|
30
|
-
const sources = sha256(JSON.stringify(entry.sources.map((s) => [s.path, s.sha256])));
|
|
36
|
+
const sources = sha256(JSON.stringify(entry.source ? [entry.source, entry.revision ?? null] : entry.sources.map((s) => [s.path, s.sha256])));
|
|
31
37
|
return sha256(JSON.stringify([
|
|
32
38
|
kind,
|
|
33
39
|
evalId,
|
|
@@ -35,16 +41,6 @@ function remarkId(kind, evalId, entry, extra = "") {
|
|
|
35
41
|
...extra ? [extra] : []
|
|
36
42
|
]));
|
|
37
43
|
}
|
|
38
|
-
/**
|
|
39
|
-
* sha256(kind, eval, the quoted source text): a model remark's id. It holds while the quote is in
|
|
40
|
-
* the file, so an edit elsewhere doesn't raise it again, and an edit to the quote acts on it.
|
|
41
|
-
*/
|
|
42
|
-
const quotedRemarkId = (kind, evalId, quote) => sha256(JSON.stringify([
|
|
43
|
-
kind,
|
|
44
|
-
evalId,
|
|
45
|
-
"quote",
|
|
46
|
-
quote
|
|
47
|
-
]));
|
|
48
44
|
const tries = (n) => `${n} ${n === 1 ? "try" : "tries"}`;
|
|
49
45
|
const capital = (s) => `${s.slice(0, 1).toUpperCase()}${s.slice(1)}`;
|
|
50
46
|
const rate = (c) => c.passed + c.failed ? c.passed / (c.passed + c.failed) : 0;
|
|
@@ -92,8 +88,7 @@ function tooFewTries(e, entry, ctx) {
|
|
|
92
88
|
fix: {
|
|
93
89
|
at: ctx.locate(entry, null),
|
|
94
90
|
edit: `set runs: 4 or more, so each side counts at least 4 tries`
|
|
95
|
-
}
|
|
96
|
-
origin: "deterministic"
|
|
91
|
+
}
|
|
97
92
|
}];
|
|
98
93
|
}
|
|
99
94
|
/** The record's tally of a check in a cell, with its variant verdict. */
|
|
@@ -150,8 +145,7 @@ function checkNeverFails(e, entries, ctx) {
|
|
|
150
145
|
fix: {
|
|
151
146
|
at: ctx.locate(entry, first.label),
|
|
152
147
|
edit: compare ? `make "${first.label}" check what the change affects, or remove it` : `make "${first.label}" strict enough to fail on a wrong result, or remove it`
|
|
153
|
-
}
|
|
154
|
-
origin: "deterministic"
|
|
148
|
+
}
|
|
155
149
|
}];
|
|
156
150
|
}
|
|
157
151
|
/**
|
|
@@ -196,8 +190,7 @@ function flakyCheck(e, entries, ctx) {
|
|
|
196
190
|
fix: {
|
|
197
191
|
at: ctx.locate(first.entry, first.label),
|
|
198
192
|
edit: `make "${first.label}" check something every good try does, or raise runs so one try doesn't decide it`
|
|
199
|
-
}
|
|
200
|
-
origin: "deterministic"
|
|
193
|
+
}
|
|
201
194
|
}];
|
|
202
195
|
}
|
|
203
196
|
/** `<plugin>:<server>` is a plugin's server; any other name is a plain `mcp` one. */
|
|
@@ -260,8 +253,7 @@ function slowMcpStart(e, entries, ctx) {
|
|
|
260
253
|
fix: {
|
|
261
254
|
at: ctx.locate(first.entry, name),
|
|
262
255
|
edit: worst.never ? `run ${server}'s command in the sandbox with c.exec to see why it doesn't answer initialize` : `install what ${server} needs in the image (install or dockerfile), so it doesn't fetch or build it at start-up`
|
|
263
|
-
}
|
|
264
|
-
origin: "deterministic"
|
|
256
|
+
}
|
|
265
257
|
});
|
|
266
258
|
}
|
|
267
259
|
return out;
|
|
@@ -309,42 +301,44 @@ function proxyOnlyEgress(e, entries, ctx) {
|
|
|
309
301
|
fix: {
|
|
310
302
|
at: ctx.locate(first.entry, null),
|
|
311
303
|
edit: "set egress: 'allow-all' on the eval if it needs raw TCP or UDP, or point the client at HTTPS_PROXY"
|
|
312
|
-
}
|
|
313
|
-
origin: "deterministic"
|
|
304
|
+
}
|
|
314
305
|
}];
|
|
315
306
|
}
|
|
316
307
|
/**
|
|
317
|
-
*
|
|
318
|
-
*
|
|
319
|
-
*
|
|
308
|
+
* setup-process-secret (concern): a setup step left a process running with a `to: 'checks'` secret
|
|
309
|
+
* without `hosts` in its environment (`setupProcesses`), so it holds the real value while the agent,
|
|
310
|
+
* as the same user, works beside it. One per eval.
|
|
320
311
|
*/
|
|
321
|
-
function
|
|
322
|
-
|
|
323
|
-
const
|
|
324
|
-
const
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
})
|
|
346
|
-
|
|
347
|
-
|
|
312
|
+
function setupProcessSecret(e, entries, ctx) {
|
|
313
|
+
const cellPaths = /* @__PURE__ */ new Map();
|
|
314
|
+
for (const u of units(e)) for (const c of u.cells) cellPaths.set(`${u.id}\0${c.key}`, c.path);
|
|
315
|
+
const hits = entries.flatMap((entry) => entry.attempts.flatMap((attempt) => attempt.setupProcesses?.length ? [{
|
|
316
|
+
entry,
|
|
317
|
+
attempt,
|
|
318
|
+
processes: attempt.setupProcesses
|
|
319
|
+
}] : []));
|
|
320
|
+
const [first] = hits;
|
|
321
|
+
if (!first) return [];
|
|
322
|
+
const secrets = [...new Set(hits.flatMap((h) => h.processes.flatMap((p) => p.secrets)))];
|
|
323
|
+
const commands = [...new Set(hits.flatMap((h) => h.processes.map((p) => p.command)))];
|
|
324
|
+
const more = commands.length > 1 ? ` (and ${commands.length - 1} more)` : "";
|
|
325
|
+
const ofTries = hits.length > 1 ? ` in ${hits.length} tries` : "";
|
|
326
|
+
const [secret = ""] = secrets;
|
|
327
|
+
const list = secrets.join(", ");
|
|
328
|
+
const which = secrets.length > 1 ? "to: 'checks' secrets" : "a to: 'checks' secret";
|
|
329
|
+
const [its, those] = commands.length > 1 ? ["their", "those processes"] : ["its", "that process"];
|
|
330
|
+
return [{
|
|
331
|
+
id: remarkId("setup-process-secret", e.path, first.entry),
|
|
332
|
+
evalId: e.path,
|
|
333
|
+
kind: "setup-process-secret",
|
|
334
|
+
severity: "concern",
|
|
335
|
+
text: `Setup left "${clip(commands[0] ?? "", 80)}"${more} running with ${list}, ${which} without hosts, in ${its} environment${ofTries}, so the agent can read the real value from ${those}.`,
|
|
336
|
+
evidence: hits.map((h) => `${cellPaths.get(`${h.entry.id}\0${h.attempt.cell}`) ?? h.entry.id}#${h.attempt.n}`),
|
|
337
|
+
fix: {
|
|
338
|
+
at: ctx.locate(first.entry, secret),
|
|
339
|
+
edit: `broker ${secret} with the hosts the process sends it to, e.g. secret({ to: 'checks', hosts: ['api.example.com'] }), so it holds a placeholder, or start the process from c.exec after the agent's turn`
|
|
340
|
+
}
|
|
341
|
+
}];
|
|
348
342
|
}
|
|
349
343
|
/** Every deterministic remark the run raises, ranked: concerns first, then by kind and eval. */
|
|
350
344
|
function deterministicRemarks(suite, record, ctx) {
|
|
@@ -356,11 +350,247 @@ function deterministicRemarks(suite, record, ctx) {
|
|
|
356
350
|
const own = units(e).flatMap((u) => entries(u.id) ?? []);
|
|
357
351
|
const first = own[0];
|
|
358
352
|
if (!first) continue;
|
|
359
|
-
out.push(...flakyCheck(e, entries, ctx), ...slowMcpStart(e, own, ctx), ...proxyOnlyEgress(e, own, ctx), ...tooFewTries(e, first, ctx), ...checkNeverFails(e, entries, ctx));
|
|
353
|
+
out.push(...flakyCheck(e, entries, ctx), ...slowMcpStart(e, own, ctx), ...proxyOnlyEgress(e, own, ctx), ...setupProcessSecret(e, own, ctx), ...tooFewTries(e, first, ctx), ...checkNeverFails(e, entries, ctx));
|
|
360
354
|
}
|
|
361
355
|
return rankRemarks(out);
|
|
362
356
|
}
|
|
363
357
|
//#endregion
|
|
364
|
-
|
|
358
|
+
//#region src/remarks/index.ts
|
|
359
|
+
/**
|
|
360
|
+
* The branch the default-branch views count: `origin`'s HEAD, else a local `main` or `master`,
|
|
361
|
+
* else in CI a pull request's base, `GITHUB_BASE_REF`. A GitHub Actions pull request build checks
|
|
362
|
+
* out the merge commit with neither ref; a push build of `main` or `master` checks it out locally.
|
|
363
|
+
*/
|
|
364
|
+
async function defaultBranch(root, env = process.env) {
|
|
365
|
+
const head = await git(root, [
|
|
366
|
+
"symbolic-ref",
|
|
367
|
+
"--quiet",
|
|
368
|
+
"--short",
|
|
369
|
+
"refs/remotes/origin/HEAD"
|
|
370
|
+
]);
|
|
371
|
+
if (head.code === 0) {
|
|
372
|
+
const ref = head.stdout.toString("utf8").trim();
|
|
373
|
+
if (ref) return ref.replace(/^origin\//, "");
|
|
374
|
+
}
|
|
375
|
+
for (const name of ["main", "master"]) if ((await git(root, [
|
|
376
|
+
"rev-parse",
|
|
377
|
+
"--verify",
|
|
378
|
+
"--quiet",
|
|
379
|
+
`refs/heads/${name}`
|
|
380
|
+
])).code === 0) return name;
|
|
381
|
+
return inCI(env) && env.GITHUB_BASE_REF || null;
|
|
382
|
+
}
|
|
383
|
+
/** The ids of every run but `runId`, recorded and scratch, with the folder each is in. */
|
|
384
|
+
async function otherRuns(config, runId) {
|
|
385
|
+
const out = [];
|
|
386
|
+
const seen = /* @__PURE__ */ new Set();
|
|
387
|
+
for (const dir of [config.history.dir, scratchDir(config.root)]) for (const name of await recordNames(dir).catch(() => null) ?? []) {
|
|
388
|
+
const id = name.slice(0, -5);
|
|
389
|
+
if (!RUN_ID.test(id) || id === runId || seen.has(id)) continue;
|
|
390
|
+
seen.add(id);
|
|
391
|
+
out.push({
|
|
392
|
+
dir,
|
|
393
|
+
runId: id
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
return out;
|
|
397
|
+
}
|
|
398
|
+
/** Each run's remarks, from its remarks.json; a run without one has none. */
|
|
399
|
+
async function remarksIn(runs) {
|
|
400
|
+
const out = [];
|
|
401
|
+
for (const run of runs) {
|
|
402
|
+
const doc = await readRemarks(remarksFile(run.dir, run.runId)).catch(() => null);
|
|
403
|
+
if (doc?.remarks.length) out.push({
|
|
404
|
+
runId: run.runId,
|
|
405
|
+
remarks: doc.remarks
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
return out;
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Each remark id an earlier run printed, with the first run that did. A run's `firstSeenRun` is a
|
|
412
|
+
* run that printed it, unless that run is itself and `printed` is false: the cap or a run without
|
|
413
|
+
* `-v` hid it.
|
|
414
|
+
*/
|
|
415
|
+
function printedBefore(runs) {
|
|
416
|
+
const first = /* @__PURE__ */ new Map();
|
|
417
|
+
for (const run of runs) for (const r of run.remarks) {
|
|
418
|
+
if (r.printed === false && r.firstSeenRun === run.runId) continue;
|
|
419
|
+
const was = first.get(r.id);
|
|
420
|
+
if (!was || r.firstSeenRun < was) first.set(r.id, r.firstSeenRun);
|
|
421
|
+
}
|
|
422
|
+
return first;
|
|
423
|
+
}
|
|
424
|
+
/** A remark as raised, before it's settled: its kind's severity, and no run-specific marks. */
|
|
425
|
+
function candidateOf(r) {
|
|
426
|
+
const { firstSeenRun: _, printed: __, ...rest } = r;
|
|
427
|
+
return {
|
|
428
|
+
...rest,
|
|
429
|
+
severity: REMARK_KINDS[r.kind].severity
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
/** The run's remarks from what it raised, once each: with the first run that printed it, ranked. */
|
|
433
|
+
async function settleRemarks(config, record, candidates) {
|
|
434
|
+
const first = printedBefore(await remarksIn(await otherRuns(config, record.runId)));
|
|
435
|
+
const seen = /* @__PURE__ */ new Set();
|
|
436
|
+
return rankRemarks(candidates.flatMap((c) => {
|
|
437
|
+
if (seen.has(c.id)) return [];
|
|
438
|
+
seen.add(c.id);
|
|
439
|
+
const was = first.get(c.id);
|
|
440
|
+
return [{
|
|
441
|
+
...candidateOf({
|
|
442
|
+
...c,
|
|
443
|
+
firstSeenRun: ""
|
|
444
|
+
}),
|
|
445
|
+
firstSeenRun: was && was < record.runId ? was : record.runId
|
|
446
|
+
}];
|
|
447
|
+
}));
|
|
448
|
+
}
|
|
449
|
+
const one = (db, sql, params) => db.query(sql, params).rows[0] ?? null;
|
|
450
|
+
function historyOf(db, branch) {
|
|
451
|
+
return {
|
|
452
|
+
streak: (evalKey, cellKey, label, fingerprint) => branch === null ? null : one(db, `SELECT streak_runs AS runs, streak_tries AS tries FROM check_streaks
|
|
453
|
+
WHERE eval_key = ? AND cell_key = ? AND label = ? AND fingerprint = ? AND branch = ?`, [
|
|
454
|
+
evalKey,
|
|
455
|
+
cellKey,
|
|
456
|
+
label,
|
|
457
|
+
fingerprint,
|
|
458
|
+
branch
|
|
459
|
+
]),
|
|
460
|
+
flaky: (evalKey, cellKey, fingerprint) => one(db, `SELECT cause FROM flaky_cells
|
|
461
|
+
WHERE eval_key = ? AND cell_key = ? AND fingerprint = ? AND verdict = 'flaky'`, [
|
|
462
|
+
evalKey,
|
|
463
|
+
cellKey,
|
|
464
|
+
fingerprint
|
|
465
|
+
]),
|
|
466
|
+
window: (evalKey, cellKey, label, fingerprint) => {
|
|
467
|
+
return db.query(`SELECT k.pass, k.fail FROM cell_checks k
|
|
468
|
+
JOIN cells c ON c.run_id = k.run_id AND c.eval_key = k.eval_key AND c.key = k.cell_key
|
|
469
|
+
WHERE k.eval_key = ? AND k.cell_key = ? AND k.label = ? AND c.comparable = ?
|
|
470
|
+
ORDER BY k.run_id DESC LIMIT 5`, [
|
|
471
|
+
evalKey,
|
|
472
|
+
cellKey,
|
|
473
|
+
label,
|
|
474
|
+
fingerprint
|
|
475
|
+
]).rows.reduce((w, r) => ({
|
|
476
|
+
runs: w.runs + 1,
|
|
477
|
+
mixed: w.mixed + (r.pass > 0 && r.fail > 0 ? 1 : 0),
|
|
478
|
+
pass: w.pass + r.pass,
|
|
479
|
+
fail: w.fail + r.fail
|
|
480
|
+
}), {
|
|
481
|
+
runs: 0,
|
|
482
|
+
mixed: 0,
|
|
483
|
+
pass: 0,
|
|
484
|
+
fail: 0
|
|
485
|
+
});
|
|
486
|
+
}
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* Finds where a remark's fix goes: the first line naming `needle` from the eval's own declaration
|
|
491
|
+
* on, so another eval in the same file using that label doesn't take it, then anywhere in its
|
|
492
|
+
* sources.
|
|
493
|
+
*/
|
|
494
|
+
function locator(root) {
|
|
495
|
+
const files = /* @__PURE__ */ new Map();
|
|
496
|
+
const text = (path) => {
|
|
497
|
+
let f = files.get(path);
|
|
498
|
+
if (!f) {
|
|
499
|
+
f = readFile(join(root, path), "utf8").catch(() => null);
|
|
500
|
+
files.set(path, f);
|
|
501
|
+
}
|
|
502
|
+
return f;
|
|
503
|
+
};
|
|
504
|
+
const found = /* @__PURE__ */ new Map();
|
|
505
|
+
return {
|
|
506
|
+
async prepare(entry, needles) {
|
|
507
|
+
const quoted = (s) => [
|
|
508
|
+
`'${s}'`,
|
|
509
|
+
`"${s}"`,
|
|
510
|
+
`\`${s}\``
|
|
511
|
+
];
|
|
512
|
+
const lines = async (path) => (await text(path))?.split("\n") ?? [];
|
|
513
|
+
const own = await lines(entry.file);
|
|
514
|
+
const name = quoted(entry.family?.name ?? entry.name);
|
|
515
|
+
const declared = Math.max(0, own.findIndex((line) => name.some((w) => line.includes(w))));
|
|
516
|
+
const scans = [
|
|
517
|
+
{
|
|
518
|
+
path: entry.file,
|
|
519
|
+
from: declared
|
|
520
|
+
},
|
|
521
|
+
{
|
|
522
|
+
path: entry.file,
|
|
523
|
+
from: 0
|
|
524
|
+
},
|
|
525
|
+
...entry.sources.filter((s) => s.path !== entry.file).map((s) => ({
|
|
526
|
+
path: s.path,
|
|
527
|
+
from: 0
|
|
528
|
+
}))
|
|
529
|
+
];
|
|
530
|
+
for (const needle of needles) {
|
|
531
|
+
const key = `${entry.id}\0${needle ?? ""}`;
|
|
532
|
+
if (found.has(key)) continue;
|
|
533
|
+
const tiers = needle === null ? [name] : [
|
|
534
|
+
[`${needle}:`, ...quoted(needle).map((q) => `${q}:`)],
|
|
535
|
+
quoted(needle),
|
|
536
|
+
[needle]
|
|
537
|
+
];
|
|
538
|
+
let at = `source:${entry.file}`;
|
|
539
|
+
search: for (const wanted of tiers) for (const { path, from } of needle === null ? scans.slice(1, 2) : scans) for (const [i, line] of (await lines(path)).entries()) if (i >= from && wanted.some((w) => line.includes(w))) {
|
|
540
|
+
at = `source:${path}#L${i + 1}`;
|
|
541
|
+
break search;
|
|
542
|
+
}
|
|
543
|
+
found.set(key, at);
|
|
544
|
+
}
|
|
545
|
+
},
|
|
546
|
+
locate: (entry, needle) => found.get(`${entry.id}\0${needle ?? ""}`) ?? `source:${entry.file}`
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* What the rules may look up in an eval's source: its name, check labels, MCP servers and the
|
|
551
|
+
* secrets setup's processes hold.
|
|
552
|
+
*/
|
|
553
|
+
function needlesOf(entry) {
|
|
554
|
+
const labels = entry.cells.flatMap((c) => (c.checks ?? []).map((k) => k.label));
|
|
555
|
+
const servers = entry.attempts.flatMap((a) => a.turns.flatMap((t) => (t.mcpStartup ?? []).map((m) => m.server.split(":")[0])));
|
|
556
|
+
const secrets = entry.attempts.flatMap((a) => (a.setupProcesses ?? []).flatMap((p) => p.secrets));
|
|
557
|
+
return [null, .../* @__PURE__ */ new Set([
|
|
558
|
+
...labels,
|
|
559
|
+
...servers,
|
|
560
|
+
...secrets
|
|
561
|
+
])];
|
|
562
|
+
}
|
|
563
|
+
/** The run's remarks, each with the run that first printed it. History-based kinds need an earlier run. */
|
|
564
|
+
async function remarksOf(config, record) {
|
|
565
|
+
const db = (await otherRuns(config, record.runId)).length ? await openHistoryDb(config.root, config.history.dir) : null;
|
|
566
|
+
try {
|
|
567
|
+
const loc = locator(config.root);
|
|
568
|
+
for (const entry of record.evals) await loc.prepare(entry, needlesOf(entry));
|
|
569
|
+
const ctx = {
|
|
570
|
+
history: db ? historyOf(db, await defaultBranch(config.root)) : null,
|
|
571
|
+
comparable: (entry, cellKey) => {
|
|
572
|
+
const cell = entry.cells.find((c) => c.key === cellKey);
|
|
573
|
+
return cell ? comparableFingerprint(cell, entry.attempts) : "";
|
|
574
|
+
},
|
|
575
|
+
locate: loc.locate
|
|
576
|
+
};
|
|
577
|
+
const candidates = deterministicRemarks(suiteResult(record), record, ctx);
|
|
578
|
+
return candidates.length ? await settleRemarks(config, record, candidates) : [];
|
|
579
|
+
} finally {
|
|
580
|
+
db?.close();
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
/** The run's remarks document. */
|
|
584
|
+
function remarksDoc(record, remarks) {
|
|
585
|
+
return {
|
|
586
|
+
kind: "reeval-remarks",
|
|
587
|
+
schemaVersion: 1,
|
|
588
|
+
runId: record.runId,
|
|
589
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
590
|
+
remarks
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
//#endregion
|
|
594
|
+
export { remarksDoc, remarksOf };
|
|
365
595
|
|
|
366
|
-
//# sourceMappingURL=
|
|
596
|
+
//# sourceMappingURL=remarks-C7i2LMRb.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remarks-C7i2LMRb.js","names":[],"sources":["../src/remarks/rules.ts","../src/remarks/index.ts"],"mappings":";;;;;;;;;;;;;;AAiDA,MAAM,aAA+B;CACnC;CACA;CACA;CACA;CACA;CACA;AACF;;AAGA,SAAgB,YAAyD,SAAmB;CAC1F,MAAM,SAAS,OAAU,EAAE,aAAa,YAAY,IAAI,KAAK,MAAM,WAAW,QAAQ,EAAE,IAAI;CAC5F,OAAO,QACJ,KAAK,GAAG,OAAO;EAAE;EAAG;CAAE,EAAE,CAAC,CACzB,MAAM,GAAG,MAAM,MAAM,EAAE,CAAC,IAAI,MAAM,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CACpD,KAAK,EAAE,QAAQ,CAAC;AACrB;;;;;AAMA,SAAgB,SAAS,MAAsB,QAAgB,OAAkB,QAAQ,IAAI;CAC3F,MAAM,UAAU,OACd,KAAK,UACH,MAAM,SACF,CAAC,MAAM,QAAQ,MAAM,YAAY,IAAI,IACrC,MAAM,QAAQ,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CACjD,CACF;CACA,OAAO,OAAO,KAAK,UAAU;EAAC;EAAM;EAAQ;EAAS,GAAI,QAAQ,CAAC,KAAK,IAAI,CAAC;CAAE,CAAC,CAAC;AAClF;AAEA,MAAM,SAAS,MAAc,GAAG,EAAE,GAAG,MAAM,IAAI,QAAQ;AACvD,MAAM,WAAW,MAAc,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,MAAM,CAAC;AACzE,MAAM,QAAQ,MAAmB,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU;AAC1F,MAAM,WAAW,MAAkB,EAAE,SAAS,EAAE;;AAGhD,SAAS,MAAM,GAA2D;CACxE,IAAI,EAAE,OAAO,OAAO,EAAE,MAAM,KAAK,OAAO;EAAE,IAAI,EAAE;EAAI,OAAO,EAAE;CAAM,EAAE;CACrE,OAAO,CAAC;EAAE,IAAI,EAAE;EAAI,OAAO,EAAE,SAAS,CAAC;CAAE,CAAC;AAC5C;AAEA,MAAM,SAAS,MAAkB,EAAE,YAAY,EAAE,gBAAgB,MAAM,aAAa,EAAE;;;;;;AAOtF,SAAS,YAAY,GAAe,OAAkB,KAAiC;CACrF,IAAI,QAAuE;CAC3E,KAAK,MAAM,QAAQ,MAAM,CAAC,GACxB,KAAK,MAAM,WAAW,KAAK,OAAO;EAChC,MAAM,OAAO,KAAK,MAAM,MAAM,MAAM,EAAE,QAAQ,QAAQ,EAAE;EACxD,IAAI,CAAC,QAAQ,MAAM,CAAC,QAAQ,CAAC,QAAQ,OAAO,KAAK,CAAC,QAAQ,IAAI,GAAG;EACjE,IAAI,KAAK,IAAI,QAAQ,OAAO,GAAG,QAAQ,IAAI,CAAC,KAAA,GAAyB;EACrE,MAAM,MAAM,KAAK,IAAI,KAAK,OAAO,IAAI,KAAK,IAAI,CAAC;EAC/C,IAAI,OAAA,OAA6B,CAAC,SAAS,MAAM,MAAM,MAAM,QAAQ;GAAE;GAAS;GAAM;EAAI;CAC5F;CAEF,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,MAAM,EAAE,SAAS,SAAS;CAE1B,MAAM,KADY,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,SAAS,MAAM,EAAE,MAAM,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,IAClE,OAAO,QAAQ,YAAY;CAClD,OAAO,CACL;EACE,IAAI,SAAS,iBAAiB,EAAE,MAAM,KAAK;EAC3C,QAAQ,EAAE;EACV,MAAM;EACN,UAAU;EACV,MAAM,GAAG,QAAQ,MAAM,OAAO,CAAC,IAAI,GAAG,UAAU,QAAQ,OAAO,MAAM,MAAM,QAAQ,OAAO,CAAC,EAAE,OAAO,MAAM,IAAI,EAAE,GAAG,KAAK,OAAO,MAAM,QAAQ,IAAI,EAAE;EACnJ,UAAU,CAAC,QAAQ,MAAM,KAAK,IAAI;EAClC,KAAK;GACH,IAAI,IAAI,OAAO,OAAO,IAAI;GAC1B,MAAM;EACR;CACF,CACF;AACF;;AAGA,MAAM,eAAe,OAAkB,SAAiB,UACtD,MAAM,MAAM,MAAM,MAAM,EAAE,QAAQ,OAAO,CAAC,EAAE,QAAQ,MAAM,MAAM,EAAE,UAAU,KAAK;;;;;;AAOnF,SAAS,gBACP,GACA,SACA,KACa;CACb,MAAM,UAAU,IAAI;CACpB,IAAI,CAAC,SAAS,OAAO,CAAC;CACtB,MAAM,MAAM,MAAM,CAAC;CACnB,MAAM,SAAS,CACb,GAAG,IAAI,IAAI,IAAI,SAAS,MAAM,EAAE,MAAM,SAAS,MAAM,EAAE,OAAO,KAAK,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CACrF;CACA,MAAM,QAAgF,CAAC;CACvF,KAAK,MAAM,SAAS,QAAQ;EAC1B,IAAI,OAAO,OAAO;EAClB,IAAI,QAAQ,OAAO;EACnB,MAAM,QAAkB,CAAC;EAmBzB,IAlBW,IAAI,OAAO,MAAM;GAC1B,MAAM,QAAQ,QAAQ,EAAE,EAAE;GAC1B,OACE,SACA,EAAE,MAAM,OAAO,SAAS;IACtB,MAAM,IAAI,KAAK,OAAO,MAAM,MAAM,EAAE,UAAU,KAAK;IACnD,IAAI,GAAG,QAAQ,WAAW,OAAO,OAAO;IACxC,IAAI,EAAE,QAAQ,UAAU,EAAE,QAAQ,WAAW,CAAC,EAAE,QAAQ,QAAQ,OAAO;IACvE,IAAI,KAAK,MAAM,YAAY,OAAO,KAAK,KAAK,KAAK,CAAC,EAAE,YAAY,aAAa,OAAO;IACpF,MAAM,IAAI,QAAQ,OAAO,EAAE,IAAI,KAAK,KAAK,OAAO,IAAI,WAAW,OAAO,KAAK,GAAG,CAAC;IAC/E,IAAI,CAAC,KAAK,EAAE,OAAA,KAA2B,EAAE,QAAA,IAA2B,OAAO;IAC3E,OAAO,KAAK,IAAI,MAAM,EAAE,IAAI;IAC5B,QAAQ,KAAK,IAAI,OAAO,EAAE,KAAK;IAC/B,MAAM,KAAK,EAAE,IAAI;IACjB,OAAO;GACT,CAAC;EAEL,CACK,KAAK,MAAM,QAAQ,MAAM,KAAK;GAAE;GAAO;GAAM,OAAO;GAAO;EAAM,CAAC;CACzE;CACA,MAAM,CAAC,SAAS;CAChB,MAAM,QAAQ,QAAQ,IAAI,EAAE,EAAE,MAAM,EAAE;CACtC,IAAI,CAAC,SAAS,CAAC,OAAO,OAAO,CAAC;CAC9B,MAAM,UAAU,IAAI,MAAM,MAAM,EAAE,MAAM,MAAM,MAAM,EAAE,EAAE,CAAC;CACzD,MAAM,OAAO,MAAM,SAAS,IAAI,SAAS,MAAM,SAAS,EAAE,UAAU;CACpE,MAAM,OAAO,UACT,IAAI,MAAM,MAAM,GAAG,KAAK,0DAA0D,MAAM,KAAK,oDAC7F,IAAI,MAAM,MAAM,GAAG,KAAK,gCAAgC,MAAM,KAAK,kBAAkB,MAAM,MAAM;CACrG,OAAO,CACL;EACE,IAAI,SAAS,qBAAqB,EAAE,MAAM,KAAK;EAC/C,QAAQ,EAAE;EACV,MAAM;EACN,UAAU;EACV;EACA,UAAU,MAAM,SAAS,MAAM,EAAE,KAAK;EACtC,KAAK;GACH,IAAI,IAAI,OAAO,OAAO,MAAM,KAAK;GACjC,MAAM,UACF,SAAS,MAAM,MAAM,iDACrB,SAAS,MAAM,MAAM;EAC3B;CACF,CACF;AACF;;;;;;AAOA,SAAS,WACP,GACA,SACA,KACa;CACb,MAAM,UAAU,IAAI;CACpB,IAAI,CAAC,SAAS,OAAO,CAAC;CACtB,MAAM,QAAuF,CAAC;CAC9F,KAAK,MAAM,KAAK,MAAM,CAAC,GAAG;EACxB,MAAM,QAAQ,QAAQ,EAAE,EAAE;EAC1B,IAAI,CAAC,OAAO;EACZ,KAAK,MAAM,QAAQ,EAAE,OAAO;GAC1B,MAAM,cAAc,IAAI,WAAW,OAAO,KAAK,GAAG;GAClD,MAAM,QAAQ,QAAQ,MAAM,EAAE,IAAI,KAAK,KAAK,WAAW;GACvD,IAAI,CAAC,SAAS,MAAM,UAAU,SAAS;GACvC,KAAK,MAAM,KAAK,KAAK,QAAQ;IAC3B,IAAI,EAAE,QAAQ,WAAW,SAAS,MAAM,MAAM,MAAM,EAAE,UAAU,EAAE,KAAK,GAAG;IAC1E,MAAM,SAAS,QAAQ,OAAO,EAAE,IAAI,KAAK,KAAK,EAAE,OAAO,WAAW;IAClE,IAAI,OAAO,SAAA,KAAuB,OAAO,OAAO,OAAO,QAAA,GACrD,MAAM,KAAK;KAAE,OAAO,EAAE;KAAO;KAAQ,MAAM,EAAE;KAAM;IAAM,CAAC;GAE9D;EACF;CACF;CACA,MAAM,CAAC,SAAS;CAChB,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,MAAM,EAAE,QAAQ,MAAM;CACtB,MAAM,OAAO,MAAM,SAAS,IAAI,SAAS,MAAM,SAAS,EAAE,UAAU;CACpE,OAAO,CACL;EACE,IAAI,SAAS,eAAe,EAAE,MAAM,MAAM,KAAK;EAC/C,QAAQ,EAAE;EACV,MAAM;EACN,UAAU;EACV,MAAM,IAAI,MAAM,MAAM,GAAG,KAAK,UAAU,EAAE,KAAK,MAAM,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK;EAC3F,UAAU,MAAM,KAAK,MAAM,EAAE,IAAI;EACjC,KAAK;GACH,IAAI,IAAI,OAAO,MAAM,OAAO,MAAM,KAAK;GACvC,MAAM,SAAS,MAAM,MAAM;EAC7B;CACF,CACF;AACF;;AAGA,SAAS,WAAW,QAAwB;CAC1C,MAAM,KAAK,OAAO,QAAQ,GAAG;CAC7B,OAAO,KAAK,IACR,OAAO,OAAO,MAAM,GAAG,EAAE,EAAE,uBAAuB,OAAO,MAAM,KAAK,CAAC,MACrE,cAAc;AACpB;;AAGA,MAAM,WAAW,OAAe;CAC9B,MAAM,IAAI,KAAK,MAAM,KAAK,GAAG,IAAI;CACjC,OAAO,GAAG,KAAK,KAAK,KAAK,MAAM,CAAC,IAAI,EAAE;AACxC;AAEA,MAAM,QAAQ,OAAsB,OAAO,QAAQ,MAAA;;;;;;;AAQnD,SAAS,aAAa,GAAe,SAAsB,KAAiC;CAC1F,MAAM,4BAAY,IAAI,IAAoB;CAC1C,KAAK,MAAM,KAAK,MAAM,CAAC,GAAG,KAAK,MAAM,KAAK,EAAE,OAAO,UAAU,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI;CAC5F,MAAM,WAAgE,QAAQ,SAAS,UACrF,MAAM,SAAS,KAAK,aAAa;EAAE;EAAO;CAAQ,EAAE,CACtD;CACA,MAAM,UAAU,CACd,GAAG,IAAI,IACL,SAAS,SAAS,EAAE,cAClB,QAAQ,MAAM,SAAS,OAAO,EAAE,cAAc,CAAC,EAAA,CAAG,KAAK,MAAM,EAAE,MAAM,CAAC,CACxE,CACF,CACF;CACA,MAAM,MAAmB,CAAC;CAC1B,KAAK,MAAM,UAAU,SAAS;EAS5B,MAAM,WAAkB,CAAC;EACzB,KAAK,MAAM,EAAE,OAAO,aAAa,UAAU;GACzC,MAAM,QAAQ,QAAQ,MACnB,KAAK,OAAO,EAAE,cAAc,CAAC,EAAA,CAAG,QAAQ,MAAM,EAAE,WAAW,MAAM,CAAC,CAAC,CACnE,QAAQ,WAAW,OAAO,SAAS,CAAC;GACvC,IAAI,CAAC,MAAM,QAAQ;GACnB,MAAM,YAAY,MAAM,QAAQ,WAAW,OAAO,MAAM,MAAM,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;GAC3E,MAAM,SAAS,MAAM,KAAK;GAC1B,SAAS,KAAK;IACZ;IACA;IACA,IAAI,OAAO,SAAS,MAAO,EAAE,OAAO,QAAQ,EAAE,MAAA,MAAoB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAE;IAC9E,OAAO,OAAO,MAAM,MAAM,EAAE,OAAO,IAAI;IACvC,OAAO,MAAM;IACb;GACF,CAAC;EACH;EACA,MAAM,OAAO,SAAS,QAAQ,MAAM,EAAE,YAAY,CAAC;EACnD,MAAM,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,MACrB,GAAG,MAAM,OAAO,EAAE,KAAK,IAAI,OAAO,EAAE,KAAK,KAAK,KAAK,IAAI,GAAG,GAAG,EAAE,EAAE,IAAI,KAAK,IAAI,GAAG,GAAG,EAAE,EAAE,CAC3F,CAAC,CAAC;EACF,IAAI,CAAC,OAAO;EACZ,MAAM,QAAQ,KAAK;EACnB,MAAM,OAAO,MAAM,QACf,GAAG,WAAW,MAAM,EAAE,+CACtB,GAAG,WAAW,MAAM,EAAE,QAAQ,QAAQ,KAAK,IAAI,GAAG,MAAM,EAAE,CAAC,EAAE;EACjE,MAAM,QACJ,MAAM,QAAQ,IACV,MAAM,cAAc,MAAM,QACxB,eAAe,MAAM,MAAM,UAC3B,OAAO,MAAM,UAAU,MAAM,MAAM,MAAM,UAC3C;EACN,MAAM,UAAU,SAAS,SAAS,IAAI,QAAQ,KAAK,OAAO,MAAM,SAAS,OAAO,UAAU;EAC1F,MAAM,OAAO,OAAO,SAAS,GAAG,IAAI,OAAO,MAAM,GAAG,OAAO,QAAQ,GAAG,CAAC,IAAI;EAC3E,IAAI,KAAK;GACP,IAAI,SAAS,kBAAkB,EAAE,MAAM,MAAM,OAAO,MAAM;GAC1D,QAAQ,EAAE;GACV,MAAM;GACN,UAAU;GACV,MAAM,GAAG,QAAQ,IAAI,IAAI,QAAQ,QAAQ;GACzC,UAAU,KAAK,KACZ,MAAM,GAAG,UAAU,IAAI,GAAG,EAAE,MAAM,GAAG,IAAI,EAAE,QAAQ,MAAM,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,QAAQ,GACzF;GACA,KAAK;IACH,IAAI,IAAI,OAAO,MAAM,OAAO,IAAI;IAChC,MAAM,MAAM,QACR,OAAO,OAAO,iFACd,gBAAgB,OAAO;GAC7B;EACF,CAAC;CACH;CACA,OAAO;AACT;;;;;AAMA,MAAM,mBACJ;;AAGF,MAAM,eAAe,MAAqB;CACxC,EAAE,OAAO,WAAW;CACpB,IAAI,EAAE,cAAc,CAAC,EAAA,CAAG,QAAQ,MAAM,EAAE,WAAW,MAAM,CAAC,CAAC,KAAK,MAAM,EAAE,WAAW,EAAE;CACrF,GAAG,EAAE,MAAM,SAAS,MAAM,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACnD;;;;;;AAOA,SAAS,gBAAgB,GAAe,SAAsB,KAAiC;CAC7F,MAAM,4BAAY,IAAI,IAAoB;CAC1C,KAAK,MAAM,KAAK,MAAM,CAAC,GAAG,KAAK,MAAM,KAAK,EAAE,OAAO,UAAU,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI;CAC5F,MAAM,OAAO,QAAQ,SAAS,UAC5B,MAAM,SAAS,SAAS,YAAY;EAClC,IAAI,QAAQ,YAAY,UAAU,QAAQ,YAAY,eAAe,OAAO,CAAC;EAC7E,MAAM,SAAS,QAAQ;EACvB,IAAI,QAAQ,WAAW,WAAW,CAAC,OAAO,YAAY,OAAO,QAAQ,QAAQ,OAAO,CAAC;EACrF,MAAM,QAAQ,YAAY,OAAO,CAAC,CAC/B,KAAK,MAAM,iBAAiB,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,CACzC,KAAK,OAAO;EACf,OAAO,QAAQ,CAAC;GAAE;GAAO;GAAS;EAAM,CAAC,IAAI,CAAC;CAChD,CAAC,CACH;CACA,MAAM,CAAC,SAAS;CAChB,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,MAAM,UAAU,KAAK,SAAS,IAAI,KAAK,KAAK,OAAO,WAAW;CAC9D,OAAO,CACL;EACE,IAAI,SAAS,qBAAqB,EAAE,MAAM,MAAM,KAAK;EACrD,QAAQ,EAAE;EACV,MAAM;EACN,UAAU;EACV,MAAM,mBAAmB,MAAM,MAAM,GAAG,QAAQ;EAChD,UAAU,KAAK,KACZ,MAAM,GAAG,UAAU,IAAI,GAAG,EAAE,MAAM,GAAG,IAAI,EAAE,QAAQ,MAAM,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,QAAQ,GACzF;EACA,KAAK;GACH,IAAI,IAAI,OAAO,MAAM,OAAO,IAAI;GAChC,MAAM;EACR;CACF,CACF;AACF;;;;;;AAOA,SAAS,mBAAmB,GAAe,SAAsB,KAAiC;CAChG,MAAM,4BAAY,IAAI,IAAoB;CAC1C,KAAK,MAAM,KAAK,MAAM,CAAC,GAAG,KAAK,MAAM,KAAK,EAAE,OAAO,UAAU,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI;CAC5F,MAAM,OAAO,QAAQ,SAAS,UAC5B,MAAM,SAAS,SAAS,YACtB,QAAQ,gBAAgB,SAAS,CAAC;EAAE;EAAO;EAAS,WAAW,QAAQ;CAAe,CAAC,IAAI,CAAC,CAC9F,CACF;CACA,MAAM,CAAC,SAAS;CAChB,IAAI,CAAC,OAAO,OAAO,CAAC;CACpB,MAAM,UAAU,CAAC,GAAG,IAAI,IAAI,KAAK,SAAS,MAAM,EAAE,UAAU,SAAS,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvF,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,KAAK,SAAS,MAAM,EAAE,UAAU,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACpF,MAAM,OAAO,SAAS,SAAS,IAAI,SAAS,SAAS,SAAS,EAAE,UAAU;CAC1E,MAAM,UAAU,KAAK,SAAS,IAAI,OAAO,KAAK,OAAO,UAAU;CAC/D,MAAM,CAAC,SAAS,MAAM;CACtB,MAAM,OAAO,QAAQ,KAAK,IAAI;CAC9B,MAAM,QAAQ,QAAQ,SAAS,IAAI,yBAAyB;CAC5D,MAAM,CAAC,KAAK,SAAS,SAAS,SAAS,IAAI,CAAC,SAAS,iBAAiB,IAAI,CAAC,OAAO,cAAc;CAChG,OAAO,CACL;EACE,IAAI,SAAS,wBAAwB,EAAE,MAAM,MAAM,KAAK;EACxD,QAAQ,EAAE;EACV,MAAM;EACN,UAAU;EACV,MAAM,eAAe,KAAK,SAAS,MAAM,IAAI,EAAE,EAAE,GAAG,KAAK,gBAAgB,KAAK,IAAI,MAAM,qBAAqB,IAAI,cAAc,QAAQ,8CAA8C,MAAM;EAC3L,UAAU,KAAK,KACZ,MAAM,GAAG,UAAU,IAAI,GAAG,EAAE,MAAM,GAAG,IAAI,EAAE,QAAQ,MAAM,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,QAAQ,GACzF;EACA,KAAK;GACH,IAAI,IAAI,OAAO,MAAM,OAAO,MAAM;GAClC,MAAM,UAAU,OAAO;EACzB;CACF,CACF;AACF;;AAGA,SAAgB,qBACd,OACA,QACA,KACa;CACb,MAAM,OAAO,IAAI,IAAI,OAAO,MAAM,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;CACvD,MAAM,WAAW,OAAe,KAAK,IAAI,EAAE;CAC3C,MAAM,MAAmB,CAAC;CAC1B,KAAK,MAAM,QAAQ,MAAM,OACvB,KAAK,MAAM,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,GAAG;EACjE,IAAI,EAAE,QAAQ,EAAE,YAAY,WAAW;EACvC,MAAM,MAAM,MAAM,CAAC,CAAC,CAAC,SAAS,MAAM,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;EACvD,MAAM,QAAQ,IAAI;EAClB,IAAI,CAAC,OAAO;EACZ,IAAI,KACF,GAAG,WAAW,GAAG,SAAS,GAAG,GAC7B,GAAG,aAAa,GAAG,KAAK,GAAG,GAC3B,GAAG,gBAAgB,GAAG,KAAK,GAAG,GAC9B,GAAG,mBAAmB,GAAG,KAAK,GAAG,GACjC,GAAG,YAAY,GAAG,OAAO,GAAG,GAC5B,GAAG,gBAAgB,GAAG,SAAS,GAAG,CACpC;CACF;CAEF,OAAO,YAAY,GAAG;AACxB;;;;;;;;AC1bA,eAAsB,cACpB,MACA,MAAyB,QAAQ,KACT;CACxB,MAAM,OAAO,MAAM,IAAI,MAAM;EAAC;EAAgB;EAAW;EAAW;CAA0B,CAAC;CAC/F,IAAI,KAAK,SAAS,GAAG;EACnB,MAAM,MAAM,KAAK,OAAO,SAAS,MAAM,CAAC,CAAC,KAAK;EAC9C,IAAI,KAAK,OAAO,IAAI,QAAQ,aAAa,EAAE;CAC7C;CACA,KAAK,MAAM,QAAQ,CAAC,QAAQ,QAAQ,GAElC,KAAI,MADY,IAAI,MAAM;EAAC;EAAa;EAAY;EAAW,cAAc;CAAM,CAAC,EAAA,CAC9E,SAAS,GAAG,OAAO;CAE3B,OAAQ,KAAK,GAAG,KAAK,IAAI,mBAAoB;AAC/C;;AAGA,eAAe,UAAU,QAAwB,OAAe;CAC9D,MAAM,MAA6C,CAAC;CACpD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,OAAO,CAAC,OAAO,QAAQ,KAAK,WAAW,OAAO,IAAI,CAAC,GAC5D,KAAK,MAAM,QAAS,MAAM,YAAY,GAAG,CAAC,CAAC,YAAY,IAAI,KAAM,CAAC,GAAG;EACnE,MAAM,KAAK,KAAK,MAAM,GAAG,EAAE;EAC3B,IAAI,CAAC,OAAO,KAAK,EAAE,KAAK,OAAO,SAAS,KAAK,IAAI,EAAE,GAAG;EACtD,KAAK,IAAI,EAAE;EACX,IAAI,KAAK;GAAE;GAAK,OAAO;EAAG,CAAC;CAC7B;CAEF,OAAO;AACT;;AAKA,eAAe,UAAU,MAAgE;CACvF,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,MAAM,MAAM,YAAY,YAAY,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,YAAY,IAAI;EAC/E,IAAI,KAAK,QAAQ,QAAQ,IAAI,KAAK;GAAE,OAAO,IAAI;GAAO,SAAS,IAAI;EAAQ,CAAC;CAC9E;CACA,OAAO;AACT;;;;;;AAOA,SAAS,cAAc,MAAgB;CACrC,MAAM,wBAAQ,IAAI,IAAoB;CACtC,KAAK,MAAM,OAAO,MAChB,KAAK,MAAM,KAAK,IAAI,SAAS;EAC3B,IAAI,EAAE,YAAY,SAAS,EAAE,iBAAiB,IAAI,OAAO;EACzD,MAAM,MAAM,MAAM,IAAI,EAAE,EAAE;EAC1B,IAAI,CAAC,OAAO,EAAE,eAAe,KAAK,MAAM,IAAI,EAAE,IAAI,EAAE,YAAY;CAClE;CAEF,OAAO;AACT;;AAGA,SAAS,YAAY,GAAsB;CACzC,MAAM,EAAE,cAAc,GAAG,SAAS,IAAI,GAAG,SAAS;CAClD,OAAO;EAAE,GAAG;EAAM,UAAU,aAAa,EAAE,KAAK,CAAC;CAAS;AAC5D;;AAGA,eAAe,cACb,QACA,QACA,YACmB;CACnB,MAAM,QAAQ,cAAc,MAAM,UAAU,MAAM,UAAU,QAAQ,OAAO,KAAK,CAAC,CAAC;CAClF,MAAM,uBAAO,IAAI,IAAY;CAC7B,OAAO,YACL,WAAW,SAAS,MAAgB;EAClC,IAAI,KAAK,IAAI,EAAE,EAAE,GAAG,OAAO,CAAC;EAC5B,KAAK,IAAI,EAAE,EAAE;EACb,MAAM,MAAM,MAAM,IAAI,EAAE,EAAE;EAC1B,OAAO,CACL;GACE,GAAG,YAAY;IAAE,GAAG;IAAG,cAAc;GAAG,CAAC;GACzC,cAAc,OAAO,MAAM,OAAO,QAAQ,MAAM,OAAO;EACzD,CACF;CACF,CAAC,CACH;AACF;AAEA,MAAM,OAAU,IAAe,KAAa,WACzC,GAAG,MAAM,KAAK,MAAM,CAAC,CAAC,KAAK,MAAwB;AAEtD,SAAS,UAAU,IAAe,QAA8D;CAC9F,OAAO;EACL,SAAS,SAAS,SAAS,OAAO,gBAChC,WAAW,OACP,OACA,IACE,IACA;oGAEA;GAAC;GAAS;GAAS;GAAO;GAAa;EAAM,CAC/C;EACN,QAAQ,SAAS,SAAS,gBACxB,IACE,IACA;yFAEA;GAAC;GAAS;GAAS;EAAW,CAChC;EACF,SAAS,SAAS,SAAS,OAAO,gBAAgB;GAQhD,OAPa,GAAG,MACd;;;0CAIA;IAAC;IAAS;IAAS;IAAO;GAAW,CACvC,CAAC,CAAC,KACU,QACT,GAAG,OAAO;IACT,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,SAAS,EAAE,OAAO,KAAK,EAAE,OAAO,IAAI,IAAI;IACjD,MAAM,EAAE,OAAO,EAAE;IACjB,MAAM,EAAE,OAAO,EAAE;GACnB,IACA;IAAE,MAAM;IAAG,OAAO;IAAG,MAAM;IAAG,MAAM;GAAE,CACxC;EACF;CACF;AACF;;;;;;AAOA,SAAS,QAAQ,MAAc;CAC7B,MAAM,wBAAQ,IAAI,IAAoC;CACtD,MAAM,QAAQ,SAAiB;EAC7B,IAAI,IAAI,MAAM,IAAI,IAAI;EACtB,IAAI,CAAC,GAAG;GACN,IAAI,SAAS,KAAK,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,YAAY,IAAI;GACvD,MAAM,IAAI,MAAM,CAAC;EACnB;EACA,OAAO;CACT;CACA,MAAM,wBAAQ,IAAI,IAAoB;CACtC,OAAO;EACL,MAAM,QAAQ,OAAkB,SAA+B;GAC7D,MAAM,UAAU,MAAc;IAAC,IAAI,EAAE;IAAI,IAAI,EAAE;IAAI,KAAK,EAAE;GAAG;GAC7D,MAAM,QAAQ,OAAO,UAAkB,MAAM,KAAK,IAAI,EAAA,EAAI,MAAM,IAAI,KAAK,CAAC;GAC1E,MAAM,MAAM,MAAM,MAAM,MAAM,IAAI;GAClC,MAAM,OAAO,OAAO,MAAM,QAAQ,QAAQ,MAAM,IAAI;GACpD,MAAM,WAAW,KAAK,IACpB,GACA,IAAI,WAAW,SAAS,KAAK,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAC5D;GACA,MAAM,QAA+C;IACnD;KAAE,MAAM,MAAM;KAAM,MAAM;IAAS;IACnC;KAAE,MAAM,MAAM;KAAM,MAAM;IAAE;IAC5B,GAAG,MAAM,QACN,QAAQ,MAAM,EAAE,SAAS,MAAM,IAAI,CAAC,CACpC,KAAK,OAAO;KAAE,MAAM,EAAE;KAAM,MAAM;IAAE,EAAE;GAC3C;GACA,KAAK,MAAM,UAAU,SAAS;IAC5B,MAAM,MAAM,GAAG,MAAM,GAAG,IAAI,UAAU;IACtC,IAAI,MAAM,IAAI,GAAG,GAAG;IAEpB,MAAM,QACJ,WAAW,OACP,CAAC,IAAI,IACL;KAAC,CAAC,GAAG,OAAO,IAAI,GAAG,OAAO,MAAM,CAAC,CAAC,KAAK,MAAM,GAAG,EAAE,EAAE,CAAC;KAAG,OAAO,MAAM;KAAG,CAAC,MAAM;IAAC;IACtF,IAAI,KAAK,UAAU,MAAM;IACzB,QAAQ,KAAK,MAAM,UAAU,OAC3B,KAAK,MAAM,EAAE,MAAM,UAAU,WAAW,OAAO,MAAM,MAAM,GAAG,CAAC,IAAI,OACjE,KAAK,MAAM,CAAC,GAAG,UAAU,MAAM,MAAM,IAAI,EAAA,CAAG,QAAQ,GAClD,IAAI,KAAK,QAAQ,OAAO,MAAM,MAAM,KAAK,SAAS,CAAC,CAAC,GAAG;KACrD,KAAK,UAAU,KAAK,IAAI,IAAI;KAC5B,MAAM;IACR;IAIN,MAAM,IAAI,KAAK,EAAE;GACnB;EACF;EACA,SAAS,OAAkB,WACzB,MAAM,IAAI,GAAG,MAAM,GAAG,IAAI,UAAU,IAAI,KAAK,UAAU,MAAM;CACjE;AACF;;;;;AAMA,SAAS,UAAU,OAAwC;CACzD,MAAM,SAAS,MAAM,MAAM,SAAS,OAAO,EAAE,UAAU,CAAC,EAAA,CAAG,KAAK,MAAM,EAAE,KAAK,CAAC;CAC9E,MAAM,UAAU,MAAM,SAAS,SAAS,MACtC,EAAE,MAAM,SAAS,OAAO,EAAE,cAAc,CAAC,EAAA,CAAG,KAAK,MAAM,EAAE,OAAO,MAAM,GAAG,CAAC,CAAC,EAAY,CAAC,CAC1F;CACA,MAAM,UAAU,MAAM,SAAS,SAAS,OAAO,EAAE,kBAAkB,CAAC,EAAA,CAAG,SAAS,MAAM,EAAE,OAAO,CAAC;CAChG,OAAO,CAAC,MAAM,mBAAG,IAAI,IAAI;EAAC,GAAG;EAAQ,GAAG;EAAS,GAAG;CAAO,CAAC,CAAC;AAC/D;;AAGA,eAAsB,UAAU,QAAwB,QAAsC;CAE5F,MAAM,MAAK,MADQ,UAAU,QAAQ,OAAO,KAAK,EAAA,CACjC,SAAS,MAAM,cAAc,OAAO,MAAM,OAAO,QAAQ,GAAG,IAAI;CAChF,IAAI;EACF,MAAM,MAAM,QAAQ,OAAO,IAAI;EAC/B,KAAK,MAAM,SAAS,OAAO,OAAO,MAAM,IAAI,QAAQ,OAAO,UAAU,KAAK,CAAC;EAC3E,MAAM,MAAqB;GACzB,SAAS,KAAK,UAAU,IAAI,MAAM,cAAc,OAAO,IAAI,CAAC,IAAI;GAChE,aAAa,OAAO,YAAY;IAC9B,MAAM,OAAO,MAAM,MAAM,MAAM,MAAM,EAAE,QAAQ,OAAO;IACtD,OAAO,OAAO,sBAAsB,MAAM,MAAM,QAAQ,IAAI;GAC9D;GACA,QAAQ,IAAI;EACd;EACA,MAAM,aAAa,qBAAqB,YAAY,MAAM,GAAG,QAAQ,GAAG;EACxE,OAAO,WAAW,SAAS,MAAM,cAAc,QAAQ,QAAQ,UAAU,IAAI,CAAC;CAChF,UAAU;EACR,IAAI,MAAM;CACZ;AACF;;AAGA,SAAgB,WAAW,QAAmB,SAA+B;CAC3E,OAAO;EACL,MAAM;EACN,eAAe;EACf,OAAO,OAAO;EACd,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;EAClC;CACF;AACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { et as cellEffort } from "./model-CTnyvh7H.js";
|
|
2
2
|
//#region src/view/header.ts
|
|
3
3
|
/** The derived header of one run: what listings, indexes and comparisons need without the record. */
|
|
4
4
|
function runHeader(record, recorded) {
|
|
@@ -40,6 +40,8 @@ function runHeader(record, recorded) {
|
|
|
40
40
|
slug: e.slug,
|
|
41
41
|
fingerprint: e.fingerprint,
|
|
42
42
|
sources: e.sources,
|
|
43
|
+
...e.source ? { source: e.source } : {},
|
|
44
|
+
...e.revision !== void 0 ? { revision: e.revision } : {},
|
|
43
45
|
outcome: e.outcome,
|
|
44
46
|
passRate: e.passRate,
|
|
45
47
|
cells: e.cells.map((c) => ({
|
|
@@ -61,7 +63,7 @@ function runHeader(record, recorded) {
|
|
|
61
63
|
passRate: c.passRate,
|
|
62
64
|
outcome: c.outcome,
|
|
63
65
|
effort: cellEffort(e, c.key),
|
|
64
|
-
...
|
|
66
|
+
...canceledIn(e, c.key),
|
|
65
67
|
...c.lift ? { lift: {
|
|
66
68
|
vs: c.lift.vs,
|
|
67
69
|
points: c.lift.points,
|
|
@@ -71,22 +73,45 @@ function runHeader(record, recorded) {
|
|
|
71
73
|
} } : {}
|
|
72
74
|
}))
|
|
73
75
|
})),
|
|
74
|
-
...record.families?.length ? { families: record.families.map(({ id, name, cases, ran, outcome }) => ({
|
|
76
|
+
...record.families?.length ? { families: record.families.map(({ id, name, cases, ran, outcome, lifts }) => ({
|
|
75
77
|
id,
|
|
76
78
|
name,
|
|
77
79
|
cases,
|
|
78
80
|
ran,
|
|
79
|
-
outcome
|
|
81
|
+
outcome,
|
|
82
|
+
caseIds: record.evals.filter((e) => e.family?.id === id).map((e) => e.id),
|
|
83
|
+
lifts: lifts.map((l) => ({
|
|
84
|
+
variant: l.variant,
|
|
85
|
+
vs: l.vs,
|
|
86
|
+
points: l.points,
|
|
87
|
+
lo: l.interval.lo,
|
|
88
|
+
hi: l.interval.hi,
|
|
89
|
+
...l.minLift !== void 0 ? { minLift: l.minLift } : {},
|
|
90
|
+
outcome: l.outcome
|
|
91
|
+
}))
|
|
80
92
|
})) } : {},
|
|
81
93
|
...record.errors?.length ? { errors: record.errors } : {}
|
|
82
94
|
};
|
|
83
95
|
}
|
|
84
|
-
/** A cell's failed attempts that the run's interrupt
|
|
85
|
-
function
|
|
96
|
+
/** A cell's failed attempts that the run's interrupt canceled, when it has any. */
|
|
97
|
+
function canceledIn(e, cell) {
|
|
86
98
|
const n = e.attempts.filter((a) => a.cell === cell && a.outcome === "fail" && a.error?.kind === "aborted").length;
|
|
87
|
-
return n ? {
|
|
99
|
+
return n ? { canceled: n } : {};
|
|
88
100
|
}
|
|
89
101
|
//#endregion
|
|
90
|
-
|
|
102
|
+
//#region src/view/remarks.ts
|
|
103
|
+
/** `<history>/<run>/remarks.json`, beside the record. */
|
|
104
|
+
const REMARKS_FILE = "remarks.json";
|
|
105
|
+
/** Each remark kind's severity. */
|
|
106
|
+
const REMARK_KINDS = {
|
|
107
|
+
"too-few-tries": { severity: "suggestion" },
|
|
108
|
+
"check-never-fails": { severity: "suggestion" },
|
|
109
|
+
"flaky-check": { severity: "concern" },
|
|
110
|
+
"slow-mcp-start": { severity: "concern" },
|
|
111
|
+
"proxy-only-egress": { severity: "concern" },
|
|
112
|
+
"setup-process-secret": { severity: "concern" }
|
|
113
|
+
};
|
|
114
|
+
//#endregion
|
|
115
|
+
export { REMARK_KINDS as n, runHeader as r, REMARKS_FILE as t };
|
|
91
116
|
|
|
92
|
-
//# sourceMappingURL=
|
|
117
|
+
//# sourceMappingURL=remarks-D_3qhBWU.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remarks-D_3qhBWU.js","names":[],"sources":["../src/view/header.ts","../src/view/remarks.ts"],"mappings":";;;AAIA,SAAgB,UAAU,QAAmB,UAAiC;CAC5E,OAAO;EACL,OAAO,OAAO;EACd;EACA,QAAQ,OAAO;EACf,WAAW,OAAO;EAClB,YAAY,OAAO;EACnB,QAAQ,OAAO;EACf,KAAK,OAAO,MACR;GAAE,KAAK,OAAO,IAAI;GAAK,QAAQ,OAAO,IAAI;GAAQ,OAAO,OAAO,IAAI;EAAM,IAC1E;EACJ,QAAQ,OAAO,OAAO;EACtB,IACE,OAAO,OAAO,SAAS,OACnB;GACE,UAAU,OAAO,OAAO;GACxB,QAAQ,OAAO,OAAO;GACtB,IAAI,OAAO,OAAO;EACpB,IACA;EACN,WAAW,OAAO,MAAM,UAAU,KAAK,MAAM,EAAE,EAAE;EACjD,SAAS,OAAO;EAChB,OAAO,OAAO,KAAK;EACnB,OAAO;GACL,UAAU,OAAO,MAAM,OAAO,KAAK,MAAM,EAAE,EAAE;GAC7C,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,MAAM,OAAO,SAAS,MAAO,EAAE,WAAW,CAAC,EAAE,QAAQ,IAAI,CAAC,CAAE,CAAC,CAAC;GAC5F,WAAW,OAAO,MAAM,UAAU,KAAK,MAAM,EAAE,EAAE;GACjD,cAAc,OAAO,aAAa,KAC/B,MAAM,GAAG,EAAE,KAAK,GAAG,EAAE,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,GAClE;GACA,YAAY,OAAO,MAAM,MAAM;GAC/B,aAAa,OAAO,MAAM,MAAM;GAChC,cAAc,OAAO,MAAM,MAAM;GACjC,WAAW,OAAO,MAAM,MAAM;GAC9B,kBAAkB,OAAO,MAAM;EACjC;EACA,OAAO,OAAO,MAAM,KAAK,OAAO;GAC9B,IAAI,EAAE;GACN,KAAK,EAAE;GACP,MAAM,EAAE;GACR,aAAa,EAAE;GACf,SAAS,EAAE;GACX,GAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,IAAI,CAAC;GACvC,GAAI,EAAE,aAAa,KAAA,IAAY,EAAE,UAAU,EAAE,SAAS,IAAI,CAAC;GAC3D,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,OAAO,EAAE,MAAM,KAAK,OAAO;IACzB,KAAK,EAAE;IACP,MAAM,EAAE;IACR,GAAI,EAAE,YAAY,KAAA,IAAY,EAAE,SAAS,EAAE,QAAQ,IAAI,CAAC;IACxD,SAAS,EAAE;IACX,OAAO,EAAE;IACT,aAAa,EAAE;IACf,mBAAmB,EAAE;IACrB,MAAM,EAAE;IACR,aAAa,EAAE;IACf,kBAAkB,EAAE;IACpB,MAAM,EAAE;IACR,MAAM,EAAE;IACR,MAAM,EAAE;IACR,YAAY,EAAE;IACd,SAAS,EAAE;IACX,UAAU,EAAE;IACZ,SAAS,EAAE;IACX,QAAQ,WAAW,GAAG,EAAE,GAAG;IAC3B,GAAG,WAAW,GAAG,EAAE,GAAG;IACtB,GAAI,EAAE,OACF,EACE,MAAM;KACJ,IAAI,EAAE,KAAK;KACX,QAAQ,EAAE,KAAK;KACf,IAAI,EAAE,KAAK,SAAS;KACpB,IAAI,EAAE,KAAK,SAAS;KACpB,aAAa,EAAE,KAAK;IACtB,EACF,IACA,CAAC;GACP,EAAE;EACJ,EAAE;EACF,GAAI,OAAO,UAAU,SACjB,EACE,UAAU,OAAO,SAAS,KAAK,EAAE,IAAI,MAAM,OAAO,KAAK,SAAS,aAAa;GAC3E;GACA;GACA;GACA;GACA;GACA,SAAS,OAAO,MAAM,QAAQ,MAAM,EAAE,QAAQ,OAAO,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,EAAE;GACxE,OAAO,MAAM,KAAK,OAAO;IACvB,SAAS,EAAE;IACX,IAAI,EAAE;IACN,QAAQ,EAAE;IACV,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,SAAS;IACf,GAAI,EAAE,YAAY,KAAA,IAAY,EAAE,SAAS,EAAE,QAAQ,IAAI,CAAC;IACxD,SAAS,EAAE;GACb,EAAE;EACJ,EAAE,EACJ,IACA,CAAC;EACL,GAAI,OAAO,QAAQ,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;CAC3D;AACF;;AAGA,SAAS,WAAW,GAA+B,MAAqC;CACtF,MAAM,IAAI,EAAE,SAAS,QAClB,MAAM,EAAE,SAAS,QAAQ,EAAE,YAAY,UAAU,EAAE,OAAO,SAAS,SACtE,CAAC,CAAC;CACF,OAAO,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC;AAChC;;;;AC7GA,MAAa,eAAe;;AAgB5B,MAAa,eAA+E;CAC1F,iBAAiB,EAAE,UAAU,aAAa;CAC1C,qBAAqB,EAAE,UAAU,aAAa;CAC9C,eAAe,EAAE,UAAU,UAAU;CACrC,kBAAkB,EAAE,UAAU,UAAU;CACxC,qBAAqB,EAAE,UAAU,UAAU;CAC3C,wBAAwB,EAAE,UAAU,UAAU;AAChD"}
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import { n as clip } from "./text-
|
|
1
|
+
import { n as clip } from "./text-DTigcA64.js";
|
|
2
2
|
/** Kinds about what happened in this run, not the eval's source: they print in every run they happen. */
|
|
3
|
-
const EVERY_RUN = [
|
|
3
|
+
const EVERY_RUN = [
|
|
4
|
+
"slow-mcp-start",
|
|
5
|
+
"proxy-only-egress",
|
|
6
|
+
"setup-process-secret"
|
|
7
|
+
];
|
|
4
8
|
/** Whether a remark may print in this run: no earlier run printed it, or it's one of `EVERY_RUN`. */
|
|
5
9
|
const printsIn = (r, runId) => r.firstSeenRun === runId || EVERY_RUN.includes(r.kind);
|
|
6
10
|
/** The remarks the section prints, in order, and how many concerns the cap left out. */
|
|
@@ -23,8 +27,8 @@ function markPrinted(remarks, runId, verbose) {
|
|
|
23
27
|
}
|
|
24
28
|
/** `evals/a.eval.ts:12` from `source:evals/a.eval.ts#L12`. */
|
|
25
29
|
const location = (at) => at.replace(/^source:/, "").replace(/#L(\d+)$/, ":$1");
|
|
26
|
-
function remarkLines(c, suite,
|
|
27
|
-
const { shown, hidden } = shownRemarks(
|
|
30
|
+
function remarkLines(c, suite, remarks, verbose) {
|
|
31
|
+
const { shown, hidden } = shownRemarks(remarks?.remarks ?? [], suite.runId, verbose);
|
|
28
32
|
if (!shown.length) return [];
|
|
29
33
|
const names = new Map(suite.files.flatMap((f) => f.evals.map((e) => [e.path, [...e.group, e.template ?? e.name].join(" › ")])));
|
|
30
34
|
return [
|
|
@@ -37,4 +41,4 @@ function remarkLines(c, suite, insights, verbose) {
|
|
|
37
41
|
//#endregion
|
|
38
42
|
export { shownRemarks as i, printsIn as n, remarkLines as r, markPrinted as t };
|
|
39
43
|
|
|
40
|
-
//# sourceMappingURL=remarks-
|
|
44
|
+
//# sourceMappingURL=remarks-dA8LyoPc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remarks-dA8LyoPc.js","names":[],"sources":["../src/ui/remarks.ts"],"mappings":";;AAeA,MAAM,YAAuC;CAC3C;CACA;CACA;AACF;;AAGA,MAAa,YAAY,GAAW,UAClC,EAAE,iBAAiB,SAAS,UAAU,SAAS,EAAE,IAAI;;AAGvD,SAAgB,aAAa,SAAmB,OAAe,SAAkB;CAC/E,MAAM,YAAY,QAAQ,QAAQ,MAAM,SAAS,GAAG,KAAK,CAAC;CAC1D,MAAM,WAAW,UAAU,QAAQ,MAAM,EAAE,aAAa,SAAS;CACjE,MAAM,QAAQ,UAAU,YAAY,SAAS,MAAM,GAAA,CAAa;CAChE,OAAO;EAAE;EAAO,QAAQ,UAAU,IAAI,SAAS,SAAS,MAAM;CAAO;AACvE;;AAGA,SAAgB,YAAY,SAAmB,OAAe,SAA4B;CACxF,MAAM,QAAQ,IAAI,IAAI,aAAa,SAAS,OAAO,OAAO,CAAC,CAAC,KAAK;CACjE,OAAO,QAAQ,KAAK,OAAO;EAAE,GAAG;EAAG,SAAS,MAAM,IAAI,CAAC;CAAE,EAAE;AAC7D;;AAGA,MAAM,YAAY,OAAe,GAAG,QAAQ,YAAY,EAAE,CAAC,CAAC,QAAQ,YAAY,KAAK;AAErF,SAAgB,YACd,GACA,OACA,SACA,SACU;CACV,MAAM,EAAE,OAAO,WAAW,aAAa,SAAS,WAAW,CAAC,GAAG,MAAM,OAAO,OAAO;CACnF,IAAI,CAAC,MAAM,QAAQ,OAAO,CAAC;CAC3B,MAAM,QAAQ,IAAI,IAChB,MAAM,MAAM,SAAS,MACnB,EAAE,MAAM,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAU,CACtF,CACF;CACA,OAAO;EACL;EACA;EACA,GAAG,MAAM,SAAS,MAAM,CACtB,KAAK,MAAM,IAAI,EAAE,MAAM,KAAK,EAAE,OAAO,IAAI,EAAE,QAC3C,EAAE,IAAI,OACF,OAAO,EAAE,IAAI,KAAK,EAAE,IAAI,SAAS,EAAE,IAAI,EAAE,EAAE,IAAI,KAAK,EAAE,IAAI,MAAM,GAAG,MACnE,OAAO,EAAE,IAAI,IAAI,EAAE,KAAK,SAAS,EAAE,IAAI,EAAE,GAC/C,CAAC;EACD,GAAI,SAAS,IAAI,CAAC,EAAE,IAAI,MAAM,OAAO,WAAW,CAAC,IAAI,CAAC;CACxD;AACF"}
|