reeval 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -39
- package/THIRD_PARTY_NOTICES.md +75 -106
- package/bin/mcp-timer.cjs +224 -0
- package/dist/{adapter-DWlpOtRn.js → adapter-DigeVFWC.js} +3 -3
- package/dist/{adapter-DWlpOtRn.js.map → adapter-DigeVFWC.js.map} +1 -1
- package/dist/{agent-CaS_2Sce.js → agent-CatUEo5h.js} +21 -6
- package/dist/{agent-CaS_2Sce.js.map → agent-CatUEo5h.js.map} +1 -1
- package/dist/{assemble-B9Alds-z.js → assemble-CF063ms6.js} +19 -17
- package/dist/assemble-CF063ms6.js.map +1 -0
- package/dist/assemble-CJoD_Rzr.js +1 -0
- package/dist/bridge/index.mjs +15 -7
- package/dist/{capture-CNb9SuU5.js → capture-BiD7lVaM.js} +26 -18
- package/dist/{capture-CNb9SuU5.js.map → capture-BiD7lVaM.js.map} +1 -1
- package/dist/capture-Dp3q7K7Y.js +1 -0
- package/dist/{cli-D303Vj5N.js → cli-B5zQCR2r.js} +266 -547
- package/dist/cli-B5zQCR2r.js.map +1 -0
- package/dist/{cli-spec-BPZbQG2p.js → cli-spec-BXIBUlaK.js} +46 -18
- package/dist/cli-spec-BXIBUlaK.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{command-BT9a37ZR.js → command-BMHXHqMT.js} +17 -14
- package/dist/command-BMHXHqMT.js.map +1 -0
- package/dist/command-C94CQNh8.js +91 -0
- package/dist/command-C94CQNh8.js.map +1 -0
- package/dist/command-CVtgyqlM.js +335 -0
- package/dist/command-CVtgyqlM.js.map +1 -0
- package/dist/{command-BLfT7vnQ.js → command-CaZgCTIw.js} +13 -17
- package/dist/command-CaZgCTIw.js.map +1 -0
- package/dist/{command-CMJtHKRa.js → command-DTgBZR0k.js} +6 -6
- package/dist/{command-CMJtHKRa.js.map → command-DTgBZR0k.js.map} +1 -1
- package/dist/{commands-1p-bIWhN.js → commands-BXzIDB3X.js} +1 -1
- package/dist/commands-CjpGBUWr.js +666 -0
- package/dist/commands-CjpGBUWr.js.map +1 -0
- package/dist/{config-DEgnUiU-.js → config-Dv5MG7Rp.js} +31 -34
- package/dist/config-Dv5MG7Rp.js.map +1 -0
- package/dist/{context-C1mxXh80.js → context-Dgn48BqT.js} +13 -2
- package/dist/context-Dgn48BqT.js.map +1 -0
- package/dist/contrasts-CcoMLC_y.js +101 -0
- package/dist/contrasts-CcoMLC_y.js.map +1 -0
- package/dist/db-BeqRiWLl.js +602 -0
- package/dist/db-BeqRiWLl.js.map +1 -0
- package/dist/db-CJDRVQal.js +2 -0
- package/dist/{dist-B8hJImYt.js → dist-CMjp3qqF.js} +81 -14
- package/dist/dist-CMjp3qqF.js.map +1 -0
- package/dist/{docker-sZrNrr82.js → docker-BbTlMmYi.js} +2 -2
- package/dist/{docker-sZrNrr82.js.map → docker-BbTlMmYi.js.map} +1 -1
- package/dist/{docker-7b_2SaH7.js → docker-CCXdbREk.js} +20 -4
- package/dist/{docker-7b_2SaH7.js.map → docker-CCXdbREk.js.map} +1 -1
- package/dist/{docker-yauK-9X8.js → docker-DC69dKxB.js} +1 -1
- package/dist/{doctor-BYNok4UO.js → doctor-CqttrZcs.js} +56 -22
- package/dist/doctor-CqttrZcs.js.map +1 -0
- package/dist/{egress-C4PZTSSe.js → egress-BfK_O9ce.js} +435 -31
- package/dist/egress-BfK_O9ce.js.map +1 -0
- package/dist/{environment-DNjRk0nU.js → environment-BBz1Yes9.js} +225 -12
- package/dist/environment-BBz1Yes9.js.map +1 -0
- package/dist/{evaluate-D4zeRGBF.js → evaluate-CXQ1o-pd.js} +539 -562
- package/dist/evaluate-CXQ1o-pd.js.map +1 -0
- package/dist/{evaluate-C1UVL21l.d.ts → evaluate-yjZKUexW.d.ts} +2 -2
- package/dist/facts-DtENlf9s.js +898 -0
- package/dist/facts-DtENlf9s.js.map +1 -0
- package/dist/{files-BXUdY8tr.js → files-B-eUpCZ1.js} +30 -7
- package/dist/files-B-eUpCZ1.js.map +1 -0
- package/dist/files-BsQbYz2i.js +65 -0
- package/dist/files-BsQbYz2i.js.map +1 -0
- package/dist/{files-DQmHE7pb.js → files-COHQLEHE.js} +6 -17
- package/dist/files-COHQLEHE.js.map +1 -0
- package/dist/files-DqCBW7i8.js +1 -0
- package/dist/files-Ha-jsQXf.js +1 -0
- package/dist/generate-Cs2TMH_k.js +1647 -0
- package/dist/generate-Cs2TMH_k.js.map +1 -0
- package/dist/{git-DfG-YJ1u.js → git-DhykwgKF.js} +2 -2
- package/dist/{git-DfG-YJ1u.js.map → git-DhykwgKF.js.map} +1 -1
- package/dist/{harness-B8QC9u0O.js → harness-BAjw1fcr.js} +3 -3
- package/dist/{harness-B8QC9u0O.js.map → harness-BAjw1fcr.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/{header-hK-Z_Po9.js → header-kSmWatcO.js} +10 -3
- package/dist/{header-hK-Z_Po9.js.map → header-kSmWatcO.js.map} +1 -1
- package/dist/{images-0On0cztu.js → images-BmJRuTpm.js} +61 -13
- package/dist/images-BmJRuTpm.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +4 -4
- package/dist/init-Bwc9byQr.js +543 -0
- package/dist/init-Bwc9byQr.js.map +1 -0
- package/dist/init-D7Y1ny1z.js +1 -0
- package/dist/insights-B2Xmhz9o.js +64 -0
- package/dist/insights-B2Xmhz9o.js.map +1 -0
- package/dist/{invocations-DbGfBAA_.js → invocations-CQTbP092.js} +3 -3
- package/dist/{invocations-DbGfBAA_.js.map → invocations-CQTbP092.js.map} +1 -1
- package/dist/{judge-DW_GDOmB.js → judge-Dt6KW6_O.js} +5 -3
- package/dist/judge-Dt6KW6_O.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +8 -7
- package/dist/judge.js.map +1 -1
- package/dist/{live-jlfszAbb.js → live-DYd2QEyR.js} +2 -2
- package/dist/{live-jlfszAbb.js.map → live-DYd2QEyR.js.map} +1 -1
- package/dist/{model-h_cDtcRr.js → model-B-mel-Gv.js} +46 -20
- package/dist/model-B-mel-Gv.js.map +1 -0
- package/dist/model-BGKYVpoj.js +1 -0
- package/dist/{model-DaVCY3rb.js → model-C7tSwT7f.js} +2 -2
- package/dist/{model-DaVCY3rb.js.map → model-C7tSwT7f.js.map} +1 -1
- package/dist/{model-oqcDplJG.js → model-CWlAqmbU.js} +150 -46
- package/dist/model-CWlAqmbU.js.map +1 -0
- package/dist/model-DHY-dfoO.js +1 -0
- package/dist/{node-B3ydHo6t.js → node-D0mtjRAO.js} +3 -3
- package/dist/{node-B3ydHo6t.js.map → node-D0mtjRAO.js.map} +1 -1
- package/dist/{options-Bh2W63dK.js → options-CPUW97_7.js} +9 -12
- package/dist/options-CPUW97_7.js.map +1 -0
- package/dist/{pairs-CnInRu-G.js → pairs-DP6XOJby.js} +2 -2
- package/dist/{pairs-CnInRu-G.js.map → pairs-DP6XOJby.js.map} +1 -1
- package/dist/{patch-Zac5aIRM.js → patch-D4oluWzH.js} +2 -2
- package/dist/{patch-Zac5aIRM.js.map → patch-D4oluWzH.js.map} +1 -1
- package/dist/{plan-C9Cy27Ay.js → plan-OkwUcJQ9.js} +31 -12
- package/dist/plan-OkwUcJQ9.js.map +1 -0
- package/dist/plugins-CUWMnSNb.js +654 -0
- package/dist/plugins-CUWMnSNb.js.map +1 -0
- package/dist/{preflight-CPWpQeBI.js → preflight-B6pY89bF.js} +1 -1
- package/dist/{preflight-D7iFPKeK.js → preflight-Bjm7_ylE.js} +14 -10
- package/dist/preflight-Bjm7_ylE.js.map +1 -0
- package/dist/{prepare-doGxaAGR.js → prepare-CQmK3SBn.js} +71 -24
- package/dist/prepare-CQmK3SBn.js.map +1 -0
- package/dist/prepare-DbdgtZKe.js +2 -0
- package/dist/{provenance-DmE9uPVt.js → provenance-B3Y17oCC.js} +2 -2
- package/dist/{provenance-DmE9uPVt.js.map → provenance-B3Y17oCC.js.map} +1 -1
- package/dist/providers-CXN2RFz-.js +1 -0
- package/dist/{providers-tAOD1NQN.js → providers-eoniqK6w.js} +24 -10
- package/dist/providers-eoniqK6w.js.map +1 -0
- package/dist/query-CLRZmWBT.js +139 -0
- package/dist/query-CLRZmWBT.js.map +1 -0
- package/dist/query-CmBDYt0_.js +1 -0
- package/dist/{questions-DjAwMxZ3.js → questions-1XOulevV.js} +6 -6
- package/dist/{questions-DjAwMxZ3.js.map → questions-1XOulevV.js.map} +1 -1
- package/dist/{record-BKmMvZ6O.js → record-kk9M1hj-.js} +8 -4
- package/dist/record-kk9M1hj-.js.map +1 -0
- package/dist/remarks-1521zwfW.js +38 -0
- package/dist/remarks-1521zwfW.js.map +1 -0
- package/dist/remarks-BhAV1H4V.js +1 -0
- package/dist/remarks-DlJCqWRs.js +278 -0
- package/dist/remarks-DlJCqWRs.js.map +1 -0
- package/dist/{remote-HDRt16ew.js → remote-cE2VWfkJ.js} +3 -3
- package/dist/{remote-HDRt16ew.js.map → remote-cE2VWfkJ.js.map} +1 -1
- package/dist/resolve-C4zeccCH.js +724 -0
- package/dist/resolve-C4zeccCH.js.map +1 -0
- package/dist/resolve-D30yZYmB.js +1 -0
- package/dist/{results-DuauQI3Z.js → results-Drtt0uFl.js} +33 -7
- package/dist/results-Drtt0uFl.js.map +1 -0
- package/dist/{resume-BRfa8E_s.js → resume-BEFazUfy.js} +4 -4
- package/dist/{resume-BRfa8E_s.js.map → resume-BEFazUfy.js.map} +1 -1
- package/dist/rules-BFzQqPIm.js +317 -0
- package/dist/rules-BFzQqPIm.js.map +1 -0
- package/dist/run-BeMy92Us.js +88 -0
- package/dist/run-BeMy92Us.js.map +1 -0
- package/dist/{sandbox-D3A8oUb7.js → sandbox-CQaJj7n9.js} +2 -2
- package/dist/{sandbox-D3A8oUb7.js.map → sandbox-CQaJj7n9.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-V4-2w0l8.js +1 -0
- package/dist/{server-XDleKaF_.js → server-xK0les4f.js} +8 -8
- package/dist/{server-XDleKaF_.js.map → server-xK0les4f.js.map} +1 -1
- package/dist/{shared-DYpIb1BI.js → shared-CA61BuPJ.js} +25 -3
- package/dist/shared-CA61BuPJ.js.map +1 -0
- package/dist/source-BDow_oW8.js +1 -0
- package/dist/{source-C8xumIWj.js → source-Q1hEKbCs.js} +2 -2
- package/dist/{source-C8xumIWj.js.map → source-Q1hEKbCs.js.map} +1 -1
- package/dist/src-B9L9yGKI.js +1 -0
- package/dist/{src-ofUaYbuY.js → src-Bcah6ZkT.js} +1 -1
- package/dist/{src-DvCg0NBs.js → src-BhmaorOU.js} +28 -6
- package/dist/{src-DvCg0NBs.js.map → src-BhmaorOU.js.map} +1 -1
- package/dist/{src-CF17Ug08.js → src-C0CeKiLU.js} +19 -4
- package/dist/{src-CF17Ug08.js.map → src-C0CeKiLU.js.map} +1 -1
- package/dist/src-C0xowZ3C.js +2 -0
- package/dist/src-CHIfIKUu.js +2 -0
- package/dist/src-DLtyz4w5.js +2 -0
- package/dist/src-DPIjmReZ.js +2 -0
- package/dist/{src-C_lPtowT.js → src-DXNh5mTr.js} +1 -1
- package/dist/{src-FGXhkEkH.js → src-DiLMkIr9.js} +142 -13
- package/dist/src-DiLMkIr9.js.map +1 -0
- package/dist/{src-B7treWa8.js → src-LekQrr_S.js} +324 -32
- package/dist/src-LekQrr_S.js.map +1 -0
- package/dist/src-bCnsA6Vp.js +2 -0
- package/dist/{src-u8_ixn_F.js → src-cbCb-MPy.js} +20 -4
- package/dist/src-cbCb-MPy.js.map +1 -0
- package/dist/{src-DvK0EIZx.js → src-qGoFkuJF.js} +55 -6
- package/dist/{src-DvK0EIZx.js.map → src-qGoFkuJF.js.map} +1 -1
- package/dist/{src-jzagbuqH.js → src-ynBVosfX.js} +1 -1
- package/dist/state-BQR_BCm4.js +1 -0
- package/dist/{state-C__9swoG.js → state-BmZ8QVZ8.js} +12 -12
- package/dist/{state-C__9swoG.js.map → state-BmZ8QVZ8.js.map} +1 -1
- package/dist/studio-app/assets/index-CnYJijyI.js +58 -0
- package/dist/studio-app/index.html +1 -1
- package/dist/suite-BaHPmAGy.js +1 -0
- package/dist/suite-C8sw2pox.js +481 -0
- package/dist/suite-C8sw2pox.js.map +1 -0
- package/dist/summary-0oQNGskv.js +837 -0
- package/dist/summary-0oQNGskv.js.map +1 -0
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +2 -2
- package/dist/{text-gEcKjAyp.js → text-CrVy5ioL.js} +18 -2
- package/dist/text-CrVy5ioL.js.map +1 -0
- package/dist/{transcript-BNNANU8A.js → transcript-BTxJpW4R.js} +2 -2
- package/dist/{transcript-BNNANU8A.js.map → transcript-BTxJpW4R.js.map} +1 -1
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-g17bRAzt.d.ts → types-GraZtfHN.d.ts} +858 -39
- package/dist/usage-BDo2q_yu.js +1 -0
- package/dist/{command-Bge8F7Ko.js → validate-BdoXUuL_.js} +35 -383
- package/dist/validate-BdoXUuL_.js.map +1 -0
- package/dist/view/index.d.ts +20 -9
- package/dist/view/index.js +7 -6
- package/dist/views-BoUoX4_A.js +125 -0
- package/dist/views-BoUoX4_A.js.map +1 -0
- package/dist/views-noVQNCUP.js +2 -0
- package/dist/{worker-entry-yOchGYYv.js → worker-entry-D0-GErmU.js} +2 -2
- package/dist/{worker-entry-yOchGYYv.js.map → worker-entry-D0-GErmU.js.map} +1 -1
- package/dist/{workspace-D8sSgpiy.js → workspace-B90YDU_a.js} +1 -1
- package/dist/{workspace-CgyHEM0l.js → workspace-tjZFkjxC.js} +4 -4
- package/dist/{workspace-CgyHEM0l.js.map → workspace-tjZFkjxC.js.map} +1 -1
- package/package.json +10 -8
- package/skills/eval-rubric/SKILL.md +130 -0
- package/dist/agent-CURAwx_q.js +0 -334
- package/dist/agent-CURAwx_q.js.map +0 -1
- package/dist/assemble-B9Alds-z.js.map +0 -1
- package/dist/assemble-J5MukZFY.js +0 -1
- package/dist/capture-jquXZBKK.js +0 -1
- package/dist/cli-D303Vj5N.js.map +0 -1
- package/dist/cli-spec-BPZbQG2p.js.map +0 -1
- package/dist/command-BLfT7vnQ.js.map +0 -1
- package/dist/command-BT9a37ZR.js.map +0 -1
- package/dist/command-Bge8F7Ko.js.map +0 -1
- package/dist/command-BkFjijN2.js +0 -1
- package/dist/commands-BFx16WB8.js +0 -225
- package/dist/commands-BFx16WB8.js.map +0 -1
- package/dist/config-DEgnUiU-.js.map +0 -1
- package/dist/context-C1mxXh80.js.map +0 -1
- package/dist/db-CdHizc4_.js +0 -1406
- package/dist/db-CdHizc4_.js.map +0 -1
- package/dist/db-Dm1cRJJP.js +0 -2
- package/dist/dist-B8hJImYt.js.map +0 -1
- package/dist/doctor-BYNok4UO.js.map +0 -1
- package/dist/duration-CEOQww97.js +0 -26
- package/dist/duration-CEOQww97.js.map +0 -1
- package/dist/egress-C4PZTSSe.js.map +0 -1
- package/dist/environment-DNjRk0nU.js.map +0 -1
- package/dist/evaluate-D4zeRGBF.js.map +0 -1
- package/dist/family-D2b7WD10.js +0 -67
- package/dist/family-D2b7WD10.js.map +0 -1
- package/dist/files-BXUdY8tr.js.map +0 -1
- package/dist/files-BhyltlDB.js +0 -1
- package/dist/files-DQmHE7pb.js.map +0 -1
- package/dist/files-ZZPqfAo6.js +0 -1
- package/dist/images-0On0cztu.js.map +0 -1
- package/dist/init-B77Rpcqc.js +0 -360
- package/dist/init-B77Rpcqc.js.map +0 -1
- package/dist/init-BrOWin3W.js +0 -1
- package/dist/judge-DW_GDOmB.js.map +0 -1
- package/dist/model-NM3ID0uR.js +0 -1
- package/dist/model-h_cDtcRr.js.map +0 -1
- package/dist/model-nwWJvaUR.js +0 -1
- package/dist/model-oqcDplJG.js.map +0 -1
- package/dist/options-Bh2W63dK.js.map +0 -1
- package/dist/plan-C9Cy27Ay.js.map +0 -1
- package/dist/preflight-D7iFPKeK.js.map +0 -1
- package/dist/prepare-BGjePFQl.js +0 -2
- package/dist/prepare-doGxaAGR.js.map +0 -1
- package/dist/process-CSVO_jsc.js +0 -119
- package/dist/process-CSVO_jsc.js.map +0 -1
- package/dist/providers-gDMko8fl.js +0 -1
- package/dist/providers-tAOD1NQN.js.map +0 -1
- package/dist/query-Bes9Kh6Y.js +0 -91
- package/dist/query-Bes9Kh6Y.js.map +0 -1
- package/dist/query-C_G_svLU.js +0 -1
- package/dist/record-BKmMvZ6O.js.map +0 -1
- package/dist/results-DuauQI3Z.js.map +0 -1
- package/dist/server-C7sg1ndo.js +0 -1
- package/dist/shared-DYpIb1BI.js.map +0 -1
- package/dist/source-fGd0SaKP.js +0 -1
- package/dist/src-B7treWa8.js.map +0 -1
- package/dist/src-DNnG4e4R.js +0 -2
- package/dist/src-DTbrY1hi.js +0 -2
- package/dist/src-DdyJEh2v.js +0 -2
- package/dist/src-Dj9PAroc.js +0 -2
- package/dist/src-Du8H85Qq.js +0 -2
- package/dist/src-FGXhkEkH.js.map +0 -1
- package/dist/src-pJZOrYR9.js +0 -1
- package/dist/src-u8_ixn_F.js.map +0 -1
- package/dist/state-CgjQo29K.js +0 -1
- package/dist/studio-app/assets/index-DrQc2Yxd.js +0 -58
- package/dist/summary-BLAVLySU.js +0 -385
- package/dist/summary-BLAVLySU.js.map +0 -1
- package/dist/summary-DgyjUh_l.js +0 -192
- package/dist/summary-DgyjUh_l.js.map +0 -1
- package/dist/text-gEcKjAyp.js.map +0 -1
- package/dist/tools-GCyNYSc1.js +0 -249
- package/dist/tools-GCyNYSc1.js.map +0 -1
- package/dist/views-BW6oCDUQ.js +0 -2
- package/dist/views-BmQF4UCI.js +0 -225
- package/dist/views-BmQF4UCI.js.map +0 -1
package/dist/summary-BLAVLySU.js
DELETED
|
@@ -1,385 +0,0 @@
|
|
|
1
|
-
import { n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
|
|
2
|
-
import { B as judgeRatesLabel, D as runtimeLabel, F as checkLabels, H as summaryJudgeRates, P as cellUsage, Q as contrastEffort, R as commandLine, S as languageLabel, V as labelledJudgements, Y as cellContrast, Z as cellSamples, _ as harnessLabel, g as formatUsage, ht as round3, l as evalMarkOutcome, tt as effortLabel, ut as liftInterval, x as isUngatedMiss } from "./model-oqcDplJG.js";
|
|
3
|
-
import { a as familyLiftLabel, i as familyHeadLabel, o as familyRateLabel, r as familyEffortLabel } from "./family-D2b7WD10.js";
|
|
4
|
-
import { n as headToHeadLines, r as pairTallies, s as sampleReasons, u as splitsLabel } from "./pairs-CnInRu-G.js";
|
|
5
|
-
import { o as warnings } from "./state-C__9swoG.js";
|
|
6
|
-
//#region src/view/contrasts.ts
|
|
7
|
-
/** Where every counted attempt that answered a check passed it (`all`), none did, or some. */
|
|
8
|
-
const sideOf = (t) => {
|
|
9
|
-
const answered = t.pass + t.fail + t.notRun;
|
|
10
|
-
if (answered === 0) return void 0;
|
|
11
|
-
return t.pass === answered ? "all" : t.pass === 0 ? "none" : "some";
|
|
12
|
-
};
|
|
13
|
-
/** Whether one check tells two cells apart; undefined when either side never answered it. */
|
|
14
|
-
function checkVerdict(a, b) {
|
|
15
|
-
const x = sideOf(a);
|
|
16
|
-
const y = sideOf(b);
|
|
17
|
-
if (x === void 0 || y === void 0) return void 0;
|
|
18
|
-
if (x === "all" && y === "all") return "both-pass";
|
|
19
|
-
if (x === "none" && y === "none") return "both-fail";
|
|
20
|
-
if (x === "all" && y === "none" || x === "none" && y === "all") return "discriminates";
|
|
21
|
-
return "mixed";
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Each unordered pair of cells on one harness and sandbox, when it has at least two variants
|
|
25
|
-
* (with one, the only pair is the variant against its control, which the cell already has).
|
|
26
|
-
* The control comes first, so a pair is `[later, earlier]`: every variant against the control,
|
|
27
|
-
* then each variant against those before it.
|
|
28
|
-
*/
|
|
29
|
-
function contrastPairs(cells) {
|
|
30
|
-
const groups = /* @__PURE__ */ new Map();
|
|
31
|
-
for (const c of cells) {
|
|
32
|
-
const id = `${c.harness}\0${c.sandbox}`;
|
|
33
|
-
groups.set(id, [...groups.get(id) ?? [], c]);
|
|
34
|
-
}
|
|
35
|
-
const pairs = [];
|
|
36
|
-
for (const group of groups.values()) {
|
|
37
|
-
const arms = [...group.filter((c) => c.role === "control"), ...group.filter((c) => c.role !== "control")];
|
|
38
|
-
if (arms.filter((c) => c.role === "variant").length < 2) continue;
|
|
39
|
-
for (let j = 1; j < arms.length; j++) for (let i = 0; i < j; i++) pairs.push([arms[j], arms[i]]);
|
|
40
|
-
}
|
|
41
|
-
return pairs;
|
|
42
|
-
}
|
|
43
|
-
const tally = ({ pass, fail, error, notRun }) => ({
|
|
44
|
-
pass,
|
|
45
|
-
fail,
|
|
46
|
-
error,
|
|
47
|
-
notRun
|
|
48
|
-
});
|
|
49
|
-
function pairLift(a, b) {
|
|
50
|
-
if (a.passRate === null || b.passRate === null) return null;
|
|
51
|
-
const variant = {
|
|
52
|
-
pass: a.pass,
|
|
53
|
-
counted: a.pass + a.fail
|
|
54
|
-
};
|
|
55
|
-
const control = {
|
|
56
|
-
pass: b.pass,
|
|
57
|
-
counted: b.pass + b.fail
|
|
58
|
-
};
|
|
59
|
-
const { points, low, high } = liftInterval({
|
|
60
|
-
pass: variant.pass,
|
|
61
|
-
n: variant.counted
|
|
62
|
-
}, {
|
|
63
|
-
pass: control.pass,
|
|
64
|
-
n: control.counted
|
|
65
|
-
});
|
|
66
|
-
return {
|
|
67
|
-
vs: b.key,
|
|
68
|
-
control,
|
|
69
|
-
variant,
|
|
70
|
-
points: round3(points),
|
|
71
|
-
interval: {
|
|
72
|
-
lo: round3(low),
|
|
73
|
-
hi: round3(high),
|
|
74
|
-
level: .95,
|
|
75
|
-
method: "newcombe-wilson"
|
|
76
|
-
},
|
|
77
|
-
withinNoise: low < 0 && high > 0
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
/** An eval's contrasts: one per pair `contrastPairs` gives; empty with fewer than two variants. */
|
|
81
|
-
function contrastsOf(e) {
|
|
82
|
-
return contrastPairs(e.cells).map(([a, b]) => ({
|
|
83
|
-
a: a.key,
|
|
84
|
-
b: b.key,
|
|
85
|
-
lift: pairLift(a, b),
|
|
86
|
-
effort: contrastEffort(cellSamples(e, a.key), cellSamples(e, b.key)),
|
|
87
|
-
checks: (a.checks ?? []).flatMap((k) => {
|
|
88
|
-
const other = b.checks?.find((x) => x.label === k.label);
|
|
89
|
-
const verdict = other && checkVerdict(k, other);
|
|
90
|
-
return other && verdict ? [{
|
|
91
|
-
label: k.label,
|
|
92
|
-
a: tally(k),
|
|
93
|
-
b: tally(other),
|
|
94
|
-
verdict
|
|
95
|
-
}] : [];
|
|
96
|
-
})
|
|
97
|
-
}));
|
|
98
|
-
}
|
|
99
|
-
//#endregion
|
|
100
|
-
//#region src/history/summary.ts
|
|
101
|
-
const OUTCOME_MARK = {
|
|
102
|
-
pass: "✓",
|
|
103
|
-
fail: "✗",
|
|
104
|
-
infra_error: "!",
|
|
105
|
-
skipped: "↓"
|
|
106
|
-
};
|
|
107
|
-
/** Enough to read the agent's conclusion; the record and transcript have it whole. */
|
|
108
|
-
const LAST_MESSAGE_LIMIT = 4e3;
|
|
109
|
-
const cell = (s) => s.replace(/\|/g, "\\|").replace(/\n/g, " ");
|
|
110
|
-
function summaryCounts(s) {
|
|
111
|
-
const parts = [
|
|
112
|
-
s.pass && `${s.pass} passed`,
|
|
113
|
-
s.fail - s.notGated > 0 && `${s.fail - s.notGated} failed`,
|
|
114
|
-
s.notGated > 0 && `${s.notGated} missed, not gated`,
|
|
115
|
-
s.infraError && `${s.infraError} infra error`,
|
|
116
|
-
s.skipped && `${s.skipped} skipped`
|
|
117
|
-
].filter(Boolean);
|
|
118
|
-
if (parts.length === 0) return "no evals ran";
|
|
119
|
-
const scope = s.attempts === s.evals ? "" : ` (${s.attempts} attempts, ${s.evals} evals)`;
|
|
120
|
-
return `${parts.join(" · ")}${scope}`;
|
|
121
|
-
}
|
|
122
|
-
const signedPoints = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
|
|
123
|
-
/** A gate policy as written in the config: `all`, or `passRate 0.8`. */
|
|
124
|
-
const gateLabel = (gate) => gate === "all" || gate === "none" ? gate : `passRate ${gate.passRate}`;
|
|
125
|
-
/**
|
|
126
|
-
* `lift +40 pts (95% −12…+72, 4/5 vs 2/5)`, and `within noise` when the interval spans 0: at a
|
|
127
|
-
* few attempts per cell, a point estimate alone says far more than the attempts can.
|
|
128
|
-
*/
|
|
129
|
-
function liftLabel(lift) {
|
|
130
|
-
const { interval, variant, control } = lift;
|
|
131
|
-
const range = `95% ${signedPoints(interval.lo)}…${signedPoints(interval.hi)}`;
|
|
132
|
-
const counts = `${variant.pass}/${variant.counted} vs ${control.pass}/${control.counted}`;
|
|
133
|
-
return `lift ${signedPoints(lift.points)} pts (${range}, ${counts})${lift.withinNoise ? ", within noise" : ""}`;
|
|
134
|
-
}
|
|
135
|
-
const answered = (t) => t.pass + t.fail + t.notRun;
|
|
136
|
-
const VERDICT_ORDER = {
|
|
137
|
-
discriminates: 0,
|
|
138
|
-
mixed: 1
|
|
139
|
-
};
|
|
140
|
-
/**
|
|
141
|
-
* A variant cell's checks against its control's, those that tell them apart first:
|
|
142
|
-
* `loads the skill 2/2 vs 0/2 · exports greet 2/2 vs 2/2 (doesn't discriminate)`. Empty
|
|
143
|
-
* without a control's tally.
|
|
144
|
-
*/
|
|
145
|
-
function checksLabel(cell) {
|
|
146
|
-
return (cell.checks ?? []).flatMap((k) => k.control && k.verdict ? [{
|
|
147
|
-
...k,
|
|
148
|
-
control: k.control
|
|
149
|
-
}] : []).map((k, i) => ({
|
|
150
|
-
k,
|
|
151
|
-
i
|
|
152
|
-
})).sort((a, b) => (VERDICT_ORDER[a.k.verdict ?? ""] ?? 2) - (VERDICT_ORDER[b.k.verdict ?? ""] ?? 2) || a.i - b.i).map(({ k }) => {
|
|
153
|
-
const { control } = k;
|
|
154
|
-
const notRun = k.notRun + control.notRun;
|
|
155
|
-
const notes = [notRun ? `${notRun} not run` : "", k.verdict === "both-pass" || k.verdict === "both-fail" ? "doesn't discriminate" : ""].filter(Boolean);
|
|
156
|
-
return `${k.label} ${k.pass}/${answered(k)} vs ${control.pass}/${answered(control)}${notes.length ? ` (${notes.join(", ")})` : ""}`;
|
|
157
|
-
}).join(" · ");
|
|
158
|
-
}
|
|
159
|
-
/** Every variant cell's checks that can't tell it from its control: they pass, or fail, on both. */
|
|
160
|
-
function undiscriminating(evals) {
|
|
161
|
-
return evals.flatMap((entry) => entry.cells.flatMap((cell) => (cell.checks ?? []).filter((k) => k.verdict === "both-pass" || k.verdict === "both-fail").map((check) => ({
|
|
162
|
-
entry,
|
|
163
|
-
cell,
|
|
164
|
-
check
|
|
165
|
-
}))));
|
|
166
|
-
}
|
|
167
|
-
/** `evals/s.eval.ts › greets fake/skilled/fake exports greet: passes on both`. */
|
|
168
|
-
const undiscriminatingLabel = ({ entry, cell, check }) => `${entry.id} ${cell.key} ${check.label}: ${check.verdict === "both-pass" ? "passes" : "fails"} on both`;
|
|
169
|
-
const rateText = (r) => `${r.name} ${r.wins}/${r.n} (${Math.round(r.wins / r.n * 100)}%)`;
|
|
170
|
-
/**
|
|
171
|
-
* One compare question's answer: `"Which is cleaner?" within: with-skill 2/2 (100%), plain 0/2
|
|
172
|
-
* (0%) · 2 votes, 0 ties, 0/1 pairs split by order · google/gemini-3.8-flash`, or why it has no
|
|
173
|
-
* votes. Each run's pair is voted in both orders, so a pooled rate counts a split once per side.
|
|
174
|
-
*/
|
|
175
|
-
function answersLabel(answer) {
|
|
176
|
-
const q = `"${clip(answer.question, 80)}"`;
|
|
177
|
-
const ties = answer.votes.filter((v) => v.winner === "tie").length;
|
|
178
|
-
const cuts = ["within", "across"].flatMap((cut) => {
|
|
179
|
-
const rates = answer.rates.filter((r) => r.cut === cut);
|
|
180
|
-
return rates.length ? [`${cut}: ${rates.map(rateText).join(", ")}`] : [];
|
|
181
|
-
});
|
|
182
|
-
const votes = answer.votes.length;
|
|
183
|
-
const splits = splitsLabel(pairTallies(answer.votes));
|
|
184
|
-
const tally = `${votes} vote${votes === 1 ? "" : "s"}, ${ties} tie${ties === 1 ? "" : "s"}${splits ? `, ${splits}` : ""}`;
|
|
185
|
-
const by = answer.resolvedModel ?? answer.model;
|
|
186
|
-
const failed = answer.error !== void 0 ? ` · failed: ${clip(answer.error, 200)}` : "";
|
|
187
|
-
if (votes === 0 && answer.error === void 0) return `${q} no pairs to judge: attempts that failed their checks are left out`;
|
|
188
|
-
return `${q} ${[
|
|
189
|
-
...cuts,
|
|
190
|
-
tally,
|
|
191
|
-
by
|
|
192
|
-
].join(" · ")}${failed}`;
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Each pair of arms on one harness, when an eval has two variants or more:
|
|
196
|
-
* `new-skill vs current-skill: lift +100 pts (…), checks Social OAuth 3/3 vs 0/3, effort tools 14
|
|
197
|
-
* vs 26 (×0.54, no overlap), …`. The harness is named when the eval has several.
|
|
198
|
-
*/
|
|
199
|
-
function contrastLines(e) {
|
|
200
|
-
const harnesses = new Set(e.cells.map((c) => c.harness)).size;
|
|
201
|
-
const cellOf = new Map(e.cells.map((c) => [c.key, c]));
|
|
202
|
-
const name = (key) => cellOf.get(key)?.environment ?? key;
|
|
203
|
-
return contrastsOf(e).map((x) => {
|
|
204
|
-
const apart = x.checks.filter((k) => k.verdict === "discriminates");
|
|
205
|
-
const parts = [
|
|
206
|
-
x.lift ? liftLabel(x.lift) : "",
|
|
207
|
-
apart.length ? `checks ${apart.map((k) => `${k.label} ${k.a.pass}/${answered(k.a)} vs ${k.b.pass}/${answered(k.b)}`).join(" · ")}` : x.checks.length ? "no check tells them apart" : "",
|
|
208
|
-
x.effort ? `effort ${effortLabel(x.effort)}` : ""
|
|
209
|
-
].filter(Boolean);
|
|
210
|
-
const on = harnesses > 1 ? ` (${cellOf.get(x.a)?.harness})` : "";
|
|
211
|
-
return `${name(x.a)} vs ${name(x.b)}${on}: ${parts.join(", ")}`;
|
|
212
|
-
});
|
|
213
|
-
}
|
|
214
|
-
/** `fake/-/fake 0/2 (control, not gated) · fake/with-skill/fake 2/2 (lift +100 pts …)`. */
|
|
215
|
-
function cellCounts(e) {
|
|
216
|
-
return e.cells.map((c) => {
|
|
217
|
-
const notes = [
|
|
218
|
-
c.role === "control" && e.cells.some((x) => x.role === "variant") ? "control" : "",
|
|
219
|
-
c.gate === "none" ? "not gated" : "",
|
|
220
|
-
c.gate !== "all" && c.gate !== "none" ? `gate ${gateLabel(c.gate)}` : "",
|
|
221
|
-
c.lift ? liftLabel(c.lift) : "",
|
|
222
|
-
...[checksLabel(c)].flatMap((checks) => checks ? [`checks ${checks}`] : []),
|
|
223
|
-
...[cellContrast(e, c.key)].flatMap((effort) => effort ? [`effort ${effortLabel(effort)}`] : []),
|
|
224
|
-
...[judgeRatesLabel(summaryJudgeRates(e, c))].filter(Boolean),
|
|
225
|
-
...[formatUsage(cellUsage(e, c.key))].filter((u) => Boolean(u)),
|
|
226
|
-
c.overrides.length ? `overrides ${c.overrides.join(", ")}` : ""
|
|
227
|
-
].filter(Boolean);
|
|
228
|
-
return `${c.key} ${c.pass}/${c.pass + c.fail}${notes.length ? ` (${notes.join(", ")})` : ""}`;
|
|
229
|
-
}).join(" · ");
|
|
230
|
-
}
|
|
231
|
-
function judgeCell(e) {
|
|
232
|
-
const judgements = e.attempts.flatMap((a) => a.judgements);
|
|
233
|
-
if (judgements.length === 0) return "";
|
|
234
|
-
return judgements.map((j) => `${j.verdict} ${j.score.toFixed(2)}/${j.threshold}`).join(", ");
|
|
235
|
-
}
|
|
236
|
-
const short = (hash) => hash.replace(/^sha256:/, "").slice(0, 12);
|
|
237
|
-
/**
|
|
238
|
-
* The state a run executed in, as label/value pairs, shared by summary.md and `reeval show`. The
|
|
239
|
-
* commit is where the run executed; the commit that adds its history file comes later.
|
|
240
|
-
*/
|
|
241
|
-
function stateRows(run) {
|
|
242
|
-
const { git, stack: env } = run;
|
|
243
|
-
const rows = [];
|
|
244
|
-
rows.push(["Executed at", git ? `${git.sha.slice(0, 12)}${git.branch ? ` on ${git.branch}` : ""}${git.dirty ? ", dirty" : ", clean"} (the commit that adds this record is later)` : "not a git repository"]);
|
|
245
|
-
if (git?.dirty) rows.push(["Uncommitted", git.worktreePatch ? `worktree.patch, ${git.worktreePatch.bytes} bytes, sha256 ${short(git.worktreePatch.sha256)}` : `not saved: ${git.worktreePatchNote ?? "unknown"}${git.diffHash ? `; sha256 ${short(git.diffHash)}` : ""}`]);
|
|
246
|
-
rows.push(["Lockfiles", run.dependencies.lockfiles.map((l) => `${l.path} ${short(l.sha256)}`).join(", ") || "none found"]);
|
|
247
|
-
rows.push(["Images", env.images.length ? env.images.map((i) => `${i.ref} · id ${short(i.id)} · ${i.repoDigests[0] ?? "no registry digest (built locally)"} · CLI ${i.agentCli ?? "n/a"}${i.dockerfile ? ` · dockerfile ${i.dockerfile.path} ${short(i.dockerfile.sha256)}` : ""}${i.install?.length ? ` · install ${i.install.length} on ${i.base}` : ""}`).join("; ") : "none (fake sandbox)"]);
|
|
248
|
-
rows.push(["Harnesses", env.harnesses.map(harnessLabel).join(", ") || "none"]);
|
|
249
|
-
rows.push(["Environments", run.environments.map((e) => `${e.name} ${short(e.digest)}${e.skills.length ? ` · skills ${e.skills.map((s) => `${s.name}${s.commit ? `@${s.commit.slice(0, 7)}` : ""}`).join(", ")}` : ""}${e.mcp.length ? ` · mcp ${e.mcp.join(", ")}` : ""}${e.secrets.length ? ` · secrets ${e.secrets.join(", ")}` : ""}`).join("; ") || "none"]);
|
|
250
|
-
rows.push(["Judge", `${env.judge.model ?? "none by default"}${env.judge.resolvedModels.length ? ` (served ${env.judge.resolvedModels.join(", ")})` : ""} · threshold ${env.judge.threshold} · version ${env.judge.version}`]);
|
|
251
|
-
rows.push(["Context", env.context.length ? `${env.context.length} file${env.context.length === 1 ? "" : "s"}: ${env.context.map((c) => `${c.path} ${short(c.sha256)}`).join(", ")}` : "no skills, rules or MCP config found in the sandbox"]);
|
|
252
|
-
rows.push(["Packages", Object.entries(env.packages).map(([name, v]) => `${name} ${v}`).join(" · ")]);
|
|
253
|
-
const language = languageLabel(env);
|
|
254
|
-
rows.push(["reeval", `${env.reeval} · ${runtimeLabel(env)}${language ? ` · ${language}` : ""} · ${env.os}/${env.arch}`]);
|
|
255
|
-
rows.push(["Runner", run.runner.kind === "ci" ? `CI (${run.runner.provider})${run.runner.runUrl ? ` ${run.runner.runUrl}` : ""}` : `local · ${run.author}`]);
|
|
256
|
-
return rows;
|
|
257
|
-
}
|
|
258
|
-
/** Each pooled `evaluate.each`: its cases, and its pooled rates or lifts. */
|
|
259
|
-
function familySection(run) {
|
|
260
|
-
if (!run.families?.length) return [];
|
|
261
|
-
const lines = [
|
|
262
|
-
"## Families",
|
|
263
|
-
"",
|
|
264
|
-
"| | Family | Pooled |",
|
|
265
|
-
"|---|---|---|"
|
|
266
|
-
];
|
|
267
|
-
for (const f of run.families) {
|
|
268
|
-
const pooled = f.lifts.length ? f.lifts.map((l) => {
|
|
269
|
-
const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)})`;
|
|
270
|
-
const effort = familyEffortLabel(l);
|
|
271
|
-
return `${l.variant} ${familyLiftLabel(l)}${gate}${effort ? `; effort ${effort}` : ""}`;
|
|
272
|
-
}) : f.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
|
|
273
|
-
lines.push(`| ${OUTCOME_MARK[f.outcome]} | ${cell(`${f.id} (${familyHeadLabel(f)})`)} | ${cell(pooled.join(" · "))} |`);
|
|
274
|
-
}
|
|
275
|
-
lines.push("");
|
|
276
|
-
return lines;
|
|
277
|
-
}
|
|
278
|
-
/** Markdown for people and agents landing on a run cold. Links are relative to the artifact dir. */
|
|
279
|
-
function renderSummary(run) {
|
|
280
|
-
const lines = [
|
|
281
|
-
`# reeval run ${run.runId}`,
|
|
282
|
-
"",
|
|
283
|
-
`${summaryCounts(run.summary)}${run.status === "complete" ? "" : ` · **${run.status === "budget" ? "budget reached" : run.status}**`} · started ${run.startedAt} (${formatDuration(run.durationMs)})`,
|
|
284
|
-
"",
|
|
285
|
-
`Plan: ${run.profile ? `profile ${run.profile.name} (${run.profile.reason}) · ` : ""}${run.plan.evals} evals · ${run.plan.cells} cells · ${run.plan.attempts} attempts${run.budget ? ` · budget ${[run.budget.attempts !== void 0 ? `${run.budget.attempts} attempts` : "", run.budget.duration ?? ""].filter(Boolean).join(", ")}` : ""}`,
|
|
286
|
-
"",
|
|
287
|
-
...run.plan.sandboxWarnings.map((w) => `> **Warning:** ${w.message}\n`),
|
|
288
|
-
"## State",
|
|
289
|
-
"",
|
|
290
|
-
"| | |",
|
|
291
|
-
"|---|---|",
|
|
292
|
-
...stateRows(run).map(([k, v]) => `| ${k} | ${cell(v)} |`),
|
|
293
|
-
"",
|
|
294
|
-
...warnings(run).map((w) => `> **Warning:** ${w}\n`),
|
|
295
|
-
`Record: [${run.runId}.json](../${run.runId}.json) · artifacts: ${run.artifacts}`,
|
|
296
|
-
"",
|
|
297
|
-
...familySection(run),
|
|
298
|
-
"## Evals",
|
|
299
|
-
"",
|
|
300
|
-
"| | Eval | Time or cells | Judge |",
|
|
301
|
-
"|---|---|---|---|"
|
|
302
|
-
];
|
|
303
|
-
for (const e of run.evals) {
|
|
304
|
-
const a = e.attempts[0];
|
|
305
|
-
const time = e.todo ? "todo" : e.attempts.length > 1 ? cellCounts(e) : a ? formatDuration(a.durationMs) : "";
|
|
306
|
-
lines.push(`| ${OUTCOME_MARK[evalMarkOutcome(e)]} | ${cell(e.id)} | ${cell(time)} | ${judgeCell(e)} |`);
|
|
307
|
-
}
|
|
308
|
-
const rank = (o) => o === "fail" || o === "infra_error" ? 0 : o === "pass" ? 1 : 2;
|
|
309
|
-
for (const e of [...run.evals].sort((x, y) => rank(x.outcome) - rank(y.outcome))) {
|
|
310
|
-
if (e.outcome === "skipped") continue;
|
|
311
|
-
lines.push("", `### ${OUTCOME_MARK[evalMarkOutcome(e)]} ${e.id}`, "");
|
|
312
|
-
const multi = e.attempts.length > 1;
|
|
313
|
-
if (multi) lines.push(`Cells: ${cellCounts(e)}`, "");
|
|
314
|
-
const contrasts = contrastLines(e);
|
|
315
|
-
if (contrasts.length) {
|
|
316
|
-
lines.push("Contrasts (each pair of arms, same harness):", "");
|
|
317
|
-
for (const line of contrasts) lines.push(`- ${cell(line)}`);
|
|
318
|
-
lines.push("");
|
|
319
|
-
}
|
|
320
|
-
if (e.answers?.length) {
|
|
321
|
-
for (const a of e.answers) {
|
|
322
|
-
const input = a.input ? ` · [what the model saw](${a.input.path})` : "";
|
|
323
|
-
lines.push(`- Question: ${cell(answersLabel(a))}${input}`);
|
|
324
|
-
for (const line of headToHeadLines(a.votes)) lines.push(` - ${cell(line)}`);
|
|
325
|
-
for (const reason of sampleReasons(a.votes)) lines.push(` - ${cell(reason)}`);
|
|
326
|
-
}
|
|
327
|
-
lines.push("");
|
|
328
|
-
}
|
|
329
|
-
const attemptRank = (a) => isUngatedMiss(e, a) ? 3 : rank(a.outcome);
|
|
330
|
-
for (const a of [...e.attempts].sort((x, y) => attemptRank(x) - attemptRank(y))) {
|
|
331
|
-
if (a.outcome === "skipped") continue;
|
|
332
|
-
if (isUngatedMiss(e, a)) {
|
|
333
|
-
if (multi) lines.push(`#### ○ ${a.id} · missed, not gated`, "");
|
|
334
|
-
lines.push("<details><summary>Why it missed</summary>", "");
|
|
335
|
-
attemptDetail(lines, a);
|
|
336
|
-
lines.push("", "</details>");
|
|
337
|
-
continue;
|
|
338
|
-
}
|
|
339
|
-
if (multi) lines.push(`#### ${OUTCOME_MARK[a.outcome]} ${a.id}`, "");
|
|
340
|
-
attemptDetail(lines, a);
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
lines.push("");
|
|
344
|
-
return lines.join("\n");
|
|
345
|
-
}
|
|
346
|
-
/** A failed verdict already says why a judge error failed the attempt; its message repeats it. */
|
|
347
|
-
const repeatsVerdict = (a) => a.error?.kind === "judge" && a.judgements.some((j) => j.verdict === "fail");
|
|
348
|
-
function attemptDetail(lines, a) {
|
|
349
|
-
if (a.error && !repeatsVerdict(a)) lines.push(`**${a.error.kind}**`, "", "```", a.error.message.split("\n").slice(0, 20).join("\n"), "```", "");
|
|
350
|
-
for (const turn of a.turns) {
|
|
351
|
-
lines.push(`- Prompt (${turn.status}, ${formatDuration(turn.durationMs)}): ${clip(cell(turn.prompt), 200)}`);
|
|
352
|
-
if (turn.error) lines.push(` - Error: ${clip(cell(turn.error), 500)}`);
|
|
353
|
-
}
|
|
354
|
-
const t = a.timings;
|
|
355
|
-
lines.push(`- Time: setup ${formatDuration(t.setupMs)} · agent ${formatDuration(t.agentMs)} · checks ${formatDuration(t.checksMs)} · judge ${formatDuration(t.judgeMs)}`);
|
|
356
|
-
const act = a.activity;
|
|
357
|
-
const web = (n, what) => n === null ? `${what} not reported` : `${n} ${what}`;
|
|
358
|
-
lines.push(`- Agent: ${act.fileEdits} file edits · ${act.tools} other tool calls · ${web(act.webSearches, "web searches")} · ${web(act.webFetches, "fetches")} · ${formatDuration(act.reasoningMs)} reasoning${act.stalledTools ? ` · ${act.stalledTools} stalled tool${act.stalledTools === 1 ? "" : "s"} (≥2m)${act.maxToolMs ? `, longest ${formatDuration(act.maxToolMs)}` : ""}` : ""}`);
|
|
359
|
-
for (const { judgement: j, label, labelled } of labelledJudgements(a)) {
|
|
360
|
-
const retried = j.retries?.length ? `, after ${j.retries.length} ${j.retries.length === 1 ? "retry" : "retries"}` : "";
|
|
361
|
-
lines.push(`- Judge${labelled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
|
|
362
|
-
}
|
|
363
|
-
const labels = checkLabels(a);
|
|
364
|
-
a.checks.forEach((c, i) => {
|
|
365
|
-
const label = labels[i];
|
|
366
|
-
lines.push(`- Check${label ? ` ${cell(label)}` : ""} exit ${c.exitCode}: \`${cell(commandLine(c.command))}\`${c.log ? ` ([log](${c.log.path}))` : ""}`);
|
|
367
|
-
});
|
|
368
|
-
if (a.changedFiles.length) lines.push(`- Changed: ${a.changedFiles.map((f) => f.count ? `\`${f.path}\` (${f.count} files)` : `${f.status} \`${f.path}\``).join(", ")}`);
|
|
369
|
-
const last = a.turns.at(-1)?.text;
|
|
370
|
-
if (last) {
|
|
371
|
-
const shown = last.length > LAST_MESSAGE_LIMIT ? `${last.slice(0, LAST_MESSAGE_LIMIT)}…` : last;
|
|
372
|
-
lines.push("", "Last message:", "", ...shown.trimEnd().split("\n").map((l) => `> ${l}`), "");
|
|
373
|
-
}
|
|
374
|
-
const links = [
|
|
375
|
-
a.files.setupLog && `[setup log](${a.files.setupLog.path})`,
|
|
376
|
-
a.files.transcript && `[transcript](${a.files.transcript.path})`,
|
|
377
|
-
a.files.diff && `[diff](${a.files.diff.path})`,
|
|
378
|
-
a.files.judgeInput && `[judge input](${a.files.judgeInput.path})`
|
|
379
|
-
].filter(Boolean);
|
|
380
|
-
if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
|
|
381
|
-
}
|
|
382
|
-
//#endregion
|
|
383
|
-
export { contrastLines as a, renderSummary as c, summaryCounts as d, undiscriminating as f, contrastsOf as g, contrastPairs as h, checksLabel as i, repeatsVerdict as l, checkVerdict as m, answersLabel as n, gateLabel as o, undiscriminatingLabel as p, cellCounts as r, liftLabel as s, OUTCOME_MARK as t, stateRows as u };
|
|
384
|
-
|
|
385
|
-
//# sourceMappingURL=summary-BLAVLySU.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"summary-BLAVLySU.js","names":[],"sources":["../src/view/contrasts.ts","../src/history/summary.ts"],"mappings":";;;;;;;AASA,MAAM,UAAU,MAAuD;CACrE,MAAM,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE;CACrC,IAAI,aAAa,GAAG,OAAO,KAAA;CAC3B,OAAO,EAAE,SAAS,WAAW,QAAQ,EAAE,SAAS,IAAI,SAAS;AAC/D;;AAGA,SAAgB,aAAa,GAAe,GAAyC;CACnF,MAAM,IAAI,OAAO,CAAC;CAClB,MAAM,IAAI,OAAO,CAAC;CAClB,IAAI,MAAM,KAAA,KAAa,MAAM,KAAA,GAAW,OAAO,KAAA;CAC/C,IAAI,MAAM,SAAS,MAAM,OAAO,OAAO;CACvC,IAAI,MAAM,UAAU,MAAM,QAAQ,OAAO;CACzC,IAAK,MAAM,SAAS,MAAM,UAAY,MAAM,UAAU,MAAM,OAAQ,OAAO;CAC3E,OAAO;AACT;;;;;;;AAUA,SAAgB,cAAkC,OAA2B;CAC3E,MAAM,yBAAS,IAAI,IAAiB;CACpC,KAAK,MAAM,KAAK,OAAO;EACrB,MAAM,KAAK,GAAG,EAAE,QAAQ,IAAI,EAAE;EAC9B,OAAO,IAAI,IAAI,CAAC,GAAI,OAAO,IAAI,EAAE,KAAK,CAAC,GAAI,CAAC,CAAC;CAC/C;CACA,MAAM,QAAuB,CAAC;CAC9B,KAAK,MAAM,SAAS,OAAO,OAAO,GAAG;EACnC,MAAM,OAAO,CACX,GAAG,MAAM,QAAQ,MAAM,EAAE,SAAS,SAAS,GAC3C,GAAG,MAAM,QAAQ,MAAM,EAAE,SAAS,SAAS,CAC7C;EACA,IAAI,KAAK,QAAQ,MAAM,EAAE,SAAS,SAAS,CAAC,CAAC,SAAS,GAAG;EACzD,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAC/B,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,KAAK,CAAC,KAAK,IAAS,KAAK,EAAO,CAAC;CAEvE;CACA,OAAO;AACT;AAaA,MAAM,SAAS,EAAE,MAAM,MAAM,OAAO,cAAsC;CACxE;CACA;CACA;CACA;AACF;AAEA,SAAS,SAAS,GAAe,GAA4B;CAC3D,IAAI,EAAE,aAAa,QAAQ,EAAE,aAAa,MAAM,OAAO;CACvD,MAAM,UAAU;EAAE,MAAM,EAAE;EAAM,SAAS,EAAE,OAAO,EAAE;CAAK;CACzD,MAAM,UAAU;EAAE,MAAM,EAAE;EAAM,SAAS,EAAE,OAAO,EAAE;CAAK;CACzD,MAAM,EAAE,QAAQ,KAAK,SAAS,aAC5B;EAAE,MAAM,QAAQ;EAAM,GAAG,QAAQ;CAAQ,GACzC;EAAE,MAAM,QAAQ;EAAM,GAAG,QAAQ;CAAQ,CAC3C;CACA,OAAO;EACL,IAAI,EAAE;EACN;EACA;EACA,QAAQ,OAAO,MAAM;EACrB,UAAU;GAAE,IAAI,OAAO,GAAG;GAAG,IAAI,OAAO,IAAI;GAAG,OAAO;GAAM,QAAQ;EAAkB;EACtF,aAAa,MAAM,KAAK,OAAO;CACjC;AACF;;AAGA,SAAgB,YAAY,GAAsD;CAChF,OAAO,cAAc,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,QAAQ;EAC7C,GAAG,EAAE;EACL,GAAG,EAAE;EACL,MAAM,SAAS,GAAG,CAAC;EACnB,QAAQ,eAAe,YAAY,GAAG,EAAE,GAAG,GAAG,YAAY,GAAG,EAAE,GAAG,CAAC;EACnE,SAAS,EAAE,UAAU,CAAC,EAAA,CAAG,SAAS,MAAM;GACtC,MAAM,QAAQ,EAAE,QAAQ,MAAM,MAAM,EAAE,UAAU,EAAE,KAAK;GACvD,MAAM,UAAU,SAAS,aAAa,GAAG,KAAK;GAC9C,OAAO,SAAS,UAAU,CAAC;IAAE,OAAO,EAAE;IAAO,GAAG,MAAM,CAAC;IAAG,GAAG,MAAM,KAAK;IAAG;GAAQ,CAAC,IAAI,CAAC;EAC3F,CAAC;CACH,EAAE;AACJ;;;ACzEA,MAAa,eAAwC;CACnD,MAAM;CACN,MAAM;CACN,aAAa;CACb,SAAS;AACX;;AAGA,MAAM,qBAAqB;AAE3B,MAAM,QAAQ,MAAc,EAAE,QAAQ,OAAO,KAAK,CAAC,CAAC,QAAQ,OAAO,GAAG;AAEtE,SAAgB,cAAc,GAAiC;CAC7D,MAAM,QAAQ;EACZ,EAAE,QAAQ,GAAG,EAAE,KAAK;EACpB,EAAE,OAAO,EAAE,WAAW,KAAK,GAAG,EAAE,OAAO,EAAE,SAAS;EAClD,EAAE,WAAW,KAAK,GAAG,EAAE,SAAS;EAChC,EAAE,cAAc,GAAG,EAAE,WAAW;EAChC,EAAE,WAAW,GAAG,EAAE,QAAQ;CAC5B,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,MAAM,QAAQ,EAAE,aAAa,EAAE,QAAQ,KAAK,KAAK,EAAE,SAAS,aAAa,EAAE,MAAM;CACjF,OAAO,GAAG,MAAM,KAAK,KAAK,IAAI;AAChC;AAEA,MAAM,gBAAgB,MAAc,GAAG,KAAK,IAAI,MAAM,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG;;AAGxF,MAAa,aAAa,SACxB,SAAS,SAAS,SAAS,SAAS,OAAO,YAAY,KAAK;;;;;AAM9D,SAAgB,UAAU,MAA+C;CACvE,MAAM,EAAE,UAAU,SAAS,YAAY;CACvC,MAAM,QAAQ,OAAO,aAAa,SAAS,EAAE,EAAE,GAAG,aAAa,SAAS,EAAE;CAC1E,MAAM,SAAS,GAAG,QAAQ,KAAK,GAAG,QAAQ,QAAQ,MAAM,QAAQ,KAAK,GAAG,QAAQ;CAChF,OAAO,QAAQ,aAAa,KAAK,MAAM,EAAE,QAAQ,MAAM,IAAI,OAAO,GAAG,KAAK,cAAc,mBAAmB;AAC7G;AAEA,MAAM,YAAY,MAAkB,EAAE,OAAO,EAAE,OAAO,EAAE;AAExD,MAAM,gBAAwC;CAAE,eAAe;CAAG,OAAO;AAAE;;;;;;AAO3E,SAAgB,YAAY,MAA0B;CAIpD,QAHc,KAAK,UAAU,CAAC,EAAA,CAAG,SAAS,MACxC,EAAE,WAAW,EAAE,UAAU,CAAC;EAAE,GAAG;EAAG,SAAS,EAAE;CAAQ,CAAC,IAAI,CAAC,CAEnD,CAAC,CACR,KAAK,GAAG,OAAO;EAAE;EAAG;CAAE,EAAE,CAAC,CACzB,MACE,GAAG,OACD,cAAc,EAAE,EAAE,WAAW,OAAO,MAAM,cAAc,EAAE,EAAE,WAAW,OAAO,MAC/E,EAAE,IAAI,EAAE,CACZ,CAAC,CACA,KAAK,EAAE,QAAQ;EACd,MAAM,EAAE,YAAY;EACpB,MAAM,SAAS,EAAE,SAAS,QAAQ;EAClC,MAAM,QAAQ,CACZ,SAAS,GAAG,OAAO,YAAY,IAC/B,EAAE,YAAY,eAAe,EAAE,YAAY,cAAc,yBAAyB,EACpF,CAAC,CAAC,OAAO,OAAO;EAChB,OAAO,GAAG,EAAE,MAAM,GAAG,EAAE,KAAK,GAAG,SAAS,CAAC,EAAE,MAAM,QAAQ,KAAK,GAAG,SAAS,OAAO,IAAI,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK;CACjI,CAAC,CAAC,CACD,KAAK,KAAK;AACf;;AAGA,SAAgB,iBAAiB,OAAoB;CACnD,OAAO,MAAM,SAAS,UACpB,MAAM,MAAM,SAAS,UAClB,KAAK,UAAU,CAAC,EAAA,CACd,QAAQ,MAAM,EAAE,YAAY,eAAe,EAAE,YAAY,WAAW,CAAC,CACrE,KAAK,WAAW;EAAE;EAAO;EAAM;CAAM,EAAE,CAC5C,CACF;AACF;;AAGA,MAAa,yBAAyB,EACpC,OACA,MACA,YAEA,GAAG,MAAM,GAAG,IAAI,KAAK,IAAI,IAAI,MAAM,MAAM,IAAI,MAAM,YAAY,cAAc,WAAW,QAAQ;AAElG,MAAM,YAAY,MAChB,GAAG,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,IAAI,KAAK,MAAO,EAAE,OAAO,EAAE,IAAK,GAAG,EAAE;;;;;;AAOlE,SAAgB,aAAa,QAA+B;CAC1D,MAAM,IAAI,IAAI,KAAK,OAAO,UAAU,EAAE,EAAE;CACxC,MAAM,OAAO,OAAO,MAAM,QAAQ,MAAM,EAAE,WAAW,KAAK,CAAC,CAAC;CAC5D,MAAM,OAAQ,CAAC,UAAU,QAAQ,CAAC,CAAW,SAAS,QAAQ;EAC5D,MAAM,QAAQ,OAAO,MAAM,QAAQ,MAAM,EAAE,QAAQ,GAAG;EACtD,OAAO,MAAM,SAAS,CAAC,GAAG,IAAI,IAAI,MAAM,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;CACzE,CAAC;CACD,MAAM,QAAQ,OAAO,MAAM;CAC3B,MAAM,SAAS,YAAY,YAAY,OAAO,KAAK,CAAC;CACpD,MAAM,QAAQ,GAAG,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,KAAK,MAAM,SAAS,IAAI,KAAK,MAAM,SAAS,KAAK,WAAW;CACrH,MAAM,KAAK,OAAO,iBAAiB,OAAO;CAC1C,MAAM,SAAS,OAAO,UAAU,KAAA,IAAY,cAAc,KAAK,OAAO,OAAO,GAAG,MAAM;CACtF,IAAI,UAAU,KAAK,OAAO,UAAU,KAAA,GAClC,OAAO,GAAG,EAAE;CAEd,OAAO,GAAG,EAAE,GAAG;EAAC,GAAG;EAAM;EAAO;CAAE,CAAC,CAAC,KAAK,KAAK,IAAI;AACpD;;;;;;AAOA,SAAgB,cAAc,GAAwB;CACpD,MAAM,YAAY,IAAI,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACzD,MAAM,SAAS,IAAI,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;CACrD,MAAM,QAAQ,QAAgB,OAAO,IAAI,GAAG,CAAC,EAAE,eAAe;CAC9D,OAAO,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM;EAC/B,MAAM,QAAQ,EAAE,OAAO,QAAQ,MAAM,EAAE,YAAY,eAAe;EAClE,MAAM,QAAQ;GACZ,EAAE,OAAO,UAAU,EAAE,IAAI,IAAI;GAC7B,MAAM,SACF,UAAU,MAAM,KAAK,MAAM,GAAG,EAAE,MAAM,GAAG,EAAE,EAAE,KAAK,GAAG,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,KAAK,GAAG,SAAS,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,MAChH,EAAE,OAAO,SACP,8BACA;GACN,EAAE,SAAS,UAAU,YAAY,EAAE,MAAM,MAAM;EACjD,CAAC,CAAC,OAAO,OAAO;EAChB,MAAM,KAAK,YAAY,IAAI,KAAK,OAAO,IAAI,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK;EAC9D,OAAO,GAAG,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,EAAE,CAAC,IAAI,GAAG,IAAI,MAAM,KAAK,IAAI;CAC9D,CAAC;AACH;;AAGA,SAAgB,WAAW,GAAsB;CAC/C,OAAO,EAAE,MACN,KAAK,MAAM;EACV,MAAM,QAAQ;GACZ,EAAE,SAAS,aAAa,EAAE,MAAM,MAAM,MAAM,EAAE,SAAS,SAAS,IAAI,YAAY;GAChF,EAAE,SAAS,SAAS,cAAc;GAClC,EAAE,SAAS,SAAS,EAAE,SAAS,SAAS,QAAQ,UAAU,EAAE,IAAI,MAAM;GACtE,EAAE,OAAO,UAAU,EAAE,IAAI,IAAI;GAC7B,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,WAAY,SAAS,CAAC,UAAU,QAAQ,IAAI,CAAC,CAAE;GAC5E,GAAG,CAAC,aAAa,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,WACnC,SAAS,CAAC,UAAU,YAAY,MAAM,GAAG,IAAI,CAAC,CAChD;GACA,GAAG,CAAC,gBAAgB,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,OAAO;GAC5D,GAAG,CAAC,YAAY,UAAU,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAmB,QAAQ,CAAC,CAAC;GAC3E,EAAE,UAAU,SAAS,aAAa,EAAE,UAAU,KAAK,IAAI,MAAM;EAC/D,CAAC,CAAC,OAAO,OAAO;EAChB,OAAO,GAAG,EAAE,IAAI,GAAG,EAAE,KAAK,GAAG,EAAE,OAAO,EAAE,OAAO,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK;CAC3F,CAAC,CAAC,CACD,KAAK,KAAK;AACf;AAEA,SAAS,UAAU,GAAsB;CACvC,MAAM,aAAa,EAAE,SAAS,SAAS,MAAM,EAAE,UAAU;CACzD,IAAI,WAAW,WAAW,GAAG,OAAO;CACpC,OAAO,WAAW,KAAK,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,KAAK,IAAI;AAC7F;AAEA,MAAM,SAAS,SAAiB,KAAK,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;;;;;AAMxE,SAAgB,UAAU,KAAyC;CACjE,MAAM,EAAE,KAAK,OAAO,QAAQ;CAC5B,MAAM,OAAgC,CAAC;CACvC,KAAK,KAAK,CACR,eACA,MACI,GAAG,IAAI,IAAI,MAAM,GAAG,EAAE,IAAI,IAAI,SAAS,OAAO,IAAI,WAAW,KAAK,IAAI,QAAQ,YAAY,UAAU,gDACpG,sBACN,CAAC;CACD,IAAI,KAAK,OACP,KAAK,KAAK,CACR,eACA,IAAI,gBACA,mBAAmB,IAAI,cAAc,MAAM,iBAAiB,MAAM,IAAI,cAAc,MAAM,MAC1F,cAAc,IAAI,qBAAqB,YAAY,IAAI,WAAW,YAAY,MAAM,IAAI,QAAQ,MAAM,IAC5G,CAAC;CAEH,KAAK,KAAK,CACR,aACA,IAAI,aAAa,UAAU,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,KAC7E,YACJ,CAAC;CACD,KAAK,KAAK,CACR,UACA,IAAI,OAAO,SACP,IAAI,OACD,KACE,MACC,GAAG,EAAE,IAAI,QAAQ,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,MAAM,qCAAqC,SAAS,EAAE,YAAY,QAAQ,EAAE,aAAa,iBAAiB,EAAE,WAAW,KAAK,GAAG,MAAM,EAAE,WAAW,MAAM,MAAM,KAAK,EAAE,SAAS,SAAS,cAAc,EAAE,QAAQ,OAAO,MAAM,EAAE,SAAS,IAC3R,CAAC,CACA,KAAK,IAAI,IACZ,qBACN,CAAC;CACD,KAAK,KAAK,CAAC,aAAa,IAAI,UAAU,IAAI,YAAY,CAAC,CAAC,KAAK,IAAI,KAAK,MAAM,CAAC;CAC7E,KAAK,KAAK,CACR,gBACA,IAAI,aACD,KACE,MACC,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,IAAI,EAAE,OAAO,SAAS,aAAa,EAAE,OAAO,KAAK,MAAM,GAAG,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE,OAAO,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,MAAM,KAAK,EAAE,IAAI,SAAS,UAAU,EAAE,IAAI,KAAK,IAAI,MAAM,KAAK,EAAE,QAAQ,SAAS,cAAc,EAAE,QAAQ,KAAK,IAAI,MAAM,IACjR,CAAC,CACA,KAAK,IAAI,KAAK,MACnB,CAAC;CACD,KAAK,KAAK,CACR,SACA,GAAG,IAAI,MAAM,SAAS,oBAAoB,IAAI,MAAM,eAAe,SAAS,YAAY,IAAI,MAAM,eAAe,KAAK,IAAI,EAAE,KAAK,GAAG,eAAe,IAAI,MAAM,UAAU,aAAa,IAAI,MAAM,SAChM,CAAC;CACD,KAAK,KAAK,CACR,WACA,IAAI,QAAQ,SACR,GAAG,IAAI,QAAQ,OAAO,OAAO,IAAI,QAAQ,WAAW,IAAI,KAAK,IAAI,IAAI,IAAI,QAAQ,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,MACrI,qDACN,CAAC;CACD,KAAK,KAAK,CACR,YACA,OAAO,QAAQ,IAAI,QAAQ,CAAC,CACzB,KAAK,CAAC,MAAM,OAAO,GAAG,KAAK,GAAG,GAAG,CAAC,CAClC,KAAK,KAAK,CACf,CAAC;CACD,MAAM,WAAW,cAAc,GAAG;CAClC,KAAK,KAAK,CACR,UACA,GAAG,IAAI,OAAO,KAAK,aAAa,GAAG,IAAI,WAAW,MAAM,aAAa,GAAG,KAAK,IAAI,GAAG,GAAG,IAAI,MAC7F,CAAC;CACD,KAAK,KAAK,CACR,UACA,IAAI,OAAO,SAAS,OAChB,OAAO,IAAI,OAAO,SAAS,GAAG,IAAI,OAAO,SAAS,IAAI,IAAI,OAAO,WAAW,OAC5E,WAAW,IAAI,QACrB,CAAC;CACD,OAAO;AACT;;AAGA,SAAS,cAAc,KAA0B;CAC/C,IAAI,CAAC,IAAI,UAAU,QAAQ,OAAO,CAAC;CACnC,MAAM,QAAQ;EAAC;EAAe;EAAI;EAAyB;CAAe;CAC1E,KAAK,MAAM,KAAK,IAAI,UAAU;EAC5B,MAAM,SAAS,EAAE,MAAM,SACnB,EAAE,MAAM,KAAK,MAAM;GACjB,MAAM,OAAO,EAAE,YAAY,KAAA,IAAY,KAAK,aAAa,KAAK,MAAM,EAAE,UAAU,GAAG,EAAE;GACrF,MAAM,SAAS,kBAAkB,CAAC;GAClC,OAAO,GAAG,EAAE,QAAQ,GAAG,gBAAgB,CAAC,IAAI,OAAO,SAAS,YAAY,WAAW;EACrF,CAAC,IACD,EAAE,MAAM,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,gBAAgB,CAAC,GAAG;EACxD,MAAM,KACJ,KAAK,aAAa,EAAE,SAAS,KAAK,KAAK,GAAG,EAAE,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,EAAE,KAAK,KAAK,OAAO,KAAK,KAAK,CAAC,EAAE,GAC1G;CACF;CACA,MAAM,KAAK,EAAE;CACb,OAAO;AACT;;AAGA,SAAgB,cAAc,KAAwB;CACpD,MAAM,QAAQ;EACZ,gBAAgB,IAAI;EACpB;EACA,GAAG,cAAc,IAAI,OAAO,IAAI,IAAI,WAAW,aAAa,KAAK,QAAQ,IAAI,WAAW,WAAW,mBAAmB,IAAI,OAAO,IAAI,aAAa,IAAI,UAAU,IAAI,eAAe,IAAI,UAAU,EAAE;EACnM;EACA,SAAS,IAAI,UAAU,WAAW,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,OAAO,QAAQ,KAAK,IAAI,KAAK,MAAM,WAAW,IAAI,KAAK,MAAM,WAAW,IAAI,KAAK,SAAS,WAAW,IAAI,SAAS,aAAa,CAAC,IAAI,OAAO,aAAa,KAAA,IAAY,GAAG,IAAI,OAAO,SAAS,aAAa,IAAI,IAAI,OAAO,YAAY,EAAE,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI,MAAM;EACzU;EACA,GAAG,IAAI,KAAK,gBAAgB,KAAK,MAAM,kBAAkB,EAAE,QAAQ,GAAG;EACtE;EACA;EACA;EACA;EACA,GAAG,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK,EAAE,KAAK,KAAK,CAAC,EAAE,GAAG;EACzD;EACA,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,MAAM,kBAAkB,EAAE,GAAG;EACnD,YAAY,IAAI,MAAM,YAAY,IAAI,MAAM,sBAAsB,IAAI;EACtE;EACA,GAAG,cAAc,GAAG;EACpB;EACA;EACA;EACA;CACF;CACA,KAAK,MAAM,KAAK,IAAI,OAAO;EACzB,MAAM,IAAI,EAAE,SAAS;EACrB,MAAM,OAAO,EAAE,OACX,SACA,EAAE,SAAS,SAAS,IAClB,WAAW,CAAC,IACZ,IACE,eAAe,EAAE,UAAU,IAC3B;EACR,MAAM,KACJ,KAAK,aAAa,gBAAgB,CAAC,GAAG,KAAK,KAAK,EAAE,EAAE,EAAE,KAAK,KAAK,IAAI,EAAE,KAAK,UAAU,CAAC,EAAE,GAC1F;CACF;CAGA,MAAM,QAAQ,MAAgB,MAAM,UAAU,MAAM,gBAAgB,IAAI,MAAM,SAAS,IAAI;CAC3F,KAAK,MAAM,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,OAAO,CAAC,GAAG;EAChF,IAAI,EAAE,YAAY,WAAW;EAC7B,MAAM,KAAK,IAAI,OAAO,aAAa,gBAAgB,CAAC,GAAG,GAAG,EAAE,MAAM,EAAE;EACpE,MAAM,QAAQ,EAAE,SAAS,SAAS;EAClC,IAAI,OAAO,MAAM,KAAK,UAAU,WAAW,CAAC,KAAK,EAAE;EACnD,MAAM,YAAY,cAAc,CAAC;EACjC,IAAI,UAAU,QAAQ;GACpB,MAAM,KAAK,gDAAgD,EAAE;GAC7D,KAAK,MAAM,QAAQ,WAAW,MAAM,KAAK,KAAK,KAAK,IAAI,GAAG;GAC1D,MAAM,KAAK,EAAE;EACf;EACA,IAAI,EAAE,SAAS,QAAQ;GACrB,KAAK,MAAM,KAAK,EAAE,SAAS;IACzB,MAAM,QAAQ,EAAE,QAAQ,2BAA2B,EAAE,MAAM,KAAK,KAAK;IACrE,MAAM,KAAK,eAAe,KAAK,aAAa,CAAC,CAAC,IAAI,OAAO;IACzD,KAAK,MAAM,QAAQ,gBAAgB,EAAE,KAAK,GAAG,MAAM,KAAK,OAAO,KAAK,IAAI,GAAG;IAC3E,KAAK,MAAM,UAAU,cAAc,EAAE,KAAK,GAAG,MAAM,KAAK,OAAO,KAAK,MAAM,GAAG;GAC/E;GACA,MAAM,KAAK,EAAE;EACf;EACA,MAAM,eAAe,MAAsB,cAAc,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,OAAO;EACnF,KAAK,MAAM,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,YAAY,CAAC,CAAC,GAAG;GAC/E,IAAI,EAAE,YAAY,WAAW;GAC7B,IAAI,cAAc,GAAG,CAAC,GAAG;IACvB,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,GAAG,uBAAuB,EAAE;IAC9D,MAAM,KAAK,6CAA6C,EAAE;IAC1D,cAAc,OAAO,CAAC;IACtB,MAAM,KAAK,IAAI,YAAY;IAC3B;GACF;GACA,IAAI,OAAO,MAAM,KAAK,QAAQ,aAAa,EAAE,SAAS,GAAG,EAAE,MAAM,EAAE;GACnE,cAAc,OAAO,CAAC;EACxB;CACF;CACA,MAAM,KAAK,EAAE;CACb,OAAO,MAAM,KAAK,IAAI;AACxB;;AAGA,MAAa,kBAAkB,MAC7B,EAAE,OAAO,SAAS,WAAW,EAAE,WAAW,MAAM,MAAM,EAAE,YAAY,MAAM;AAE5E,SAAS,cAAc,OAAiB,GAAwB;CAC9D,IAAI,EAAE,SAAS,CAAC,eAAe,CAAC,GAC9B,MAAM,KACJ,KAAK,EAAE,MAAM,KAAK,KAClB,IACA,OACA,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,GAClD,OACA,EACF;CAEF,KAAK,MAAM,QAAQ,EAAE,OAAO;EAC1B,MAAM,KACJ,aAAa,KAAK,OAAO,IAAI,eAAe,KAAK,UAAU,EAAE,KAAK,KAAK,KAAK,KAAK,MAAM,GAAG,GAAG,GAC/F;EACA,IAAI,KAAK,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,KAAK,KAAK,GAAG,GAAG,GAAG;CACxE;CACA,MAAM,IAAI,EAAE;CACZ,MAAM,KACJ,iBAAiB,eAAe,EAAE,OAAO,EAAE,WAAW,eAAe,EAAE,OAAO,EAAE,YAAY,eAAe,EAAE,QAAQ,EAAE,WAAW,eAAe,EAAE,OAAO,GAC5J;CACA,MAAM,MAAM,EAAE;CACd,MAAM,OAAO,GAAkB,SAC7B,MAAM,OAAO,GAAG,KAAK,iBAAiB,GAAG,EAAE,GAAG;CAChD,MAAM,KACJ,YAAY,IAAI,UAAU,gBAAgB,IAAI,MAAM,sBAAsB,IAAI,IAAI,aAAa,cAAc,EAAE,KAAK,IAAI,IAAI,YAAY,SAAS,EAAE,KAAK,eAAe,IAAI,WAAW,EAAE,YAAY,IAAI,eAAe,MAAM,IAAI,aAAa,eAAe,IAAI,iBAAiB,IAAI,KAAK,IAAI,QAAQ,IAAI,YAAY,aAAa,eAAe,IAAI,SAAS,MAAM,OAAO,IAC/W;CACA,KAAK,MAAM,EAAE,WAAW,GAAG,OAAO,cAAc,mBAAmB,CAAC,GAAG;EACrE,MAAM,UAAU,EAAE,SAAS,SACvB,WAAW,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,WAAW,IAAI,UAAU,cAClE;EACJ,MAAM,KACJ,UAAU,WAAW,IAAI,KAAK,KAAK,MAAM,GAAG,KAAK,EAAE,QAAQ,KAAK,EAAE,MAAM,QAAQ,CAAC,EAAE,cAAc,EAAE,UAAU,IAAI,EAAE,QAAQ,QAAQ,KAAK,EAAE,UAC5I;CACF;CACA,MAAM,SAAS,YAAY,CAAC;CAC5B,EAAE,OAAO,SAAS,GAAG,MAAM;EACzB,MAAM,QAAQ,OAAO;EACrB,MAAM,KACJ,UAAU,QAAQ,IAAI,KAAK,KAAK,MAAM,GAAG,QAAQ,EAAE,SAAS,MAAM,KAAK,YAAY,EAAE,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,WAAW,EAAE,IAAI,KAAK,MAAM,IACzI;CACF,CAAC;CACD,IAAI,EAAE,aAAa,QACjB,MAAM,KACJ,cAAc,EAAE,aAAa,KAAK,MAAO,EAAE,QAAQ,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,WAAW,GAAG,EAAE,OAAO,KAAK,EAAE,KAAK,GAAI,CAAC,CAAC,KAAK,IAAI,GACjI;CAEF,MAAM,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE;CAC7B,IAAI,MAAM;EACR,MAAM,QAAQ,KAAK,SAAS,qBAAqB,GAAG,KAAK,MAAM,GAAG,kBAAkB,EAAE,KAAK;EAC3F,MAAM,KACJ,IACA,iBACA,IACA,GAAG,MACA,QAAQ,CAAC,CACT,MAAM,IAAI,CAAC,CACX,KAAK,MAAM,KAAK,GAAG,GACtB,EACF;CACF;CACA,MAAM,QAAQ;EACZ,EAAE,MAAM,YAAY,eAAe,EAAE,MAAM,SAAS,KAAK;EACzD,EAAE,MAAM,cAAc,gBAAgB,EAAE,MAAM,WAAW,KAAK;EAC9D,EAAE,MAAM,QAAQ,UAAU,EAAE,MAAM,KAAK,KAAK;EAC5C,EAAE,MAAM,cAAc,iBAAiB,EAAE,MAAM,WAAW,KAAK;CACjE,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,MAAM,QAAQ,MAAM,KAAK,YAAY,MAAM,KAAK,KAAK,GAAG;AAC9D"}
|
package/dist/summary-DgyjUh_l.js
DELETED
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
import { a as providerModel } from "./model-h_cDtcRr.js";
|
|
2
|
-
import { l as writeCached } from "./files-DQmHE7pb.js";
|
|
3
|
-
import { r as fromRecorded, t as attemptSummary } from "./tools-GCyNYSc1.js";
|
|
4
|
-
import { n as allRuns } from "./commands-BFx16WB8.js";
|
|
5
|
-
import { a as failedAnalysis, i as defaultBase, o as prepareAnalysis, s as promptOf, t as analyseDigest } from "./command-Bge8F7Ko.js";
|
|
6
|
-
import { join } from "node:path";
|
|
7
|
-
import { z } from "zod";
|
|
8
|
-
import { generateText, stepCountIs, tool } from "ai";
|
|
9
|
-
/** Items the fake reads of each kind, as a careful model would: a few, not all of them. */
|
|
10
|
-
const FAKE_READS = 3;
|
|
11
|
-
const SUMMARY_TOOLS_PROMPT = `You are about to analyse a reeval run from the DIGEST in the prompt. First, read what the digest leaves out, with the tools:
|
|
12
|
-
- read_evidence: an evidence item dropped from the digest to fit its size cap (the ids in "dropped").
|
|
13
|
-
- open_attempt: one attempt's turns, steps, checks, verdicts and diff, by its id in the attempts table (e.g. r1/skill/control#1).
|
|
14
|
-
|
|
15
|
-
Open failing attempts, and attempts whose facts look odd; at most six items in all, never every attempt. Then reply with one line saying what you read, and stop: you write the analysis itself in the next step.`;
|
|
16
|
-
/** The tools over a prepared digest, with every result counted against `maxChars`. */
|
|
17
|
-
function summaryTools(p, maxChars) {
|
|
18
|
-
let left = maxChars;
|
|
19
|
-
const spend = (value) => {
|
|
20
|
-
const size = JSON.stringify(value).length;
|
|
21
|
-
if (size > left) return { error: "The input budget (analysis.maxInputTokens) is spent: write the analysis from what you have." };
|
|
22
|
-
left -= size;
|
|
23
|
-
return value;
|
|
24
|
-
};
|
|
25
|
-
const runs = [p.headRun, p.baseRun].flatMap((r) => r ? [r] : []);
|
|
26
|
-
return {
|
|
27
|
-
read_evidence: tool({
|
|
28
|
-
description: "An evidence item dropped from the digest to fit its cap: its lines (transcript lines keep their L<n>), what it is and why it was chosen.",
|
|
29
|
-
inputSchema: z.object({ id: z.string().describe("An evidence id from the digest’s dropped") }),
|
|
30
|
-
execute: async ({ id }) => {
|
|
31
|
-
const item = p.dropped.find((e) => e.id === id);
|
|
32
|
-
if (item) {
|
|
33
|
-
const { priority: _priority, ...rest } = item;
|
|
34
|
-
return spend(rest);
|
|
35
|
-
}
|
|
36
|
-
return { error: p.digest.evidence.some((e) => e.id === id) ? `${id} is in the digest already.` : `No dropped evidence ${id}. Dropped: ${p.digest.dropped.join(", ") || "none"}.` };
|
|
37
|
-
}
|
|
38
|
-
}),
|
|
39
|
-
open_attempt: tool({
|
|
40
|
-
description: "One attempt: each turn's prompt, steps (tool calls, edits, errors) and final message, its checks, verdicts and diff.",
|
|
41
|
-
inputSchema: z.object({ id: z.string().describe("An attempt id from the digest’s attempts table") }),
|
|
42
|
-
execute: async ({ id }) => {
|
|
43
|
-
const a = p.digest.attempts[id];
|
|
44
|
-
const run = a && runs.find((r) => r.record.runId === a.runId);
|
|
45
|
-
const record = run?.record.evals.flatMap((e) => e.attempts).find((x) => x.key === a?.key);
|
|
46
|
-
if (!a || !run || !record) return { error: `No attempt ${id}. The attempts are ${Object.keys(p.digest.attempts).join(", ")}.` };
|
|
47
|
-
const view = await fromRecorded(join(run.dir, run.record.runId), record);
|
|
48
|
-
return spend({
|
|
49
|
-
...attemptSummary(view),
|
|
50
|
-
id
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
})
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
const isError = (v) => typeof v === "object" && v !== null && "error" in v;
|
|
57
|
-
/** What the tools returned, as evidence to cite and as the text the structured call is shown. */
|
|
58
|
-
function toolReads(reads) {
|
|
59
|
-
const ok = reads.filter((r) => !isError(r.output));
|
|
60
|
-
const evidence = ok.filter((r) => r.tool === "read_evidence").map((r) => ({
|
|
61
|
-
...r.output,
|
|
62
|
-
priority: 1
|
|
63
|
-
}));
|
|
64
|
-
if (!ok.length) return {
|
|
65
|
-
evidence,
|
|
66
|
-
text: ""
|
|
67
|
-
};
|
|
68
|
-
return {
|
|
69
|
-
evidence,
|
|
70
|
-
text: `\n\nREAD WITH TOOLS (cite a dropped evidence item read here by its id, as you would the digest's):\n${ok.map((r) => `${r.tool} ${JSON.stringify(r.input)}: ${JSON.stringify(r.output)}`).join("\n")}`
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
/** Lets the model read with the tools; what it read, for the structured call. */
|
|
74
|
-
async function investigate(p, model, config) {
|
|
75
|
-
const result = await generateText({
|
|
76
|
-
model: model.fake ? fakeSummaryModel(p) : await providerModel(model),
|
|
77
|
-
system: SUMMARY_TOOLS_PROMPT,
|
|
78
|
-
prompt: promptOf(p.digest),
|
|
79
|
-
tools: summaryTools(p, config.analysis.maxInputTokens * 4),
|
|
80
|
-
stopWhen: stepCountIs(4),
|
|
81
|
-
prepareStep: ({ stepNumber }) => stepNumber === 3 ? { toolChoice: "none" } : void 0,
|
|
82
|
-
temperature: .2,
|
|
83
|
-
maxOutputTokens: config.analysis.maxOutputTokens,
|
|
84
|
-
maxRetries: 3
|
|
85
|
-
});
|
|
86
|
-
return {
|
|
87
|
-
reads: result.steps.flatMap((s) => s.toolResults.map((r) => ({
|
|
88
|
-
tool: r.toolName,
|
|
89
|
-
input: r.input,
|
|
90
|
-
output: r.output
|
|
91
|
-
}))),
|
|
92
|
-
usage: {
|
|
93
|
-
inputTokens: result.totalUsage?.inputTokens ?? 0,
|
|
94
|
-
outputTokens: result.totalUsage?.outputTokens ?? 0
|
|
95
|
-
}
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* Writes the run's summary into the cache. A failed call is cached as a document with `error`, never
|
|
100
|
-
* a throw; only reading the run (before any call) can throw.
|
|
101
|
-
*/
|
|
102
|
-
async function writeRunSummary(options) {
|
|
103
|
-
const { config, runId, model } = options;
|
|
104
|
-
const entries = await allRuns(config);
|
|
105
|
-
const head = entries.find((e) => e.runId === runId);
|
|
106
|
-
if (!head) throw new Error(`${runId} isn't in .reeval/runs`);
|
|
107
|
-
const prepared = await prepareAnalysis(config, head, await defaultBase(config, entries, head), null);
|
|
108
|
-
const started = Date.now();
|
|
109
|
-
let doc;
|
|
110
|
-
try {
|
|
111
|
-
const { reads, usage } = await investigate(prepared, model, config);
|
|
112
|
-
doc = await analyseDigest(config, prepared, model, toolReads(reads));
|
|
113
|
-
const plus = (a, b) => a === null ? null : a + b;
|
|
114
|
-
doc.usage = {
|
|
115
|
-
inputTokens: plus(doc.usage.inputTokens, usage.inputTokens),
|
|
116
|
-
outputTokens: plus(doc.usage.outputTokens, usage.outputTokens)
|
|
117
|
-
};
|
|
118
|
-
doc.latencyMs = Date.now() - started;
|
|
119
|
-
} catch (error) {
|
|
120
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
121
|
-
doc = await failedAnalysis(config, prepared, model, message, Date.now() - started);
|
|
122
|
-
}
|
|
123
|
-
const path = await writeCached(config.root, prepared.key, doc);
|
|
124
|
-
return {
|
|
125
|
-
doc,
|
|
126
|
-
key: prepared.key,
|
|
127
|
-
path
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
|
-
/**
|
|
131
|
-
* A language model that reads as the prompt asks a real one to: the dropped evidence and the
|
|
132
|
-
* failing attempts, a few of each, in one round, then says it's done, through the AI SDK's own tool
|
|
133
|
-
* loop, so the tools and the budget run as they would.
|
|
134
|
-
*/
|
|
135
|
-
function fakeSummaryModel(p) {
|
|
136
|
-
const usage = {
|
|
137
|
-
inputTokens: {
|
|
138
|
-
total: 0,
|
|
139
|
-
noCache: 0,
|
|
140
|
-
cacheRead: 0,
|
|
141
|
-
cacheWrite: 0
|
|
142
|
-
},
|
|
143
|
-
outputTokens: {
|
|
144
|
-
total: 0,
|
|
145
|
-
text: 0,
|
|
146
|
-
reasoning: 0
|
|
147
|
-
}
|
|
148
|
-
};
|
|
149
|
-
const answer = (content, finish) => ({
|
|
150
|
-
content,
|
|
151
|
-
finishReason: {
|
|
152
|
-
unified: finish,
|
|
153
|
-
raw: finish === "stop" ? "stop" : "tool_calls"
|
|
154
|
-
},
|
|
155
|
-
usage,
|
|
156
|
-
warnings: [],
|
|
157
|
-
response: { modelId: "fake-analysis" }
|
|
158
|
-
});
|
|
159
|
-
return {
|
|
160
|
-
specificationVersion: "v4",
|
|
161
|
-
provider: "fake",
|
|
162
|
-
modelId: "fake-analysis",
|
|
163
|
-
supportedUrls: {},
|
|
164
|
-
doGenerate: async (options) => {
|
|
165
|
-
const readSome = options.prompt.some((m) => m.role === "tool" && Array.isArray(m.content) && m.content.some((c) => c.type === "tool-result"));
|
|
166
|
-
const calls = [...p.digest.dropped.slice(0, FAKE_READS).map((id) => ({
|
|
167
|
-
name: "read_evidence",
|
|
168
|
-
input: { id }
|
|
169
|
-
})), ...Object.entries(p.digest.attempts).filter(([, a]) => a.outcome === "fail").slice(0, FAKE_READS).map(([id]) => ({
|
|
170
|
-
name: "open_attempt",
|
|
171
|
-
input: { id }
|
|
172
|
-
}))];
|
|
173
|
-
if (readSome || !calls.length) return answer([{
|
|
174
|
-
type: "text",
|
|
175
|
-
text: `Read ${readSome ? "what I needed" : "nothing"}.`
|
|
176
|
-
}], "stop");
|
|
177
|
-
return answer(calls.map((c, i) => ({
|
|
178
|
-
type: "tool-call",
|
|
179
|
-
toolCallId: `fake_${c.name}_${i}`,
|
|
180
|
-
toolName: c.name,
|
|
181
|
-
input: JSON.stringify(c.input)
|
|
182
|
-
})), "tool-calls");
|
|
183
|
-
},
|
|
184
|
-
doStream: async () => {
|
|
185
|
-
throw new Error("the fake analysis model does not stream");
|
|
186
|
-
}
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
//#endregion
|
|
190
|
-
export { writeRunSummary };
|
|
191
|
-
|
|
192
|
-
//# sourceMappingURL=summary-DgyjUh_l.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"summary-DgyjUh_l.js","names":[],"sources":["../src/analysis/agent/summary.ts"],"mappings":";;;;;;;;;AA4BA,MAAM,aAAa;AAEnB,MAAa,uBAAuB;;;;;;AAOpC,SAAgB,aAAa,GAAqB,UAAkB;CAClE,IAAI,OAAO;CACX,MAAM,SAAY,UAAoC;EACpD,MAAM,OAAO,KAAK,UAAU,KAAK,CAAC,CAAC;EACnC,IAAI,OAAO,MACT,OAAO,EACL,OACE,8FACJ;EAEF,QAAQ;EACR,OAAO;CACT;CACA,MAAM,OAAO,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,MAAO,IAAI,CAAC,CAAC,IAAI,CAAC,CAAE;CACjE,OAAO;EACL,eAAe,KAAK;GAClB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,0CAA+C,EACzE,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,OAAO,EAAE;IAC9C,IAAI,MAAM;KACR,MAAM,EAAE,UAAU,WAAW,GAAG,SAAS;KACzC,OAAO,MAAM,IAAI;IACnB;IACA,OAAO,EACL,OAAO,EAAE,OAAO,SAAS,MAAM,MAAM,EAAE,OAAO,EAAE,IAC5C,GAAG,GAAG,8BACN,uBAAuB,GAAG,aAAa,EAAE,OAAO,QAAQ,KAAK,IAAI,KAAK,OAAO,GACnF;GACF;EACF,CAAC;EACD,cAAc,KAAK;GACjB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,gDAAqD,EAC/E,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,IAAI,EAAE,OAAO,SAAS;IAC5B,MAAM,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,OAAO,UAAU,EAAE,KAAK;IAC5D,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,MAAM,EAAE,QAAQ,GAAG,GAAG;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QACjB,OAAO,EACL,OAAO,cAAc,GAAG,qBAAqB,OAAO,KAAK,EAAE,OAAO,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,GACzF;IAEF,MAAM,OAAO,MAAM,aAAa,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,GAAG,MAAM;IACvE,OAAO,MAAM;KAAE,GAAG,eAAe,IAAI;KAAG;IAAG,CAAC;GAC9C;EACF,CAAC;CACH;AACF;AAIA,MAAM,WAAW,MAAe,OAAO,MAAM,YAAY,MAAM,QAAQ,WAAW;;AAGlF,SAAgB,UAAU,OAA8B;CACtD,MAAM,KAAK,MAAM,QAAQ,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC;CACjD,MAAM,WAAW,GACd,QAAQ,MAAM,EAAE,SAAS,eAAe,CAAC,CACzC,KAAK,OAAO;EAAE,GAAI,EAAE;EAA6C,UAAU;CAAE,EAAE;CAClF,IAAI,CAAC,GAAG,QAAQ,OAAO;EAAE;EAAU,MAAM;CAAG;CAC5C,OAAO;EACL;EACA,MAAM,uGAAuG,GAC1G,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,KAAK,UAAU,EAAE,KAAK,EAAE,IAAI,KAAK,UAAU,EAAE,MAAM,GAAG,CAAC,CAC/E,KAAK,IAAI;CACd;AACF;;AAGA,eAAsB,YACpB,GACA,OACA,QACsF;CACtF,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,OAAO,iBAAiB,CAAC,IAAI,MAAM,cAAc,KAAK;EACnE,QAAQ;EACR,QAAQ,SAAS,EAAE,MAAM;EACzB,OAAO,aAAa,GAAG,OAAO,SAAS,iBAAiB,CAAC;EACzD,UAAU,YAAA,CAAyB;EACnC,cAAc,EAAE,iBACd,eAAe,IAAoB,EAAE,YAAY,OAAgB,IAAI,KAAA;EACvE,aAAa;EACb,iBAAiB,OAAO,SAAS;EACjC,YAAY;CACd,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,SAAS,MAC3B,EAAE,YAAY,KAAK,OAAO;GAAE,MAAM,EAAE;GAAU,OAAO,EAAE;GAAO,QAAQ,EAAE;EAAO,EAAE,CACnF;EACA,OAAO;GACL,aAAa,OAAO,YAAY,eAAe;GAC/C,cAAc,OAAO,YAAY,gBAAgB;EACnD;CACF;AACF;;;;;AAQA,eAAsB,gBAAgB,SAId;CACtB,MAAM,EAAE,QAAQ,OAAO,UAAU;CACjC,MAAM,UAAU,MAAM,QAAQ,MAAM;CACpC,MAAM,OAAO,QAAQ,MAAM,MAAM,EAAE,UAAU,KAAK;CAClD,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM,uBAAuB;CAC3D,MAAM,WAAW,MAAM,gBACrB,QACA,MACA,MAAM,YAAY,QAAQ,SAAS,IAAI,GACvC,IACF;CACA,MAAM,UAAU,KAAK,IAAI;CACzB,IAAI;CACJ,IAAI;EACF,MAAM,EAAE,OAAO,UAAU,MAAM,YAAY,UAAU,OAAO,MAAM;EAClE,MAAM,MAAM,cAAc,QAAQ,UAAU,OAAO,UAAU,KAAK,CAAC;EACnE,MAAM,QAAQ,GAAkB,MAAe,MAAM,OAAO,OAAO,IAAI;EACvE,IAAI,QAAQ;GACV,aAAa,KAAK,IAAI,MAAM,aAAa,MAAM,WAAW;GAC1D,cAAc,KAAK,IAAI,MAAM,cAAc,MAAM,YAAY;EAC/D;EACA,IAAI,YAAY,KAAK,IAAI,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrE,MAAM,MAAM,eAAe,QAAQ,UAAU,OAAO,SAAS,KAAK,IAAI,IAAI,OAAO;CACnF;CACA,MAAM,OAAO,MAAM,YAAY,OAAO,MAAM,SAAS,KAAK,GAAG;CAC7D,OAAO;EAAE;EAAK,KAAK,SAAS;EAAK;CAAK;AACxC;;;;;;AASA,SAAS,iBAAiB,GAAoC;CAC5D,MAAM,QAAQ;EACZ,aAAa;GAAE,OAAO;GAAG,SAAS;GAAG,WAAW;GAAG,YAAY;EAAE;EACjE,cAAc;GAAE,OAAO;GAAG,MAAM;GAAG,WAAW;EAAE;CAClD;CACA,MAAM,UAAU,SAAoB,YAAmC;EACrE;EACA,cAAc;GAAE,SAAS;GAAQ,KAAK,WAAW,SAAS,SAAS;EAAa;EAChF;EACA,UAAU,CAAC;EACX,UAAU,EAAE,SAAS,gBAAgB;CACvC;CACA,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAmE;GACpF,MAAM,WAAW,QAAQ,OAAO,MAC7B,MACC,EAAE,SAAS,UACX,MAAM,QAAQ,EAAE,OAAO,KACtB,EAAE,QAAyB,MAAM,MAAM,EAAE,SAAS,aAAa,CACpE;GACA,MAAM,QAAQ,CACZ,GAAG,EAAE,OAAO,QACT,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAQ;IAAE,MAAM;IAAiB,OAAO,EAAE,GAAG;GAAE,EAAE,GACzD,GAAG,OAAO,QAAQ,EAAE,OAAO,QAAQ,CAAC,CACjC,QAAQ,GAAG,OAAO,EAAE,YAAY,MAAM,CAAC,CACvC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,CAAC,SAAS;IAAE,MAAM;IAAgB,OAAO,EAAE,GAAG;GAAE,EAAE,CAC5D;GACA,IAAI,YAAY,CAAC,MAAM,QACrB,OAAO,OACL,CAAC;IAAE,MAAM;IAAQ,MAAM,QAAQ,WAAW,kBAAkB,UAAU;GAAG,CAAC,GAC1E,MACF;GAEF,OAAO,OACL,MAAM,KAAK,GAAG,OAAO;IACnB,MAAM;IACN,YAAY,QAAQ,EAAE,KAAK,GAAG;IAC9B,UAAU,EAAE;IACZ,OAAO,KAAK,UAAU,EAAE,KAAK;GAC/B,EAAE,GACF,YACF;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"text-gEcKjAyp.js","names":[],"sources":["../src/text.ts"],"mappings":";;AAGA,SAAgB,eAAe,IAAoB;CACjD,IAAI,KAAK,KAAM,OAAO,GAAG,KAAK,MAAM,EAAE,EAAE;CACxC,MAAM,IAAI,KAAK,MAAM,KAAK,GAAI;CAC9B,OAAO,IAAI,KAAK,GAAG,EAAE,KAAK,GAAG,KAAK,MAAM,IAAI,EAAE,EAAE,IAAI,OAAO,IAAI,EAAE,CAAC,CAAC,SAAS,GAAG,GAAG,EAAE;AACtF;;AAGA,SAAgB,KAAK,MAAc,KAAqB;CACtD,IAAI,KAAK,UAAU,KAAK,OAAO;CAC/B,MAAM,MAAM,KAAK,MAAM,GAAG,MAAM,CAAC;CACjC,MAAM,QAAQ,IAAI,OAAO,QAAQ;CACjC,OAAO,IAAI,QAAQ,MAAM,IAAI,IAAI,MAAM,GAAG,KAAK,IAAI,IAAA,CAAK,QAAQ,EAAE;AACpE;;AAGA,MAAa,cAAc;;AAG3B,SAAgB,UAAU,MAAc,MAAM,aAAqB;CACjE,IAAI,KAAK,UAAU,KAAK,OAAO;CAC/B,MAAM,OAAO,KAAK,MAAM,MAAM,GAAI;CAClC,MAAM,OAAO,MAAM;CACnB,OAAO,GAAG,KAAK,MAAM,GAAG,IAAI,EAAE,OAAO,KAAK,SAAS,OAAO,KAAK,sBAAsB,KAAK,MAAM,CAAC,IAAI;AACvG;;AAGA,MAAa,sBAAsB,MACjC,GAAG,EAAE,SAAS,SAAS,YAAY,UAAU,IAAI,EAAE"}
|