reeval 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +16 -16
- package/THIRD_PARTY_NOTICES.md +0 -2068
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +65 -50
- package/dist/{adapter-dHdepWKb.js → adapter-BHMbIeH9.js} +7 -7
- package/dist/{adapter-dHdepWKb.js.map → adapter-BHMbIeH9.js.map} +1 -1
- package/dist/{agent-CZpiwA_r.js → agent-D67VCd6Q.js} +8 -8
- package/dist/{agent-CZpiwA_r.js.map → agent-D67VCd6Q.js.map} +1 -1
- package/dist/{agents-CEapsHV-.js → agents-Dnv1MmgY.js} +40 -59
- package/dist/agents-Dnv1MmgY.js.map +1 -0
- package/dist/{analysis-BnG_1x_O.js → analysis-8vvMnUJB.js} +4 -5
- package/dist/analysis-8vvMnUJB.js.map +1 -0
- package/dist/api-ZeGnedB3.js +141 -0
- package/dist/api-ZeGnedB3.js.map +1 -0
- package/dist/assemble-BlA7FWRn.js +1 -0
- package/dist/{assemble-Dw1bF13I.js → assemble-CuvlsOaE.js} +117 -66
- package/dist/assemble-CuvlsOaE.js.map +1 -0
- package/dist/{attach-DvYiMu6w.js → attach-6av4DmO3.js} +24 -6
- package/dist/attach-6av4DmO3.js.map +1 -0
- package/dist/{attempt-DxR1agFo.js → attempt-D5WH0yFa.js} +3 -3
- package/dist/{attempt-DxR1agFo.js.map → attempt-D5WH0yFa.js.map} +1 -1
- package/dist/{attempt-host-DgeJFSZw.js → attempt-host-Be_-uDKJ.js} +37 -14
- package/dist/attempt-host-Be_-uDKJ.js.map +1 -0
- package/dist/boat-BCNNEJni.js +298 -0
- package/dist/boat-BCNNEJni.js.map +1 -0
- package/dist/boat-CwBW2sbp.js +1 -0
- package/dist/{boat-CUNgSc75.js → boat-CygRcaM0.js} +115 -196
- package/dist/boat-CygRcaM0.js.map +1 -0
- package/dist/{boat-images-BuFigiEb.js → boat-images-asGLQXD5.js} +19 -23
- package/dist/boat-images-asGLQXD5.js.map +1 -0
- package/dist/{bundle-yEgI3RsI.js → bundle-B7P-Awnm.js} +105 -17
- package/dist/bundle-B7P-Awnm.js.map +1 -0
- package/dist/{capture-DdEKVrwk.js → capture-BwLa-4s-.js} +20 -19
- package/dist/{capture-DdEKVrwk.js.map → capture-BwLa-4s-.js.map} +1 -1
- package/dist/capture-CG-J-1-j.js +1 -0
- package/dist/cases-Bsp-G-ZL.js +1 -0
- package/dist/{cases-B7QkvMGE.js → cases-C6vnV-gn.js} +60 -42
- package/dist/cases-C6vnV-gn.js.map +1 -0
- package/dist/classify-BIJQ2Xqo.js +163 -0
- package/dist/classify-BIJQ2Xqo.js.map +1 -0
- package/dist/{cli-Z7aw8k25.js → cli-BoCPYLUQ.js} +305 -124
- package/dist/cli-BoCPYLUQ.js.map +1 -0
- package/dist/{cli-spec-3zvLC9YW.js → cli-spec-DgEWHOt3.js} +164 -50
- package/dist/cli-spec-DgEWHOt3.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client-Dj65gXG3.js +105 -0
- package/dist/client-Dj65gXG3.js.map +1 -0
- package/dist/{cloud-owVBaSRt.js → cloud-TuNUlZ0L.js} +3 -3
- package/dist/{cloud-owVBaSRt.js.map → cloud-TuNUlZ0L.js.map} +1 -1
- package/dist/{cloud-files-BjinrTTe.js → cloud-files-CBovrfZA.js} +8 -8
- package/dist/{cloud-files-BjinrTTe.js.map → cloud-files-CBovrfZA.js.map} +1 -1
- package/dist/{cloud-DuHcNQVV.js → cloud-igGIS4Bm.js} +43 -20
- package/dist/cloud-igGIS4Bm.js.map +1 -0
- package/dist/cloud-project-Ch7jjlwh.js +110 -0
- package/dist/cloud-project-Ch7jjlwh.js.map +1 -0
- package/dist/cloud-project-DdUVB4EE.js +1 -0
- package/dist/cloud-push-DvnqOgRd.js +817 -0
- package/dist/cloud-push-DvnqOgRd.js.map +1 -0
- package/dist/cloud-push-MXKreNyq.js +1 -0
- package/dist/{cloud-run-DqMkSGYB.js → cloud-run-CbGF8Lwt.js} +285 -48
- package/dist/cloud-run-CbGF8Lwt.js.map +1 -0
- package/dist/cloud-run-CghE2dG6.js +1 -0
- package/dist/{cloud-secrets-Bak83tik.js → cloud-secrets-BWoV4ddM.js} +43 -35
- package/dist/cloud-secrets-BWoV4ddM.js.map +1 -0
- package/dist/cloud-secrets-DlPRfS2l.js +1 -0
- package/dist/command-Bps8kBML.js +141 -0
- package/dist/command-Bps8kBML.js.map +1 -0
- package/dist/{command-CtmSJzPC.js → command-ByUmSm2D.js} +29 -23
- package/dist/command-ByUmSm2D.js.map +1 -0
- package/dist/{command-CeygdoVe.js → command-CQk7FCI1.js} +92 -26
- package/dist/command-CQk7FCI1.js.map +1 -0
- package/dist/{command-CoHN1ibe.js → command-CxuGky7A.js} +44 -20
- package/dist/command-CxuGky7A.js.map +1 -0
- package/dist/{command-efTGfUYb.js → command-pxfK2bkV.js} +26 -16
- package/dist/command-pxfK2bkV.js.map +1 -0
- package/dist/commands-Cl3Y_UXg.js +1 -0
- package/dist/{commands-DR-NolV2.js → commands-DGWNNyRD.js} +54 -38
- package/dist/commands-DGWNNyRD.js.map +1 -0
- package/dist/{config-Bn2OhCN_.js → config-mU7wFvQq.js} +46 -32
- package/dist/config-mU7wFvQq.js.map +1 -0
- package/dist/core-BXYzOEyQ.js +2058 -0
- package/dist/core-BXYzOEyQ.js.map +1 -0
- package/dist/{slug-IHh3f770.js → cost-DQ0ayy6h.js} +37 -2
- package/dist/cost-DQ0ayy6h.js.map +1 -0
- package/dist/{cursor-catalog-BYIDSsLn.js → cursor-catalog-EOFft6LJ.js} +4 -4
- package/dist/{cursor-catalog-BYIDSsLn.js.map → cursor-catalog-EOFft6LJ.js.map} +1 -1
- package/dist/{daytona-CWs9j9mt.js → daytona-DtCjv8rH.js} +22 -17
- package/dist/daytona-DtCjv8rH.js.map +1 -0
- package/dist/daytona-image-Xy5o53Ji.js +1 -0
- package/dist/{daytona-image-DmGlFAyq.js → daytona-image-w4uTRZ0z.js} +6 -6
- package/dist/{daytona-image-DmGlFAyq.js.map → daytona-image-w4uTRZ0z.js.map} +1 -1
- package/dist/db-BcplCdzu.js +1 -0
- package/dist/{db-D6yQib7M.js → db-yigMuHM2.js} +31 -119
- package/dist/db-yigMuHM2.js.map +1 -0
- package/dist/debug-CGg7KHKq.js +1 -0
- package/dist/{debug-CIXzB4wK.js → debug-T29p6HOX.js} +2 -10
- package/dist/debug-T29p6HOX.js.map +1 -0
- package/dist/{default-effort-Ck6ViSaI.js → default-effort-B1b8t81b.js} +1 -1
- package/dist/{default-effort-Bpz5ULfC.js → default-effort-BDYXBzu5.js} +3 -3
- package/dist/{default-effort-Bpz5ULfC.js.map → default-effort-BDYXBzu5.js.map} +1 -1
- package/dist/{dist-CZtW7eI5.js → dist-DCG7vZGQ.js} +3 -3
- package/dist/{dist-CZtW7eI5.js.map → dist-DCG7vZGQ.js.map} +1 -1
- package/dist/{docker-Dgw4aaRk.js → docker-DcnX2WpM.js} +4 -4
- package/dist/{docker-Dgw4aaRk.js.map → docker-DcnX2WpM.js.map} +1 -1
- package/dist/{docker-Bbi4kmFj.js → docker-XCyDUd7_.js} +1 -1
- package/dist/{doctor-BfS6jswW.js → doctor-DTKa8rrq.js} +27 -27
- package/dist/{doctor-BfS6jswW.js.map → doctor-DTKa8rrq.js.map} +1 -1
- package/dist/duration-Bz2H7pXb.js +34 -0
- package/dist/duration-Bz2H7pXb.js.map +1 -0
- package/dist/{environment-BY8lVFFK.js → environment-BCBg6c0p.js} +38 -12
- package/dist/environment-BCBg6c0p.js.map +1 -0
- package/dist/errors-BjzEWESY.js +93 -0
- package/dist/errors-BjzEWESY.js.map +1 -0
- package/dist/{errors-P8oHhNzy.js → errors-Cda7UzRA.js} +2 -2
- package/dist/{errors-P8oHhNzy.js.map → errors-Cda7UzRA.js.map} +1 -1
- package/dist/{eval-source-BB8In3xo.js → eval-source-CSPyKuV6.js} +6 -6
- package/dist/{eval-source-BB8In3xo.js.map → eval-source-CSPyKuV6.js.map} +1 -1
- package/dist/{evaluate-B9_nukrc.d.ts → evaluate-Nnh0iPuo.d.ts} +1 -1
- package/dist/{evaluate-B8I9AKgw.js → evaluate-wgW1lGQj.js} +280 -60
- package/dist/evaluate-wgW1lGQj.js.map +1 -0
- package/dist/{events-D97ZzAeq.js → events-DG_Cq5Pn.js} +154 -33
- package/dist/events-DG_Cq5Pn.js.map +1 -0
- package/dist/events-RaK1pDmT.js +1 -0
- package/dist/{fake-bootstrap-4DFtzib0.js → fake-bootstrap-uCyHhnMO.js} +2 -2
- package/dist/{fake-bootstrap-4DFtzib0.js.map → fake-bootstrap-uCyHhnMO.js.map} +1 -1
- package/dist/files-BnsRehge.js +1 -0
- package/dist/{files-B0DbptiO.js → files-Bv6ipFjc.js} +4 -4
- package/dist/{files-B0DbptiO.js.map → files-Bv6ipFjc.js.map} +1 -1
- package/dist/{files-GBTUBAek.js → files-CcnSILye.js} +3 -3
- package/dist/{files-GBTUBAek.js.map → files-CcnSILye.js.map} +1 -1
- package/dist/{files-BesexM88.js → files-CjPm3yyj.js} +173 -41
- package/dist/files-CjPm3yyj.js.map +1 -0
- package/dist/files-CkmCV5rH.js +1 -0
- package/dist/{files-DE13BqBC.js → files-ClUbQPj9.js} +2 -2
- package/dist/{files-DE13BqBC.js.map → files-ClUbQPj9.js.map} +1 -1
- package/dist/{files-B4K1k__s.js → files-CqfVnRXw.js} +5 -5
- package/dist/{files-B4K1k__s.js.map → files-CqfVnRXw.js.map} +1 -1
- package/dist/files-D4Hb9HZS.js +1 -0
- package/dist/files-DCcRkJN4.js +1 -0
- package/dist/{files-TwSVeWwM.js → files-GAocj_Z1.js} +27 -41
- package/dist/files-GAocj_Z1.js.map +1 -0
- package/dist/gateway.d.ts +1 -1
- package/dist/{git-DatgUcnf.js → git-CRPr6I2T.js} +3 -3
- package/dist/{git-DatgUcnf.js.map → git-CRPr6I2T.js.map} +1 -1
- package/dist/{harness-FcTyyPco.js → harness-Dcd3ffkY.js} +3 -2
- package/dist/harness-Dcd3ffkY.js.map +1 -0
- package/dist/{harness-EZNoeUBf.js → harness-DvqNGwl6.js} +2 -2
- package/dist/{harness-EZNoeUBf.js.map → harness-DvqNGwl6.js.map} +1 -1
- package/dist/{harness-install-HgR73tpF.js → harness-install-B0Y2GjnO.js} +4 -4
- package/dist/{harness-install-HgR73tpF.js.map → harness-install-B0Y2GjnO.js.map} +1 -1
- package/dist/harness.d.ts +1 -1
- package/dist/harness.js +1 -1
- package/dist/http-DtlSzw4A.js +96 -0
- package/dist/http-DtlSzw4A.js.map +1 -0
- package/dist/{ids-DqEvrBFZ.js → ids-CtxRn13a.js} +4 -3
- package/dist/ids-CtxRn13a.js.map +1 -0
- package/dist/{images-SRE4f3SM.js → images-JspTw1vt.js} +13 -13
- package/dist/{images-SRE4f3SM.js.map → images-JspTw1vt.js.map} +1 -1
- package/dist/index.d.ts +7 -2
- package/dist/index.js +4 -5
- package/dist/init-BHIWfE2L.js +1 -0
- package/dist/{init-D5F9MYPC.js → init-wmMbauok.js} +6 -6
- package/dist/{init-D5F9MYPC.js.map → init-wmMbauok.js.map} +1 -1
- package/dist/{ink-list-CLq6YVB8.js → ink-list-B5Gul5iZ.js} +7 -6
- package/dist/ink-list-B5Gul5iZ.js.map +1 -0
- package/dist/{inputs-B_FsOD2N.js → inputs-BWTtz1-v.js} +6 -6
- package/dist/{inputs-B_FsOD2N.js.map → inputs-BWTtz1-v.js.map} +1 -1
- package/dist/inputs-load-C8HEtAej.js +1 -0
- package/dist/{inputs-load-B4MFkp0Q.js → inputs-load-DQRC3dUA.js} +3 -3
- package/dist/{inputs-load-B4MFkp0Q.js.map → inputs-load-DQRC3dUA.js.map} +1 -1
- package/dist/internal/vm-api/boat.d.ts +46 -0
- package/dist/internal/vm-api/boat.js +1 -0
- package/dist/internal/vm-api/daytona.d.ts +26 -0
- package/dist/internal/vm-api/daytona.js +220 -0
- package/dist/internal/vm-api/daytona.js.map +1 -0
- package/dist/internal/vm-api.d.ts +86 -0
- package/dist/internal/vm-api.js +3 -0
- package/dist/internal/vm.d.ts +1 -1
- package/dist/internal/vm.js +1 -1
- package/dist/{invocations-BxQqe5GU.js → invocations-CSA9OKFn.js} +5 -5
- package/dist/{invocations-BxQqe5GU.js.map → invocations-CSA9OKFn.js.map} +1 -1
- package/dist/issue-ipG9Yy1h.js +1 -0
- package/dist/{issue-iErT9KkK.js → issue-lKqtL3iX.js} +4 -4
- package/dist/{issue-iErT9KkK.js.map → issue-lKqtL3iX.js.map} +1 -1
- package/dist/{jiti-CtahaHaY.js → jiti-aNfS_lSj.js} +2 -2
- package/dist/{jiti-CtahaHaY.js.map → jiti-aNfS_lSj.js.map} +1 -1
- package/dist/{judge-MOKPsr6R.js → judge-etHC4K02.js} +5 -5
- package/dist/{judge-MOKPsr6R.js.map → judge-etHC4K02.js.map} +1 -1
- package/dist/judge.d.ts +1 -1
- package/dist/judge.js +1 -1
- package/dist/{judgments-Dur4D2pj.js → judgments-CVCUWT-g.js} +7 -4
- package/dist/{judgments-Dur4D2pj.js.map → judgments-CVCUWT-g.js.map} +1 -1
- package/dist/{kinds-Dohijzi-.js → kinds-BQX5ZxpK.js} +37 -11
- package/dist/{kinds-Dohijzi-.js.map → kinds-BQX5ZxpK.js.map} +1 -1
- package/dist/labels-Dtyf5-pd.js +251 -0
- package/dist/labels-Dtyf5-pd.js.map +1 -0
- package/dist/{layout-BhhJAEgf.js → layout-Co4tPHWI.js} +3 -3
- package/dist/{layout-BhhJAEgf.js.map → layout-Co4tPHWI.js.map} +1 -1
- package/dist/link-Cqs2DhxR.js +127 -0
- package/dist/link-Cqs2DhxR.js.map +1 -0
- package/dist/{list-ClDb8qiX.js → list-CjcNH-5e.js} +10 -7
- package/dist/list-CjcNH-5e.js.map +1 -0
- package/dist/{live-QFILMmql.js → live-BRF0ogR6.js} +3 -5
- package/dist/live-BRF0ogR6.js.map +1 -0
- package/dist/live-DTpM9b4D.js +56 -0
- package/dist/live-DTpM9b4D.js.map +1 -0
- package/dist/{machine-service-xFhakTs5.js → machine-service-BetvBWHz.js} +146 -13
- package/dist/machine-service-BetvBWHz.js.map +1 -0
- package/dist/{machine-service-DpJScNtA.js → machine-service-XFymGCHj.js} +1 -1
- package/dist/{machines-Cm-rU7jp.js → machines-CkDTbMZ6.js} +25 -25
- package/dist/{machines-Cm-rU7jp.js.map → machines-CkDTbMZ6.js.map} +1 -1
- package/dist/markdown-COts-jxf.js +653 -0
- package/dist/markdown-COts-jxf.js.map +1 -0
- package/dist/{model-Rmv-L9Jb.js → model-BYHXnYLe.js} +1 -1
- package/dist/model-CrsA2Evf.js +1 -0
- package/dist/{model-CcvSrrq9.js → model-DNC53cPL.js} +4 -4
- package/dist/{model-CcvSrrq9.js.map → model-DNC53cPL.js.map} +1 -1
- package/dist/{model-D9WMMEWD.js → model-F4UH_YG6.js} +4 -4
- package/dist/{model-D9WMMEWD.js.map → model-F4UH_YG6.js.map} +1 -1
- package/dist/{model-mTaaEkeQ.js → model-xemZQ7DK.js} +251 -65
- package/dist/model-xemZQ7DK.js.map +1 -0
- package/dist/{node-Bl2CtSjB.js → node-gPT36n_n.js} +19 -8
- package/dist/{node-Bl2CtSjB.js.map → node-gPT36n_n.js.map} +1 -1
- package/dist/options-CAZfxE0h.js +1 -0
- package/dist/otlp-BxPnO99e.js +1 -0
- package/dist/otlp-DbNz-occ.js +25 -0
- package/dist/otlp-DbNz-occ.js.map +1 -0
- package/dist/outcome-1LBk2e3w.js +38 -0
- package/dist/outcome-1LBk2e3w.js.map +1 -0
- package/dist/{patch-DhJDHRFs.js → patch-Dxzv8msl.js} +3 -3
- package/dist/{patch-DhJDHRFs.js.map → patch-Dxzv8msl.js.map} +1 -1
- package/dist/{patch-C7yBztZj.js → patch-uwo6t4Zy.js} +26 -2
- package/dist/patch-uwo6t4Zy.js.map +1 -0
- package/dist/{pi-providers-HygmL1y-.js → pi-providers-BjcQv2A1.js} +23 -9
- package/dist/{pi-providers-HygmL1y-.js.map → pi-providers-BjcQv2A1.js.map} +1 -1
- package/dist/{plain-Ctn8oUdp.js → plain-DXUNwqpO.js} +2 -2
- package/dist/{plain-Ctn8oUdp.js.map → plain-DXUNwqpO.js.map} +1 -1
- package/dist/{plan-CC0bKbm9.js → plan-DSJX7Oyq.js} +9 -9
- package/dist/{plan-CC0bKbm9.js.map → plan-DSJX7Oyq.js.map} +1 -1
- package/dist/{plugins-PwEtW8GH.js → plugins-DVEOMsYw.js} +7 -7
- package/dist/{plugins-PwEtW8GH.js.map → plugins-DVEOMsYw.js.map} +1 -1
- package/dist/{preflight-BHUEfA4m.js → preflight-B3oLZttl.js} +4 -4
- package/dist/{preflight-BHUEfA4m.js.map → preflight-B3oLZttl.js.map} +1 -1
- package/dist/preflight-CLYRuQ3r.js +1 -0
- package/dist/{preflight-DEV_vrdB.js → preflight-DXMD6jou.js} +1 -1
- package/dist/{preflight-BzvLYQ70.js → preflight-dUe0QiPK.js} +9 -9
- package/dist/{preflight-BzvLYQ70.js.map → preflight-dUe0QiPK.js.map} +1 -1
- package/dist/{prepare-D7hPrh7Q.js → prepare-BTaX7vcC.js} +8 -8
- package/dist/{prepare-D7hPrh7Q.js.map → prepare-BTaX7vcC.js.map} +1 -1
- package/dist/{prepare-DS42k409.js → prepare-QdiYPOwR.js} +2 -2
- package/dist/prices-CfnkJOth.js +1 -0
- package/dist/{prices-DXJsP2Ha.js → prices-CoLymMRd.js} +3 -3
- package/dist/{prices-DXJsP2Ha.js.map → prices-CoLymMRd.js.map} +1 -1
- package/dist/{process-RPlEhpHz.js → process-C_nh1H7X.js} +9 -3
- package/dist/process-C_nh1H7X.js.map +1 -0
- package/dist/provenance-Cylt_YxT.d.ts +222 -0
- package/dist/{provenance-DgQJ-dq7.js → provenance-Di42xQIS.js} +5 -5
- package/dist/{provenance-DgQJ-dq7.js.map → provenance-Di42xQIS.js.map} +1 -1
- package/dist/provider-B24hZX95.js +1 -0
- package/dist/{provider-ieWOn4YS.js → provider-Cn7lmECy.js} +2 -2
- package/dist/{provider-ieWOn4YS.js.map → provider-Cn7lmECy.js.map} +1 -1
- package/dist/{provider-stop-CxzWW7Ft.js → provider-stop-DjBMfZW5.js} +2 -2
- package/dist/{provider-stop-CxzWW7Ft.js.map → provider-stop-DjBMfZW5.js.map} +1 -1
- package/dist/{providers-N4HDMeeo.js → providers-Bm5NLaRh.js} +1 -1
- package/dist/{providers-B5Vr0hKH.js → providers-CTTfYbm0.js} +28 -28
- package/dist/{providers-B5Vr0hKH.js.map → providers-CTTfYbm0.js.map} +1 -1
- package/dist/query-CTMdWXbx.js +1 -0
- package/dist/{query-CmTjYbqh.js → query-TzNtTvwf.js} +2 -2
- package/dist/{query-CmTjYbqh.js.map → query-TzNtTvwf.js.map} +1 -1
- package/dist/{questions-DnqkJga0.js → questions-BqisU7A0.js} +19 -13
- package/dist/questions-BqisU7A0.js.map +1 -0
- package/dist/{recipe-B0NeT_Sn.js → recipe-DzjXne35.js} +4 -4
- package/dist/{recipe-B0NeT_Sn.js.map → recipe-DzjXne35.js.map} +1 -1
- package/dist/{record-2liAelrB.js → record-Cj3yWR0E.js} +43 -12
- package/dist/record-Cj3yWR0E.js.map +1 -0
- package/dist/{redact-DMueYS-v.js → redact-CfTHYWLv.js} +30 -1
- package/dist/redact-CfTHYWLv.js.map +1 -0
- package/dist/{remarks-D5qOZzcX.js → remarks-BPKGorBW.js} +4 -2
- package/dist/remarks-BPKGorBW.js.map +1 -0
- package/dist/{remarks-D-9f1y1B.js → remarks-CYXYiB_O.js} +21 -11
- package/dist/{remarks-D-9f1y1B.js.map → remarks-CYXYiB_O.js.map} +1 -1
- package/dist/{remote-B7BIRUTK.js → remote-D4UEa7Rs.js} +10 -8
- package/dist/{remote-B7BIRUTK.js.map → remote-D4UEa7Rs.js.map} +1 -1
- package/dist/renderer/ascii.d.ts +6 -0
- package/dist/renderer/ascii.js +7 -0
- package/dist/renderer/ascii.js.map +1 -0
- package/dist/renderer/markdown.d.ts +31 -0
- package/dist/renderer/markdown.js +1 -0
- package/dist/report-BSi-R8hC.js +941 -0
- package/dist/report-BSi-R8hC.js.map +1 -0
- package/dist/report-inputs-5XwZBIRK.js +113 -0
- package/dist/report-inputs-5XwZBIRK.js.map +1 -0
- package/dist/{resolve-B60w6DcB.js → resolve-BEltmYhD.js} +6 -5
- package/dist/resolve-BEltmYhD.js.map +1 -0
- package/dist/resolve-C7A5Q2Jj.js +1 -0
- package/dist/{resolve-t9sOkGGS.js → resolve-nRzGt4Ff.js} +30 -13
- package/dist/resolve-nRzGt4Ff.js.map +1 -0
- package/dist/{results-Df910ONm.js → results-DXBaxfIM.js} +11 -7
- package/dist/{results-Df910ONm.js.map → results-DXBaxfIM.js.map} +1 -1
- package/dist/{resume-4Q9q8AcE.js → resume-DZPjjToJ.js} +46 -21
- package/dist/resume-DZPjjToJ.js.map +1 -0
- package/dist/{retry-C15f5yyy.js → retry-rLdXCinf.js} +5 -5
- package/dist/{retry-C15f5yyy.js.map → retry-rLdXCinf.js.map} +1 -1
- package/dist/{root-Cm93-6ix.js → root-Co0jhOT5.js} +2 -2
- package/dist/{root-Cm93-6ix.js.map → root-Co0jhOT5.js.map} +1 -1
- package/dist/run-D_o_8Fkv.js +128 -0
- package/dist/run-D_o_8Fkv.js.map +1 -0
- package/dist/{client-kUwtq-vt.js → runtime-DsRjnydE.js} +525 -105
- package/dist/runtime-DsRjnydE.js.map +1 -0
- package/dist/sandbox/boat.d.ts +1 -1
- package/dist/sandbox/boat.js +2 -2
- package/dist/sandbox/daytona.d.ts +1 -1
- package/dist/sandbox/daytona.js +1 -1
- package/dist/{sandbox-BffNervX.js → sandbox-B8gDRs5E.js} +3 -3
- package/dist/{sandbox-BffNervX.js.map → sandbox-B8gDRs5E.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/{secrets-file-Dtd3h3kY.js → secrets-file-B8rUJV2L.js} +4 -4
- package/dist/{secrets-file-Dtd3h3kY.js.map → secrets-file-B8rUJV2L.js.map} +1 -1
- package/dist/server-eUWbjZoi.js +1 -0
- package/dist/server-mJzPXBMM.js +204 -0
- package/dist/server-mJzPXBMM.js.map +1 -0
- package/dist/{services-BG8xuU82.js → services-CHh83UDC.js} +4 -2
- package/dist/services-CHh83UDC.js.map +1 -0
- package/dist/{shared-DtZwIAfy.js → shared-bXaqVxk-.js} +5 -34
- package/dist/shared-bXaqVxk-.js.map +1 -0
- package/dist/skill-dirs-B7o83G5I.js +63 -0
- package/dist/skill-dirs-B7o83G5I.js.map +1 -0
- package/dist/source-C6JgdJdJ.js +1 -0
- package/dist/{source-zoRWRMql.js → source-DxgCge_u.js} +33 -18
- package/dist/source-DxgCge_u.js.map +1 -0
- package/dist/spans-B4MwgEyy.js +1 -0
- package/dist/spans-Dq3fo6HM.js +6 -0
- package/dist/spans-Dq3fo6HM.js.map +1 -0
- package/dist/spans-tf3biLpT.js +72 -0
- package/dist/spans-tf3biLpT.js.map +1 -0
- package/dist/src-B2psni1B.js +1 -0
- package/dist/{src-DiS13D_q.js → src-BI35cTig.js} +2 -2
- package/dist/src-BLePHsjw.js +2 -0
- package/dist/{src-Bj3WJs3t.js → src-CftydE9Q.js} +2 -2
- package/dist/{src-CDoAEO4u.js → src-ClMhIgi3.js} +2 -2
- package/dist/{src-CDoAEO4u.js.map → src-ClMhIgi3.js.map} +1 -1
- package/dist/{src-CUrcaZ91.js → src-CnOj4_n4.js} +1 -1
- package/dist/src-Cs3nmbrr.js +2 -0
- package/dist/{src-TMKittwH.js → src-D0GezGkJ.js} +2 -2
- package/dist/{src-OC89dAjO.js → src-D1Q6ElaF.js} +182 -36
- package/dist/src-D1Q6ElaF.js.map +1 -0
- package/dist/{src-CFBg2ecq.js → src-D_9qqp-Y.js} +2 -2
- package/dist/{src-DwcTn4JY.js → src-DhZuOcgq.js} +5 -5
- package/dist/{src-DwcTn4JY.js.map → src-DhZuOcgq.js.map} +1 -1
- package/dist/{src-CGBwMZgM.js → src-Dm2YIeXn.js} +8 -8
- package/dist/{src-CGBwMZgM.js.map → src-Dm2YIeXn.js.map} +1 -1
- package/dist/{src-CcbV6VpV.js → src-Dpzfe-vl.js} +3 -3
- package/dist/{src-c3ZQu7Uu.js → src-NYxoDS3b.js} +5 -5
- package/dist/{src-c3ZQu7Uu.js.map → src-NYxoDS3b.js.map} +1 -1
- package/dist/{src-C3OhA5pw.js → src-Y4AiZxLt.js} +5 -5
- package/dist/{src-C3OhA5pw.js.map → src-Y4AiZxLt.js.map} +1 -1
- package/dist/{src-BQWAxhKD.js → src-YpmRtJmP.js} +3 -3
- package/dist/{src-BQWAxhKD.js.map → src-YpmRtJmP.js.map} +1 -1
- package/dist/{src-Bb3k9Kn_.js → src-nIPDQ297.js} +66 -11
- package/dist/{src-Bb3k9Kn_.js.map → src-nIPDQ297.js.map} +1 -1
- package/dist/ssh-pool-B9bnSQIj.js +1 -0
- package/dist/{ssh-pool-DwboLN9O.js → ssh-pool-Gv8gAsgZ.js} +19 -16
- package/dist/ssh-pool-Gv8gAsgZ.js.map +1 -0
- package/dist/{start-BwMyDneA.js → start-YOYgbaNU.js} +45 -13
- package/dist/start-YOYgbaNU.js.map +1 -0
- package/dist/state-Bvsi2WgZ.js +1 -0
- package/dist/{state-CXTaT0t8.js → state-CJlYQtdl.js} +21 -38
- package/dist/state-CJlYQtdl.js.map +1 -0
- package/dist/store/local.d.ts +1 -1
- package/dist/store/s3.d.ts +1 -1
- package/dist/studio-share-4T6GYO7l.js +209 -0
- package/dist/studio-share-4T6GYO7l.js.map +1 -0
- package/dist/suite-fO-u8jlS.js +2 -0
- package/dist/{suite-C4XSg6Ls.js → suite-oH7uiCRq.js} +107 -106
- package/dist/suite-oH7uiCRq.js.map +1 -0
- package/dist/summary-CTMWfnOF.js +75 -0
- package/dist/summary-CTMWfnOF.js.map +1 -0
- package/dist/{sync-bnI6WWaL.js → sync-BNduS38K.js} +6 -6
- package/dist/{sync-bnI6WWaL.js.map → sync-BNduS38K.js.map} +1 -1
- package/dist/{tart-CSP5Q0qV.js → tart-B6Uz4S3I.js} +43 -13
- package/dist/tart-B6Uz4S3I.js.map +1 -0
- package/dist/{tart-DmVuVsmX.js → tart-DNrumWZu.js} +1 -1
- package/dist/{tart-CzViW3WZ.js → tart-Duqp33Zh.js} +2 -2
- package/dist/{tart-CzViW3WZ.js.map → tart-Duqp33Zh.js.map} +1 -1
- package/dist/{tart-bake-d9UvIfe3.js → tart-bake-B4QyNb7f.js} +1 -1
- package/dist/{tart-bake-BCKRfyKm.js → tart-bake-s99wBzxO.js} +9 -9
- package/dist/{tart-bake-BCKRfyKm.js.map → tart-bake-s99wBzxO.js.map} +1 -1
- package/dist/{tart-cli-CqSpUYGV.js → tart-cli-B0xVYBti.js} +6 -4
- package/dist/tart-cli-B0xVYBti.js.map +1 -0
- package/dist/{tart-cli-C4d5FXG1.js → tart-cli-DfteJCN9.js} +1 -1
- package/dist/{tart-hosts-Du7SZM4w.js → tart-hosts-B3Ze8aMp.js} +1 -1
- package/dist/{tart-hosts-G0EUDt07.js → tart-hosts-DfVUcZqA.js} +6 -6
- package/dist/{tart-hosts-G0EUDt07.js.map → tart-hosts-DfVUcZqA.js.map} +1 -1
- package/dist/{tart-images-De0-ib7R.js → tart-images-yzIyb0bQ.js} +8 -3
- package/dist/tart-images-yzIyb0bQ.js.map +1 -0
- package/dist/telemetry-bBhK3DSs.js +399 -0
- package/dist/telemetry-bBhK3DSs.js.map +1 -0
- package/dist/testing.d.ts +11 -2
- package/dist/testing.js +4 -4
- package/dist/{tool-class-B3ZFTSOw.js → tool-class-CB-m4PiF.js} +2 -2
- package/dist/{tool-class-B3ZFTSOw.js.map → tool-class-CB-m4PiF.js.map} +1 -1
- package/dist/{trace-CfppUzD9.js → trace-CJQLlAbB.js} +2 -2
- package/dist/{trace-CfppUzD9.js.map → trace-CJQLlAbB.js.map} +1 -1
- package/dist/{traits-Bj1PM5F7.js → traits-DPIffsN8.js} +2 -2
- package/dist/{traits-Bj1PM5F7.js.map → traits-DPIffsN8.js.map} +1 -1
- package/dist/{transcript-CW02A4s9.js → transcript-TWiaAuoL.js} +2 -2
- package/dist/{transcript-CW02A4s9.js.map → transcript-TWiaAuoL.js.map} +1 -1
- package/dist/{resources-DqdwihhT.js → turn-error-Dmv_Muj7.js} +15 -2
- package/dist/turn-error-Dmv_Muj7.js.map +1 -0
- package/dist/{types-D4WEnzhY.d.ts → types-BZaItiui.d.ts} +159 -13
- package/dist/types-Buut5yGe.js +29 -0
- package/dist/{types-CbcnJxCy.js.map → types-Buut5yGe.js.map} +1 -1
- package/dist/types-C48zuZbT.d.ts +156 -0
- package/dist/types-CYz4dqAd.d.ts +1366 -0
- package/dist/{ui-DfvPNmPP.js → ui-BIYUGm8D.js} +21 -17
- package/dist/ui-BIYUGm8D.js.map +1 -0
- package/dist/{vertex-BLxYasfw.js → vertex-DQQGhUn7.js} +2 -2
- package/dist/{vertex-BLxYasfw.js.map → vertex-DQQGhUn7.js.map} +1 -1
- package/dist/vertex-DtWe0Xf6.js +1 -0
- package/dist/view/index.d.ts +142 -809
- package/dist/view/index.js +17 -12
- package/dist/view/report/index.d.ts +217 -0
- package/dist/view/report/index.js +4 -0
- package/dist/views-DOYgIWRs.js +2 -0
- package/dist/{views-ChoSQxfF.js → views-DPsEJBBP.js} +6 -6
- package/dist/{views-ChoSQxfF.js.map → views-DPsEJBBP.js.map} +1 -1
- package/dist/{vm-container-DOtIi_rd.js → vm-container-rHUmgBRf.js} +13 -11
- package/dist/vm-container-rHUmgBRf.js.map +1 -0
- package/dist/{vm-images-CHutQVml.js → vm-images-BC2ZjZl5.js} +19 -21
- package/dist/vm-images-BC2ZjZl5.js.map +1 -0
- package/dist/wait-CixM6W0G.js +65 -0
- package/dist/wait-CixM6W0G.js.map +1 -0
- package/dist/{worker-entry-CKBfEt0h.js → worker-entry-CN1eVLzz.js} +7 -7
- package/dist/{worker-entry-CKBfEt0h.js.map → worker-entry-CN1eVLzz.js.map} +1 -1
- package/dist/{workspace-Cip2dVwU.js → workspace-BS-FUp5I.js} +9 -9
- package/dist/{workspace-Cip2dVwU.js.map → workspace-BS-FUp5I.js.map} +1 -1
- package/dist/workspace-D8HqjfsQ.js +1 -0
- package/package.json +38 -1
- package/skills/reeval/references/cli.md +1 -1
- package/skills/reeval/references/config.md +1 -1
- package/skills/reeval/references/environments.md +1 -1
- package/dist/agents-CEapsHV-.js.map +0 -1
- package/dist/analysis-BnG_1x_O.js.map +0 -1
- package/dist/app-files-Dsr3U1Q7.js +0 -10
- package/dist/app-files-Dsr3U1Q7.js.map +0 -1
- package/dist/assemble-DUtbByvj.js +0 -1
- package/dist/assemble-Dw1bF13I.js.map +0 -1
- package/dist/attach-DvYiMu6w.js.map +0 -1
- package/dist/attempt-host-DgeJFSZw.js.map +0 -1
- package/dist/boat-BBPOc_bs.js +0 -1
- package/dist/boat-CUNgSc75.js.map +0 -1
- package/dist/boat-images-BuFigiEb.js.map +0 -1
- package/dist/bundle-yEgI3RsI.js.map +0 -1
- package/dist/capture-DxxB3d7p.js +0 -1
- package/dist/cases-B7QkvMGE.js.map +0 -1
- package/dist/cases-CA7gxxdr.js +0 -1
- package/dist/cli-Z7aw8k25.js.map +0 -1
- package/dist/cli-spec-3zvLC9YW.js.map +0 -1
- package/dist/client-kUwtq-vt.js.map +0 -1
- package/dist/cloud-DuHcNQVV.js.map +0 -1
- package/dist/cloud-run-C08P_Uap.js +0 -1
- package/dist/cloud-run-DqMkSGYB.js.map +0 -1
- package/dist/cloud-secrets-Bak83tik.js.map +0 -1
- package/dist/command-Bs6Knfm4.js +0 -298
- package/dist/command-Bs6Knfm4.js.map +0 -1
- package/dist/command-CeygdoVe.js.map +0 -1
- package/dist/command-CoHN1ibe.js.map +0 -1
- package/dist/command-CtmSJzPC.js.map +0 -1
- package/dist/command-efTGfUYb.js.map +0 -1
- package/dist/commands-B_cAs8nG.js +0 -1
- package/dist/commands-DR-NolV2.js.map +0 -1
- package/dist/config-Bn2OhCN_.js.map +0 -1
- package/dist/context-Ba7PNMMe.js +0 -120
- package/dist/context-Ba7PNMMe.js.map +0 -1
- package/dist/daytona-CWs9j9mt.js.map +0 -1
- package/dist/daytona-image-BjVpi3KS.js +0 -1
- package/dist/db-Bjq6qr6c.js +0 -2
- package/dist/db-D6yQib7M.js.map +0 -1
- package/dist/debug-CF4p4Lp_.js +0 -1
- package/dist/debug-CIXzB4wK.js.map +0 -1
- package/dist/environment-BY8lVFFK.js.map +0 -1
- package/dist/evaluate-B8I9AKgw.js.map +0 -1
- package/dist/events-D97ZzAeq.js.map +0 -1
- package/dist/files-5HsFIpxw.js +0 -1
- package/dist/files-BNg3wgw_.js +0 -1
- package/dist/files-BesexM88.js.map +0 -1
- package/dist/files-DNgQwgA0.js +0 -1
- package/dist/files-DvDUSrWL.js +0 -1
- package/dist/files-TwSVeWwM.js.map +0 -1
- package/dist/gate-CgYdaprG.js +0 -49
- package/dist/gate-CgYdaprG.js.map +0 -1
- package/dist/harness-FcTyyPco.js.map +0 -1
- package/dist/ids-DqEvrBFZ.js.map +0 -1
- package/dist/init-CS5kEaVg.js +0 -1
- package/dist/ink-list-CLq6YVB8.js.map +0 -1
- package/dist/inputs-load-D4IsEjq3.js +0 -1
- package/dist/issue-DmSorsTF.js +0 -1
- package/dist/list-ClDb8qiX.js.map +0 -1
- package/dist/live-C7iGaDCA.js +0 -120
- package/dist/live-C7iGaDCA.js.map +0 -1
- package/dist/live-C7kN5v2P.js +0 -1
- package/dist/live-QFILMmql.js.map +0 -1
- package/dist/machine-service-xFhakTs5.js.map +0 -1
- package/dist/model-Colu5mEp.js +0 -1
- package/dist/model-mTaaEkeQ.js.map +0 -1
- package/dist/options-CmRy82gl.js +0 -1
- package/dist/pairs-BGb134Vu.js +0 -230
- package/dist/pairs-BGb134Vu.js.map +0 -1
- package/dist/patch-C7yBztZj.js.map +0 -1
- package/dist/paths-Dwmi7L4X.js +0 -75
- package/dist/paths-Dwmi7L4X.js.map +0 -1
- package/dist/preflight-xa8_eGOW.js +0 -1
- package/dist/prices-C_mbZa5a.js +0 -1
- package/dist/process-RPlEhpHz.js.map +0 -1
- package/dist/provider-DHT20Gl8.js +0 -1
- package/dist/query-BXV6pRyI.js +0 -1
- package/dist/questions-DnqkJga0.js.map +0 -1
- package/dist/record-2liAelrB.js.map +0 -1
- package/dist/redact-DMueYS-v.js.map +0 -1
- package/dist/remarks-D5qOZzcX.js.map +0 -1
- package/dist/resolve-B60w6DcB.js.map +0 -1
- package/dist/resolve-CCLiJX4i.js +0 -1
- package/dist/resolve-t9sOkGGS.js.map +0 -1
- package/dist/resources-DqdwihhT.js.map +0 -1
- package/dist/resume-4Q9q8AcE.js.map +0 -1
- package/dist/runtime-CIsljVBy.js +0 -457
- package/dist/runtime-CIsljVBy.js.map +0 -1
- package/dist/server-CeYUmuuu.js +0 -1
- package/dist/server-fSNuT3Z4.js +0 -284
- package/dist/server-fSNuT3Z4.js.map +0 -1
- package/dist/services-BG8xuU82.js.map +0 -1
- package/dist/shared-DtZwIAfy.js.map +0 -1
- package/dist/slug-IHh3f770.js.map +0 -1
- package/dist/source-n7nUVm9p.js +0 -1
- package/dist/source-zoRWRMql.js.map +0 -1
- package/dist/src-9V3C6A7C.js +0 -2
- package/dist/src-CB5cZhaN.js +0 -2
- package/dist/src-DB0vml0O.js +0 -1
- package/dist/src-OC89dAjO.js.map +0 -1
- package/dist/ssh-pool-ChpT-fDU.js +0 -1
- package/dist/ssh-pool-DwboLN9O.js.map +0 -1
- package/dist/start-BwMyDneA.js.map +0 -1
- package/dist/state-CXTaT0t8.js.map +0 -1
- package/dist/state-CltRWlCu.js +0 -1
- package/dist/suite-C4XSg6Ls.js.map +0 -1
- package/dist/suite-CIsyPqPA.js +0 -2
- package/dist/summary-DeeQJAR5.js +0 -169
- package/dist/summary-DeeQJAR5.js.map +0 -1
- package/dist/tart-CSP5Q0qV.js.map +0 -1
- package/dist/tart-cli-CqSpUYGV.js.map +0 -1
- package/dist/tart-images-De0-ib7R.js.map +0 -1
- package/dist/types-CbcnJxCy.js +0 -15
- package/dist/ui-DfvPNmPP.js.map +0 -1
- package/dist/vertex-DZDUpp0W.js +0 -1
- package/dist/views-DU0PM0UW.js +0 -2
- package/dist/vm-container-DOtIi_rd.js.map +0 -1
- package/dist/vm-images-CHutQVml.js.map +0 -1
- package/dist/workspace-BKcNJAPb.js +0 -1
|
@@ -0,0 +1,2058 @@
|
|
|
1
|
+
import { r as parseDuration } from "./duration-Bz2H7pXb.js";
|
|
2
|
+
import { o as totalTokens } from "./usage-CwIRPPZ9.js";
|
|
3
|
+
import { r as runCost } from "./cost-DQ0ayy6h.js";
|
|
4
|
+
import { A as runtimeLabel, S as languageLabel, _ as harnessLabel, i as attemptAnswerRows, x as isUngatedMiss } from "./model-xemZQ7DK.js";
|
|
5
|
+
import { I as filesChanged, L as formatCount, N as effortSample, r as caseName, v as contrastsOf, z as formatSeconds } from "./cases-C6vnV-gn.js";
|
|
6
|
+
import { a as liftInterval, r as containsZero } from "./stats-C18W4xPX.js";
|
|
7
|
+
import { d as labeledJudgments } from "./judgments-CVCUWT-g.js";
|
|
8
|
+
import { r as runResourcesLine, t as turnErrorKind } from "./turn-error-Dmv_Muj7.js";
|
|
9
|
+
import { l as messageLine, o as factSegment } from "./suite-oH7uiCRq.js";
|
|
10
|
+
import { A as tokensText, D as rateOf, E as ptsInterval, O as spreadOf, T as ptsDelta, _ as countFigure, b as durationFigure, f as attemptAnchor, g as selectedEvals, h as idOf, i as PASS_RATE_OUTCOME_LABEL, j as usdFigure, k as tokensFigure, l as measurePhrase, m as attemptIds, u as passRateOutcomeFromMatchers, v as countOf, w as pctDelta, y as diffLinesText } from "./labels-Dtyf5-pd.js";
|
|
11
|
+
// src/view/sha256.ts
|
|
12
|
+
const K = new Uint32Array([
|
|
13
|
+
1116352408,
|
|
14
|
+
1899447441,
|
|
15
|
+
3049323471,
|
|
16
|
+
3921009573,
|
|
17
|
+
961987163,
|
|
18
|
+
1508970993,
|
|
19
|
+
2453635748,
|
|
20
|
+
2870763221,
|
|
21
|
+
3624381080,
|
|
22
|
+
310598401,
|
|
23
|
+
607225278,
|
|
24
|
+
1426881987,
|
|
25
|
+
1925078388,
|
|
26
|
+
2162078206,
|
|
27
|
+
2614888103,
|
|
28
|
+
3248222580,
|
|
29
|
+
3835390401,
|
|
30
|
+
4022224774,
|
|
31
|
+
264347078,
|
|
32
|
+
604807628,
|
|
33
|
+
770255983,
|
|
34
|
+
1249150122,
|
|
35
|
+
1555081692,
|
|
36
|
+
1996064986,
|
|
37
|
+
2554220882,
|
|
38
|
+
2821834349,
|
|
39
|
+
2952996808,
|
|
40
|
+
3210313671,
|
|
41
|
+
3336571891,
|
|
42
|
+
3584528711,
|
|
43
|
+
113926993,
|
|
44
|
+
338241895,
|
|
45
|
+
666307205,
|
|
46
|
+
773529912,
|
|
47
|
+
1294757372,
|
|
48
|
+
1396182291,
|
|
49
|
+
1695183700,
|
|
50
|
+
1986661051,
|
|
51
|
+
2177026350,
|
|
52
|
+
2456956037,
|
|
53
|
+
2730485921,
|
|
54
|
+
2820302411,
|
|
55
|
+
3259730800,
|
|
56
|
+
3345764771,
|
|
57
|
+
3516065817,
|
|
58
|
+
3600352804,
|
|
59
|
+
4094571909,
|
|
60
|
+
275423344,
|
|
61
|
+
430227734,
|
|
62
|
+
506948616,
|
|
63
|
+
659060556,
|
|
64
|
+
883997877,
|
|
65
|
+
958139571,
|
|
66
|
+
1322822218,
|
|
67
|
+
1537002063,
|
|
68
|
+
1747873779,
|
|
69
|
+
1955562222,
|
|
70
|
+
2024104815,
|
|
71
|
+
2227730452,
|
|
72
|
+
2361852424,
|
|
73
|
+
2428436474,
|
|
74
|
+
2756734187,
|
|
75
|
+
3204031479,
|
|
76
|
+
3329325298
|
|
77
|
+
]);
|
|
78
|
+
const rotr = (x, n) => x >>> n | x << 32 - n;
|
|
79
|
+
/** The lowercase hex SHA-256 of `input`; a string is hashed as UTF-8. */
|
|
80
|
+
function sha256Hex(input) {
|
|
81
|
+
const data = typeof input === "string" ? new TextEncoder().encode(input) : input;
|
|
82
|
+
const bytes = new Uint8Array(Math.ceil((data.length + 9) / 64) * 64);
|
|
83
|
+
bytes.set(data);
|
|
84
|
+
bytes[data.length] = 128;
|
|
85
|
+
const view = new DataView(bytes.buffer);
|
|
86
|
+
const bits = data.length * 8;
|
|
87
|
+
view.setUint32(bytes.length - 8, Math.floor(bits / 2 ** 32));
|
|
88
|
+
view.setUint32(bytes.length - 4, bits >>> 0);
|
|
89
|
+
const h = [
|
|
90
|
+
1779033703,
|
|
91
|
+
3144134277,
|
|
92
|
+
1013904242,
|
|
93
|
+
2773480762,
|
|
94
|
+
1359893119,
|
|
95
|
+
2600822924,
|
|
96
|
+
528734635,
|
|
97
|
+
1541459225
|
|
98
|
+
];
|
|
99
|
+
const w = /* @__PURE__ */ new Uint32Array(64);
|
|
100
|
+
const word = (i) => w[i] ?? 0;
|
|
101
|
+
for (let off = 0; off < bytes.length; off += 64) {
|
|
102
|
+
for (let i = 0; i < 16; i++) w[i] = view.getUint32(off + i * 4);
|
|
103
|
+
for (let i = 16; i < 64; i++) {
|
|
104
|
+
const x = word(i - 15);
|
|
105
|
+
const y = word(i - 2);
|
|
106
|
+
const s0 = rotr(x, 7) ^ rotr(x, 18) ^ x >>> 3;
|
|
107
|
+
const s1 = rotr(y, 17) ^ rotr(y, 19) ^ y >>> 10;
|
|
108
|
+
w[i] = word(i - 16) + s0 + word(i - 7) + s1;
|
|
109
|
+
}
|
|
110
|
+
let [a = 0, b = 0, c = 0, d = 0, e = 0, f = 0, g = 0, k = 0] = h;
|
|
111
|
+
for (let i = 0; i < 64; i++) {
|
|
112
|
+
const ch = e & f ^ ~e & g;
|
|
113
|
+
const t1 = k + (rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25)) + ch + (K[i] ?? 0) + word(i) >>> 0;
|
|
114
|
+
const t2 = (rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22)) + (a & b ^ a & c ^ b & c) >>> 0;
|
|
115
|
+
k = g;
|
|
116
|
+
g = f;
|
|
117
|
+
f = e;
|
|
118
|
+
e = d + t1 >>> 0;
|
|
119
|
+
d = c;
|
|
120
|
+
c = b;
|
|
121
|
+
b = a;
|
|
122
|
+
a = t1 + t2 >>> 0;
|
|
123
|
+
}
|
|
124
|
+
const next = [
|
|
125
|
+
a,
|
|
126
|
+
b,
|
|
127
|
+
c,
|
|
128
|
+
d,
|
|
129
|
+
e,
|
|
130
|
+
f,
|
|
131
|
+
g,
|
|
132
|
+
k
|
|
133
|
+
];
|
|
134
|
+
for (let i = 0; i < 8; i++) h[i] = (h[i] ?? 0) + (next[i] ?? 0) >>> 0;
|
|
135
|
+
}
|
|
136
|
+
let hex = "";
|
|
137
|
+
for (const v of h) hex += v.toString(16).padStart(8, "0");
|
|
138
|
+
return hex;
|
|
139
|
+
}
|
|
140
|
+
// src/view/scope.ts
|
|
141
|
+
/** JSON with every object's keys sorted and `undefined` members left out, so equal data hashes equal. */
|
|
142
|
+
function canonical(value) {
|
|
143
|
+
if (Array.isArray(value)) return `[${value.map((v) => canonical(v ?? null)).join(",")}]`;
|
|
144
|
+
if (value && typeof value === "object") return `{${Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`;
|
|
145
|
+
return JSON.stringify(value);
|
|
146
|
+
}
|
|
147
|
+
/** A duration as milliseconds, so `'10m'` and `600000` hash equal; a value it can't read, as written. */
|
|
148
|
+
function durationMs(value) {
|
|
149
|
+
if (value === void 0) return void 0;
|
|
150
|
+
try {
|
|
151
|
+
return parseDuration(value);
|
|
152
|
+
} catch {
|
|
153
|
+
return value;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** In the hashed input; bumped by any change to what `configKey` hashes or how. */
|
|
157
|
+
const CONFIG_KEY_VERSION = 1;
|
|
158
|
+
const sorted = (list) => list ? [...list].sort() : void 0;
|
|
159
|
+
function profileKey(profile) {
|
|
160
|
+
const { defaults, limit } = profile;
|
|
161
|
+
return {
|
|
162
|
+
name: profile.name,
|
|
163
|
+
defaults: {
|
|
164
|
+
...defaults,
|
|
165
|
+
harness: sorted(defaults.harness),
|
|
166
|
+
env: sorted(defaults.env),
|
|
167
|
+
timeout: durationMs(defaults.timeout)
|
|
168
|
+
},
|
|
169
|
+
limit: {
|
|
170
|
+
...limit,
|
|
171
|
+
harness: sorted(limit.harness),
|
|
172
|
+
traits: limit.traits ? Object.fromEntries(Object.entries(limit.traits).map(([k, v]) => [k, typeof v === "string" ? v : sorted(v)])) : void 0
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
/** The run's scope, from its record alone; a record without `invocation` reads as one run with none. */
|
|
177
|
+
function runScope(record) {
|
|
178
|
+
const invocation = record.invocation;
|
|
179
|
+
const filters = invocation?.filters ?? [];
|
|
180
|
+
const tags = invocation?.tags ?? [];
|
|
181
|
+
const traits = invocation?.traits ?? [];
|
|
182
|
+
const { config } = record;
|
|
183
|
+
const key = {
|
|
184
|
+
v: CONFIG_KEY_VERSION,
|
|
185
|
+
profile: record.profile ? profileKey(record.profile) : null,
|
|
186
|
+
invocation: {
|
|
187
|
+
runs: invocation?.runs,
|
|
188
|
+
filters: sorted(filters),
|
|
189
|
+
tags: sorted(tags),
|
|
190
|
+
traits: sorted(traits)
|
|
191
|
+
},
|
|
192
|
+
config: {
|
|
193
|
+
environment: config.environment,
|
|
194
|
+
runs: config.runs,
|
|
195
|
+
bestOf: config.bestOf,
|
|
196
|
+
judgeModel: config.judgeModel,
|
|
197
|
+
threshold: config.threshold,
|
|
198
|
+
timeoutMs: config.timeoutMs
|
|
199
|
+
},
|
|
200
|
+
fingerprintScope: sorted(record.stack.fingerprintScope)
|
|
201
|
+
};
|
|
202
|
+
return {
|
|
203
|
+
profile: record.profile?.name ?? null,
|
|
204
|
+
subset: filters.length > 0 || tags.length > 0 || traits.length > 0 || invocation?.runs !== void 0,
|
|
205
|
+
configKey: sha256Hex(canonical(key))
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
// src/view/report/links.ts
|
|
209
|
+
/** Prefix every relative artifact link: `linkBase` as a folder, `./` by default. */
|
|
210
|
+
function withBase(linkBase, path) {
|
|
211
|
+
const base = !linkBase ? linkBase ?? "./" : linkBase.endsWith("/") ? linkBase : `${linkBase}/`;
|
|
212
|
+
if (path.startsWith("#")) return path;
|
|
213
|
+
if (path.startsWith("../")) return path;
|
|
214
|
+
return `${base}${path.replace(/^\.\//, "")}`;
|
|
215
|
+
}
|
|
216
|
+
/** The record's link: `href`, or `../<runId>.json`, the record beside the run's folder. */
|
|
217
|
+
function recordHref(runId, href) {
|
|
218
|
+
return href ?? `../${runId}.json`;
|
|
219
|
+
}
|
|
220
|
+
function kept(ref) {
|
|
221
|
+
return ref?.path ?? null;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Citation target for an attempt: transcript, else diff, else first capture, else its anchor in
|
|
225
|
+
* the attempt index (`attemptAnchor`).
|
|
226
|
+
* `FileRef.path` values are already relative to the run directory.
|
|
227
|
+
* When `used` is set, a path already claimed by another attempt falls back to the anchor
|
|
228
|
+
* so baseline and variant cites stay distinct.
|
|
229
|
+
*/
|
|
230
|
+
function attemptHref(_evalEntry, attempt, linkBase, used) {
|
|
231
|
+
const files = attempt.files;
|
|
232
|
+
const transcript = kept(files.transcript);
|
|
233
|
+
const preferred = transcript ? withBase(linkBase, transcript) : (() => {
|
|
234
|
+
const diff = kept(files.diff);
|
|
235
|
+
if (diff) return withBase(linkBase, diff);
|
|
236
|
+
const capture = files.artifacts?.[0];
|
|
237
|
+
if (capture) return withBase(linkBase, capture.path);
|
|
238
|
+
return attemptAnchor(attempt.key);
|
|
239
|
+
})();
|
|
240
|
+
if (!used) return preferred;
|
|
241
|
+
if (!used.has(preferred) || preferred.startsWith("#")) {
|
|
242
|
+
used.add(preferred);
|
|
243
|
+
return preferred;
|
|
244
|
+
}
|
|
245
|
+
const fallback = attemptAnchor(attempt.key);
|
|
246
|
+
used.add(fallback);
|
|
247
|
+
return fallback;
|
|
248
|
+
}
|
|
249
|
+
function attemptRef(ids, evalEntry, attempt, linkBase, used, hrefByKey) {
|
|
250
|
+
const id = idOf(ids, attempt.key);
|
|
251
|
+
if (!id) return null;
|
|
252
|
+
const assigned = hrefByKey?.get(attempt.key);
|
|
253
|
+
if (assigned) return {
|
|
254
|
+
id,
|
|
255
|
+
key: attempt.key,
|
|
256
|
+
href: assigned
|
|
257
|
+
};
|
|
258
|
+
return {
|
|
259
|
+
id,
|
|
260
|
+
key: attempt.key,
|
|
261
|
+
href: attemptHref(evalEntry, attempt, linkBase, used)
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
/** Capture PNG link, or null when not kept (caller prints `name (not kept)`). */
|
|
265
|
+
function captureHref(path, linkBase) {
|
|
266
|
+
if (!path) return null;
|
|
267
|
+
return withBase(linkBase, path);
|
|
268
|
+
}
|
|
269
|
+
/** Question input JSON link; omitted when not kept. Paths are run-relative. */
|
|
270
|
+
function questionHref(path, linkBase) {
|
|
271
|
+
if (!path) return null;
|
|
272
|
+
return withBase(linkBase, path);
|
|
273
|
+
}
|
|
274
|
+
/** Worktree patch link, or the "not saved" note when absent. */
|
|
275
|
+
function worktreePatchLink(patch, note, linkBase) {
|
|
276
|
+
if (patch) return {
|
|
277
|
+
href: withBase(linkBase, patch.path),
|
|
278
|
+
text: "worktree.patch"
|
|
279
|
+
};
|
|
280
|
+
return {
|
|
281
|
+
href: null,
|
|
282
|
+
text: `not saved: ${note ?? "unknown"}`
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
/** GFM heading anchor: lowercase, punctuation stripped, spaces to hyphens. */
|
|
286
|
+
function gfmAnchor(heading) {
|
|
287
|
+
return heading.toLowerCase().replace(/[^\p{L}\p{N}\s-]/gu, "").trim().replace(/\s+/g, "-");
|
|
288
|
+
}
|
|
289
|
+
function evalAnchorHref(anchor) {
|
|
290
|
+
return `#${anchor}`;
|
|
291
|
+
}
|
|
292
|
+
// src/view/report/policy.ts
|
|
293
|
+
/**
|
|
294
|
+
* Report layout and delta defaults. Flip one field to change every surface that
|
|
295
|
+
* renders a `RunReport`; the shape of `RunReport` itself does not change.
|
|
296
|
+
*/
|
|
297
|
+
const REPORT_POLICY = {
|
|
298
|
+
/**
|
|
299
|
+
* Measure vs baseline: raw median, then a signed integer `%` delta in
|
|
300
|
+
* parentheses. Absolute value below 2 is `±0%`. Rates take `pts` instead.
|
|
301
|
+
*/
|
|
302
|
+
deltas: "pct",
|
|
303
|
+
/**
|
|
304
|
+
* When baseline and the best variant both pass every applicable assertion,
|
|
305
|
+
* the headline is `tied-ceiling` on the largest decisive measure delta
|
|
306
|
+
* (absolute value at least 10%), not a `±0 pts` lift.
|
|
307
|
+
*/
|
|
308
|
+
tiedCeiling: true,
|
|
309
|
+
/**
|
|
310
|
+
* Two or more variants: one N-arm summary table, then a harness-by-arm effort
|
|
311
|
+
* pivot, then pairwise contrasts.
|
|
312
|
+
*/
|
|
313
|
+
variantLayout: "n-arm"
|
|
314
|
+
};
|
|
315
|
+
// src/view/report/provenance.ts
|
|
316
|
+
const short = (hash) => hash.replace(/^sha256:/, "").slice(0, 12);
|
|
317
|
+
/**
|
|
318
|
+
* The state a run executed in, as label/value pairs. Pure and browser-safe.
|
|
319
|
+
* The `Context` row is split out into `ProvenanceState.context` so Markdown can
|
|
320
|
+
* put long file lists under `<details>`.
|
|
321
|
+
*/
|
|
322
|
+
function stateRows(run) {
|
|
323
|
+
const { git, stack: env } = run;
|
|
324
|
+
const rows = [];
|
|
325
|
+
rows.push({
|
|
326
|
+
field: "Executed at",
|
|
327
|
+
value: git ? `${git.sha.slice(0, 12)}${git.branch ? ` on ${git.branch}` : ""}${git.dirty ? ", dirty" : ", clean"} (the commit that adds this record is later)` : "not a git repository"
|
|
328
|
+
});
|
|
329
|
+
if (git?.dirty) {
|
|
330
|
+
const patch = worktreePatchLink(git.worktreePatch, git.worktreePatchNote);
|
|
331
|
+
rows.push({
|
|
332
|
+
field: "Uncommitted",
|
|
333
|
+
value: git.worktreePatch ? `worktree.patch, ${git.worktreePatch.bytes} bytes, sha256 ${short(git.worktreePatch.sha256)}` : `${patch.text}${git.diffHash ? `; sha256 ${short(git.diffHash)}` : ""}`
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
rows.push({
|
|
337
|
+
field: "Lockfiles",
|
|
338
|
+
value: run.dependencies.lockfiles.map((l) => `${l.path} ${short(l.sha256)}`).join(", ") || "none found"
|
|
339
|
+
});
|
|
340
|
+
rows.push({
|
|
341
|
+
field: "Images",
|
|
342
|
+
value: env.images.length ? env.images.map((i) => `${i.ref} · id ${short(i.id)} · ${i.repoDigests[0] ?? "no registry digest (built locally)"} · CLI ${i.agentCli ?? "n/a"}${i.dockerfile ? ` · dockerfile ${i.dockerfile.path} ${short(i.dockerfile.sha256)}` : ""}${i.install?.length ? ` · install ${i.install.length} on ${i.base}` : ""}`).join("; ") : "none (fake sandbox)"
|
|
343
|
+
});
|
|
344
|
+
rows.push({
|
|
345
|
+
field: "Harnesses",
|
|
346
|
+
value: env.harnesses.map(harnessLabel).join(", ") || "none"
|
|
347
|
+
});
|
|
348
|
+
const resources = runResourcesLine(run.evals.flatMap((e) => e.attempts.map((a) => a.resources)));
|
|
349
|
+
if (resources) rows.push({
|
|
350
|
+
field: "Resources",
|
|
351
|
+
value: resources
|
|
352
|
+
});
|
|
353
|
+
rows.push({
|
|
354
|
+
field: "Environments",
|
|
355
|
+
value: run.environments.map((e) => `${e.name} ${short(e.digest)}${e.skills.length ? ` · skills ${e.skills.map((s) => `${s.name}${s.commit ? `@${s.commit.slice(0, 7)}` : ""}`).join(", ")}` : ""}${e.mcp.length ? ` · mcp ${e.mcp.join(", ")}` : ""}${e.secrets.length ? ` · secrets ${e.secrets.join(", ")}` : ""}`).join("; ") || "none"
|
|
356
|
+
});
|
|
357
|
+
rows.push({
|
|
358
|
+
field: "Judge",
|
|
359
|
+
value: `${env.judge.model ?? "none by default"}${env.judge.resolvedModels.length ? ` (served ${env.judge.resolvedModels.join(", ")})` : ""} · threshold ${env.judge.threshold} · version ${env.judge.version}`
|
|
360
|
+
});
|
|
361
|
+
rows.push({
|
|
362
|
+
field: "Packages",
|
|
363
|
+
value: Object.entries(env.packages).map(([name, v]) => `${name} ${v}`).join(" · ")
|
|
364
|
+
});
|
|
365
|
+
const language = languageLabel(env);
|
|
366
|
+
rows.push({
|
|
367
|
+
field: "reeval",
|
|
368
|
+
value: `${env.reeval} · ${runtimeLabel(env)}${language ? ` · ${language}` : ""} · ${env.os}/${env.arch}`
|
|
369
|
+
});
|
|
370
|
+
rows.push({
|
|
371
|
+
field: "Runner",
|
|
372
|
+
value: run.runner.kind === "ci" ? `CI (${run.runner.provider})${run.runner.runUrl ? ` ${run.runner.runUrl}` : ""}` : `local · ${run.author}`
|
|
373
|
+
});
|
|
374
|
+
return rows;
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* One-line warnings for states that stop an exact rerun. Derived when shown,
|
|
378
|
+
* never stored. Pure and browser-safe.
|
|
379
|
+
*/
|
|
380
|
+
function warnings(run) {
|
|
381
|
+
const out = [];
|
|
382
|
+
for (const w of run.plan.sandboxWarnings) out.push(w.message);
|
|
383
|
+
for (const image of run.stack.images.filter((i) => i.repoDigests.length === 0)) out.push(`Image ${image.ref} isn't pushed, so an exact rerun needs this machine's image (id ${image.id.replace(/^sha256:/, "").slice(0, 12)}).`);
|
|
384
|
+
if (!run.git) out.push("Not a git repository, so the code under test isn't pinned.");
|
|
385
|
+
else if (run.git.dirty && !run.git.worktreePatch) out.push(`Uncommitted changes weren't saved (${run.git.worktreePatchNote ?? "no patch"}), so this exact state can't be recreated.`);
|
|
386
|
+
return out;
|
|
387
|
+
}
|
|
388
|
+
/** Provenance block for a run report. */
|
|
389
|
+
function provenanceState(run, recordLink) {
|
|
390
|
+
const contextFiles = run.stack.context.map((c) => ({
|
|
391
|
+
path: c.path,
|
|
392
|
+
sha256: c.sha256
|
|
393
|
+
}));
|
|
394
|
+
return {
|
|
395
|
+
rows: stateRows(run),
|
|
396
|
+
context: {
|
|
397
|
+
count: contextFiles.length,
|
|
398
|
+
files: contextFiles
|
|
399
|
+
},
|
|
400
|
+
artifacts: run.artifacts,
|
|
401
|
+
record: recordHref(run.runId, recordLink),
|
|
402
|
+
warnings: warnings(run)
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
// src/view/report/build.ts
|
|
406
|
+
const MEASURE_KEYS = [
|
|
407
|
+
"tokens",
|
|
408
|
+
"toolCalls",
|
|
409
|
+
"turns",
|
|
410
|
+
"costUsd",
|
|
411
|
+
"agentMs",
|
|
412
|
+
"filesChanged",
|
|
413
|
+
"linesAdded",
|
|
414
|
+
"linesRemoved"
|
|
415
|
+
];
|
|
416
|
+
const LOCKFILE_NAMES = /* @__PURE__ */ new Set([
|
|
417
|
+
"pnpm-lock.yaml",
|
|
418
|
+
"package-lock.json",
|
|
419
|
+
"npm-shrinkwrap.json",
|
|
420
|
+
"yarn.lock",
|
|
421
|
+
"bun.lock",
|
|
422
|
+
"bun.lockb",
|
|
423
|
+
"pubspec.lock",
|
|
424
|
+
"Cargo.lock",
|
|
425
|
+
"go.sum",
|
|
426
|
+
"poetry.lock",
|
|
427
|
+
"uv.lock",
|
|
428
|
+
"Pipfile.lock",
|
|
429
|
+
"Gemfile.lock",
|
|
430
|
+
"composer.lock",
|
|
431
|
+
"Podfile.lock",
|
|
432
|
+
"gradle.lockfile"
|
|
433
|
+
]);
|
|
434
|
+
const isLockfilePath = (path) => LOCKFILE_NAMES.has(path.split("/").pop() ?? path);
|
|
435
|
+
const countedOutcome = (o) => o === "pass" || o === "fail";
|
|
436
|
+
const cellOf = (e, key) => e.cells.find((c) => c.key === key);
|
|
437
|
+
function assignAttemptHrefs(ctx) {
|
|
438
|
+
for (const e of ctx.evals) for (const a of e.attempts) {
|
|
439
|
+
if (a.seq === null) continue;
|
|
440
|
+
if (!ctx.ids.get(a.key)) continue;
|
|
441
|
+
const href = attemptHref(e, a, ctx.linkBase, ctx.hrefs);
|
|
442
|
+
ctx.hrefByKey.set(a.key, href);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
const repoBasename = (remote) => {
|
|
446
|
+
if (!remote) return null;
|
|
447
|
+
const cleaned = remote.replace(/\.git$/, "");
|
|
448
|
+
const slash = cleaned.lastIndexOf("/");
|
|
449
|
+
const colon = cleaned.lastIndexOf(":");
|
|
450
|
+
const at = Math.max(slash, colon);
|
|
451
|
+
return at === -1 ? cleaned : cleaned.slice(at + 1) || null;
|
|
452
|
+
};
|
|
453
|
+
function emptyMeasures() {
|
|
454
|
+
return {
|
|
455
|
+
tokens: null,
|
|
456
|
+
toolCalls: null,
|
|
457
|
+
turns: null,
|
|
458
|
+
costUsd: null,
|
|
459
|
+
agentMs: null,
|
|
460
|
+
filesChanged: null,
|
|
461
|
+
linesAdded: null,
|
|
462
|
+
linesRemoved: null
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
function emptyMeasureVsBaseline() {
|
|
466
|
+
const blank = () => ({
|
|
467
|
+
value: null,
|
|
468
|
+
delta: null,
|
|
469
|
+
lower: null
|
|
470
|
+
});
|
|
471
|
+
return {
|
|
472
|
+
tokens: blank(),
|
|
473
|
+
toolCalls: blank(),
|
|
474
|
+
turns: blank(),
|
|
475
|
+
costUsd: blank(),
|
|
476
|
+
agentMs: blank(),
|
|
477
|
+
filesChanged: blank(),
|
|
478
|
+
linesAdded: blank(),
|
|
479
|
+
linesRemoved: blank()
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
/** Diff line totals for an attempt; lockfiles excluded. Null counts when not recorded. */
|
|
483
|
+
function attemptDiffStats(a) {
|
|
484
|
+
const files = filesChanged(a.changedFiles);
|
|
485
|
+
if (a.changedFiles.filter((f) => {
|
|
486
|
+
const row = f;
|
|
487
|
+
return !isLockfilePath(f.path) && !row.lockfile && (row.added !== void 0 || row.removed !== void 0);
|
|
488
|
+
}).length === 0) return {
|
|
489
|
+
files,
|
|
490
|
+
added: null,
|
|
491
|
+
removed: null
|
|
492
|
+
};
|
|
493
|
+
let added = 0;
|
|
494
|
+
let removed = 0;
|
|
495
|
+
for (const f of a.changedFiles) {
|
|
496
|
+
const row = f;
|
|
497
|
+
if (isLockfilePath(f.path) || row.lockfile) continue;
|
|
498
|
+
added += row.added ?? 0;
|
|
499
|
+
removed += row.removed ?? 0;
|
|
500
|
+
}
|
|
501
|
+
return {
|
|
502
|
+
files,
|
|
503
|
+
added,
|
|
504
|
+
removed
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
function measureValues(attempts) {
|
|
508
|
+
const out = {
|
|
509
|
+
tokens: [],
|
|
510
|
+
toolCalls: [],
|
|
511
|
+
turns: [],
|
|
512
|
+
costUsd: [],
|
|
513
|
+
agentMs: [],
|
|
514
|
+
filesChanged: [],
|
|
515
|
+
linesAdded: [],
|
|
516
|
+
linesRemoved: []
|
|
517
|
+
};
|
|
518
|
+
for (const a of attempts) {
|
|
519
|
+
if (!countedOutcome(a.outcome)) continue;
|
|
520
|
+
const sample = effortSample(a);
|
|
521
|
+
if (sample.tokens !== void 0) out.tokens.push(sample.tokens);
|
|
522
|
+
out.toolCalls.push(sample.tools);
|
|
523
|
+
out.turns.push(a.turns.length);
|
|
524
|
+
if (sample.usd !== void 0) out.costUsd.push(sample.usd);
|
|
525
|
+
out.agentMs.push(sample.agentMs);
|
|
526
|
+
out.filesChanged.push(sample.filesChanged);
|
|
527
|
+
const diff = attemptDiffStats(a);
|
|
528
|
+
if (diff.added !== null) out.linesAdded.push(diff.added);
|
|
529
|
+
if (diff.removed !== null) out.linesRemoved.push(diff.removed);
|
|
530
|
+
}
|
|
531
|
+
return out;
|
|
532
|
+
}
|
|
533
|
+
function measuresOf(attempts) {
|
|
534
|
+
const values = measureValues(attempts);
|
|
535
|
+
const fmt = {
|
|
536
|
+
tokens: tokensText,
|
|
537
|
+
toolCalls: formatCount,
|
|
538
|
+
turns: formatCount,
|
|
539
|
+
costUsd: (n) => usdFigure(n).text,
|
|
540
|
+
agentMs: formatSeconds,
|
|
541
|
+
filesChanged: formatCount,
|
|
542
|
+
linesAdded: formatCount,
|
|
543
|
+
linesRemoved: formatCount
|
|
544
|
+
};
|
|
545
|
+
const cells = emptyMeasures();
|
|
546
|
+
for (const key of MEASURE_KEYS) cells[key] = spreadOf(values[key], fmt[key]);
|
|
547
|
+
return cells;
|
|
548
|
+
}
|
|
549
|
+
function measureVs(value, baseline, lower) {
|
|
550
|
+
if (!value) return {
|
|
551
|
+
value: null,
|
|
552
|
+
delta: null,
|
|
553
|
+
lower: null
|
|
554
|
+
};
|
|
555
|
+
return {
|
|
556
|
+
value,
|
|
557
|
+
delta: baseline && baseline.median.value !== null && value.median.value !== null ? pctDelta(value.median.value, baseline.median.value) : null,
|
|
558
|
+
lower: lower ? countOf(lower.k, lower.n) : null
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
function deriveTally(evals) {
|
|
562
|
+
let passed = 0;
|
|
563
|
+
let failed = 0;
|
|
564
|
+
let ungatedPassed = 0;
|
|
565
|
+
let ungatedFailed = 0;
|
|
566
|
+
let infraError = 0;
|
|
567
|
+
let reclassified = 0;
|
|
568
|
+
let skipped = 0;
|
|
569
|
+
for (const e of evals) for (const a of e.attempts) {
|
|
570
|
+
const ungated = cellOf(e, a.cell)?.gate === "none";
|
|
571
|
+
if (a.outcome === "skipped") {
|
|
572
|
+
skipped++;
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
if (a.outcome === "infra_error") {
|
|
576
|
+
infraError++;
|
|
577
|
+
if (wasReclassified(a)) reclassified++;
|
|
578
|
+
continue;
|
|
579
|
+
}
|
|
580
|
+
if (a.outcome === "pass") {
|
|
581
|
+
if (ungated) ungatedPassed++;
|
|
582
|
+
else passed++;
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
if (a.outcome === "fail") {
|
|
586
|
+
if (ungated || isUngatedMiss(e, a)) ungatedFailed++;
|
|
587
|
+
else failed++;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
return {
|
|
591
|
+
counted: passed + failed,
|
|
592
|
+
passed,
|
|
593
|
+
failed,
|
|
594
|
+
ungated: {
|
|
595
|
+
passed: ungatedPassed,
|
|
596
|
+
failed: ungatedFailed
|
|
597
|
+
},
|
|
598
|
+
infraError,
|
|
599
|
+
reclassified,
|
|
600
|
+
skipped
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
/** Older records: a turn error on a pass/fail attempt is treated as reclassified. */
|
|
604
|
+
function wasReclassified(a) {
|
|
605
|
+
if (a.turnStatusRead === true) return false;
|
|
606
|
+
if (a.error && "turn" in a.error && a.error.turn) return true;
|
|
607
|
+
return a.turns.some((t) => t.status === "error") && countedOutcome(a.outcome);
|
|
608
|
+
}
|
|
609
|
+
function turnErrorMessage(a) {
|
|
610
|
+
const turn = a.turns.find((t) => t.status === "error" && t.error);
|
|
611
|
+
if (!turn?.error) return null;
|
|
612
|
+
return `${turnErrorKind(turn.error)}: ${messageLine(turn.error)}`;
|
|
613
|
+
}
|
|
614
|
+
function runTitle(record, evals, title) {
|
|
615
|
+
if (title) return title;
|
|
616
|
+
if (evals.length === 1) return evals[0].name;
|
|
617
|
+
const groups = evals.map((e) => e.group[0]).filter(Boolean);
|
|
618
|
+
if (groups.length === evals.length && new Set(groups).size === 1) return groups[0];
|
|
619
|
+
return `Run ${record.runId}`;
|
|
620
|
+
}
|
|
621
|
+
function runsPerCell(evals) {
|
|
622
|
+
const runs = new Set(evals.flatMap((e) => e.cells.map((c) => c.runs)));
|
|
623
|
+
return runs.size === 1 ? [...runs][0] ?? null : null;
|
|
624
|
+
}
|
|
625
|
+
function buildHeader(ctx) {
|
|
626
|
+
const { record, evals } = ctx;
|
|
627
|
+
const tally = deriveTally(evals);
|
|
628
|
+
const usageAttempts = evals.flatMap((e) => e.attempts).filter((a) => a.usage);
|
|
629
|
+
let total = null;
|
|
630
|
+
let cacheRead = null;
|
|
631
|
+
if (usageAttempts.length) {
|
|
632
|
+
total = 0;
|
|
633
|
+
cacheRead = 0;
|
|
634
|
+
for (const a of usageAttempts) {
|
|
635
|
+
if (!a.usage) continue;
|
|
636
|
+
total += totalTokens(a.usage);
|
|
637
|
+
cacheRead += a.usage.cacheRead;
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
const cost = runCost(record.cost);
|
|
641
|
+
const agent = record.cost?.agentUsd;
|
|
642
|
+
const judge = record.cost?.judgeUsd;
|
|
643
|
+
let basis = null;
|
|
644
|
+
if (agent || judge) {
|
|
645
|
+
const bases = [agent?.basis, judge?.basis].filter(Boolean);
|
|
646
|
+
basis = bases.every((b) => b === "reported") ? "reported" : bases.every((b) => b === "estimated") ? "estimated" : "mixed";
|
|
647
|
+
}
|
|
648
|
+
return {
|
|
649
|
+
title: runTitle(record, evals),
|
|
650
|
+
status: record.status,
|
|
651
|
+
startedAt: record.startedAt,
|
|
652
|
+
author: record.author,
|
|
653
|
+
reeval: record.stack.reeval,
|
|
654
|
+
profile: record.profile?.name ?? null,
|
|
655
|
+
scope: runScope(record),
|
|
656
|
+
runsPerCell: runsPerCell(evals),
|
|
657
|
+
wall: durationFigure(record.durationMs),
|
|
658
|
+
tally,
|
|
659
|
+
tokens: total === null ? null : {
|
|
660
|
+
total: tokensFigure(total),
|
|
661
|
+
cacheRead: tokensFigure(cacheRead)
|
|
662
|
+
},
|
|
663
|
+
cost: basis ? {
|
|
664
|
+
agent: agent ? usdFigure(agent.usd) : null,
|
|
665
|
+
judge: judge ? usdFigure(judge.usd) : null,
|
|
666
|
+
total: agent && judge ? usdFigure(agent.usd + judge.usd) : null,
|
|
667
|
+
basis
|
|
668
|
+
} : cost ? {
|
|
669
|
+
agent: null,
|
|
670
|
+
judge: null,
|
|
671
|
+
total: usdFigure(cost.usd),
|
|
672
|
+
basis: cost.basis === "reported" ? "reported" : "estimated"
|
|
673
|
+
} : null,
|
|
674
|
+
git: record.git ? {
|
|
675
|
+
sha: record.git.sha,
|
|
676
|
+
branch: record.git.branch,
|
|
677
|
+
dirty: record.git.dirty,
|
|
678
|
+
repo: repoBasename(record.git.remote)
|
|
679
|
+
} : null
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
function uniqueEvalNames(evals) {
|
|
683
|
+
const counts = /* @__PURE__ */ new Map();
|
|
684
|
+
for (const e of evals) counts.set(e.name, (counts.get(e.name) ?? 0) + 1);
|
|
685
|
+
const out = /* @__PURE__ */ new Map();
|
|
686
|
+
for (const e of evals) out.set(e.key, (counts.get(e.name) ?? 0) > 1 ? `${e.name} (${e.file})` : e.name);
|
|
687
|
+
return out;
|
|
688
|
+
}
|
|
689
|
+
function shapesOf(e, prior) {
|
|
690
|
+
const shapes = [];
|
|
691
|
+
if (prior) shapes.push("regression");
|
|
692
|
+
if (e.compare) shapes.push("ab");
|
|
693
|
+
if (e.attempts.some((a) => (a.assertions ?? []).some((x) => /toHaveCalledSkill|toHaveFetched|toHaveCalledMcp|toHaveReadSkill/i.test(x.matcher)))) shapes.push("discoverability");
|
|
694
|
+
if (!e.compare && !prior) shapes.push("task");
|
|
695
|
+
return shapes.length ? shapes : ["task"];
|
|
696
|
+
}
|
|
697
|
+
function designOf(e) {
|
|
698
|
+
const harnesses = new Set(e.cells.map((c) => c.harness)).size;
|
|
699
|
+
const arms = new Set(e.cells.map((c) => c.environment)).size;
|
|
700
|
+
const prompts = new Set(e.attempts.flatMap((a) => a.turns[0]?.prompt ? [a.turns[0].prompt] : [])).size;
|
|
701
|
+
const attempts = e.attempts.filter((a) => a.seq !== null).length;
|
|
702
|
+
const parts = [];
|
|
703
|
+
if (prompts > 1) parts.push(`${prompts} prompts`);
|
|
704
|
+
parts.push(`${harnesses} coding harness${harnesses === 1 ? "" : "es"}`);
|
|
705
|
+
if (arms > 1) parts.push(`${arms} arms`);
|
|
706
|
+
parts.push(`${attempts} attempt${attempts === 1 ? "" : "s"}`);
|
|
707
|
+
return parts.join(", ");
|
|
708
|
+
}
|
|
709
|
+
function assertionLabels(e) {
|
|
710
|
+
const seen = /* @__PURE__ */ new Map();
|
|
711
|
+
for (const a of e.attempts) for (const x of a.assertions ?? []) {
|
|
712
|
+
if (seen.has(x.label)) continue;
|
|
713
|
+
seen.set(x.label, {
|
|
714
|
+
label: x.label,
|
|
715
|
+
matcher: x.matcher,
|
|
716
|
+
at: x.at ?? null
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
return [...seen.values()];
|
|
720
|
+
}
|
|
721
|
+
function checkCellOf(attempts, label) {
|
|
722
|
+
let passed = 0;
|
|
723
|
+
let failed = 0;
|
|
724
|
+
let errored = 0;
|
|
725
|
+
let notApplicable = 0;
|
|
726
|
+
let preSatisfied = 0;
|
|
727
|
+
let notRun = 0;
|
|
728
|
+
let reached = 0;
|
|
729
|
+
let reason = null;
|
|
730
|
+
for (const a of attempts) {
|
|
731
|
+
if (!countedOutcome(a.outcome) && a.outcome !== "infra_error") continue;
|
|
732
|
+
const hit = (a.assertions ?? []).find((x) => x.label === label);
|
|
733
|
+
if (!hit) {
|
|
734
|
+
notRun++;
|
|
735
|
+
continue;
|
|
736
|
+
}
|
|
737
|
+
reached++;
|
|
738
|
+
const status = hit.status;
|
|
739
|
+
if (status === "not_applicable") {
|
|
740
|
+
notApplicable++;
|
|
741
|
+
if (!reason && hit.reason) reason = hit.reason;
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
if (hit.preSatisfied) preSatisfied++;
|
|
745
|
+
if (status === "pass") passed++;
|
|
746
|
+
else if (status === "fail") failed++;
|
|
747
|
+
else if (status === "error") errored++;
|
|
748
|
+
}
|
|
749
|
+
const answered = passed + failed;
|
|
750
|
+
if (reached === 0 || answered === 0 && notApplicable > 0 && errored === 0) {
|
|
751
|
+
const na = reason ? `n/a (${reason})` : "n/a";
|
|
752
|
+
return {
|
|
753
|
+
status: reached === 0 ? "not-run" : "na",
|
|
754
|
+
passed: 0,
|
|
755
|
+
answered: 0,
|
|
756
|
+
notApplicable,
|
|
757
|
+
preSatisfied,
|
|
758
|
+
text: reached === 0 ? "n/a" : na,
|
|
759
|
+
reason: reached === 0 ? null : reason
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
if (errored && answered === 0) return {
|
|
763
|
+
status: "error",
|
|
764
|
+
passed,
|
|
765
|
+
answered,
|
|
766
|
+
notApplicable,
|
|
767
|
+
preSatisfied,
|
|
768
|
+
text: `${errored} error${errored === 1 ? "" : "s"}`,
|
|
769
|
+
reason: null
|
|
770
|
+
};
|
|
771
|
+
const status = answered === 0 ? "na" : failed === 0 && errored === 0 ? "pass" : passed === 0 ? "fail" : "mixed";
|
|
772
|
+
const base = answered === 0 ? reason ? `n/a (${reason})` : "n/a" : `${passed} of ${answered}`;
|
|
773
|
+
const text = preSatisfied > 0 ? `${base}, ${preSatisfied} pre-satisfied` : base;
|
|
774
|
+
return {
|
|
775
|
+
status,
|
|
776
|
+
passed,
|
|
777
|
+
answered,
|
|
778
|
+
notApplicable,
|
|
779
|
+
preSatisfied,
|
|
780
|
+
text,
|
|
781
|
+
reason: status === "na" ? reason : null
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
function passRateForAttempts(attempts) {
|
|
785
|
+
const counted = attempts.filter((a) => countedOutcome(a.outcome));
|
|
786
|
+
const passed = counted.filter((a) => a.outcome === "pass").length;
|
|
787
|
+
return rateOf(passed, counted.length);
|
|
788
|
+
}
|
|
789
|
+
function armLabel(cell, digestClash) {
|
|
790
|
+
if (cell.role === "control") return cell.environment;
|
|
791
|
+
const name = cell.variant ?? cell.environment;
|
|
792
|
+
if (!digestClash || !cell.environmentDigest) return name;
|
|
793
|
+
return `${name} (${cell.environmentDigest.replace(/^sha256:/, "").slice(0, 7)})`;
|
|
794
|
+
}
|
|
795
|
+
function armsOf(e) {
|
|
796
|
+
const byEnv = /* @__PURE__ */ new Map();
|
|
797
|
+
for (const c of e.cells) {
|
|
798
|
+
const list = byEnv.get(c.environment) ?? [];
|
|
799
|
+
list.push(c);
|
|
800
|
+
byEnv.set(c.environment, list);
|
|
801
|
+
}
|
|
802
|
+
const digestClash = [...byEnv.values()].some((cells) => {
|
|
803
|
+
return new Set(cells.map((c) => c.environmentDigest ?? c.environment)).size > 1;
|
|
804
|
+
});
|
|
805
|
+
const controls = e.cells.filter((c) => c.role === "control");
|
|
806
|
+
const variants = e.cells.filter((c) => c.role === "variant");
|
|
807
|
+
const order = [];
|
|
808
|
+
const seen = /* @__PURE__ */ new Set();
|
|
809
|
+
for (const c of [...controls, ...variants]) {
|
|
810
|
+
const key = c.environmentDigest ?? c.environment;
|
|
811
|
+
if (seen.has(key)) continue;
|
|
812
|
+
seen.add(key);
|
|
813
|
+
order.push(c);
|
|
814
|
+
}
|
|
815
|
+
return order.map((c) => {
|
|
816
|
+
const attempts = e.attempts.filter((a) => cellOf(e, a.cell)?.environment === c.environment && countedOutcome(a.outcome));
|
|
817
|
+
const prompts = [...new Set(attempts.flatMap((a) => {
|
|
818
|
+
const p = a.turns[0]?.prompt;
|
|
819
|
+
return p ? [p] : [];
|
|
820
|
+
}))];
|
|
821
|
+
return {
|
|
822
|
+
id: c.environmentDigest ?? c.environment,
|
|
823
|
+
label: armLabel(c, digestClash),
|
|
824
|
+
environment: c.environment,
|
|
825
|
+
environmentDigest: c.environmentDigest,
|
|
826
|
+
role: c.role === "control" ? "baseline" : "variant",
|
|
827
|
+
change: c.role === "variant" ? c.variant ?? c.environment : null,
|
|
828
|
+
prompts: prompts.map((p) => {
|
|
829
|
+
let h = 0;
|
|
830
|
+
for (let i = 0; i < p.length; i++) h = h * 31 + p.charCodeAt(i) | 0;
|
|
831
|
+
return `p${(h >>> 0).toString(16).padStart(8, "0")}`;
|
|
832
|
+
})
|
|
833
|
+
};
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
function fairnessOf(nA, nB, interval) {
|
|
837
|
+
if (Math.min(nA, nB) < 4) return "too-few-runs";
|
|
838
|
+
if (interval && containsZero(interval.lo, interval.hi)) return "within-noise";
|
|
839
|
+
return "clear";
|
|
840
|
+
}
|
|
841
|
+
function floorSentence(fairness, n) {
|
|
842
|
+
if (fairness === "too-few-runs") return `${n} per side is below 4, so this cannot come back helped or made worse.`;
|
|
843
|
+
if (fairness === "within-noise") return "The interval spans 0: within noise.";
|
|
844
|
+
return "";
|
|
845
|
+
}
|
|
846
|
+
function headlinePrimaryText(primary) {
|
|
847
|
+
switch (primary.kind) {
|
|
848
|
+
case "pass-rate": return `${primary.rate.text} ${PASS_RATE_OUTCOME_LABEL[primary.outcome]}`;
|
|
849
|
+
case "lift": return `${primary.variant.label} vs ${primary.baseline.label}: ${primary.points.text} pass rate (${primary.variantRate.text} vs ${primary.baselineRate.text})`;
|
|
850
|
+
case "tied-ceiling": return `${primary.variant.label} vs ${primary.baseline.label}: tied at ${primary.rate.text} pass rate, median ${measurePhrase(primary.lead.measure)} ${primary.lead.delta.text} (${primary.lead.variant.text} vs ${primary.lead.baseline.text})`;
|
|
851
|
+
case "choice": return `${primary.question}: ${primary.options.map((o) => `${o.label} ${o.count}`).join(", ")}`;
|
|
852
|
+
case "measure": return `Median ${primary.label}: ${primary.spread.median.text}`;
|
|
853
|
+
case "judge": return `${primary.label}: median ${primary.spread.median.text} on ${primary.scale[0]} to ${primary.scale[1]}`;
|
|
854
|
+
case "text": return primary.question;
|
|
855
|
+
case "regression": {
|
|
856
|
+
const parts = [primary.worse ? `${primary.worse} worse` : null, primary.fixed ? `${primary.fixed} fixed` : null].filter(Boolean);
|
|
857
|
+
const counts = parts.length ? parts.join(", ") : "no gated flips";
|
|
858
|
+
if (!primary.lead) return counts;
|
|
859
|
+
return `${counts}: ${primary.lead.signal}, ${primary.lead.prior} to ${primary.lead.current}`;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
function chooseHeadline(e, arms, prior, regression) {
|
|
864
|
+
const harnesses = new Set(e.cells.map((c) => c.harness)).size;
|
|
865
|
+
const assertions = assertionLabels(e);
|
|
866
|
+
const compare = Boolean(e.compare);
|
|
867
|
+
let kicker = null;
|
|
868
|
+
if (regression && prior) {
|
|
869
|
+
const p = regression.prior;
|
|
870
|
+
kicker = `Compared with ${[
|
|
871
|
+
p.startedAt.slice(0, 10),
|
|
872
|
+
p.gitSha ? p.gitSha.slice(0, 7) : null,
|
|
873
|
+
p.runId
|
|
874
|
+
].filter(Boolean).join(", ")}`;
|
|
875
|
+
} else if (compare && arms.length >= 2) {
|
|
876
|
+
const baseline = arms.find((a) => a.role === "baseline");
|
|
877
|
+
const variant = arms.find((a) => a.role === "variant");
|
|
878
|
+
if (baseline && variant) kicker = `${variant.label} against the ${baseline.label} baseline`;
|
|
879
|
+
}
|
|
880
|
+
if (regression) {
|
|
881
|
+
const worse = regression.rows.filter((r) => r.status === "regressed").length;
|
|
882
|
+
const fixed = regression.rows.filter((r) => r.status === "fixed").length;
|
|
883
|
+
const lead = regression.rows.find((r) => r.gate && (r.status === "regressed" || r.status === "fixed")) ?? regression.rows.find((r) => r.status === "regressed" || r.status === "fixed") ?? regression.rows[0] ?? null;
|
|
884
|
+
const sampleNs = (prior?.rows ?? []).flatMap((row) => (row.perCell ?? []).map((c) => Math.min(c.base.pass + c.base.fail, c.head.pass + c.head.fail)));
|
|
885
|
+
const minN = sampleNs.length ? Math.min(...sampleNs) : 4;
|
|
886
|
+
const fairness = minN < 4 ? "too-few-runs" : "clear";
|
|
887
|
+
const footParts = [
|
|
888
|
+
`${worse} worse, ${fixed} fixed since ${regression.prior.runId}.`,
|
|
889
|
+
regression.prior.gitSha ? `Prior ${regression.prior.startedAt.slice(0, 10)} at ${regression.prior.gitSha.slice(0, 7)}.` : `Prior ${regression.prior.startedAt.slice(0, 10)}.`,
|
|
890
|
+
fairness === "too-few-runs" ? floorSentence("too-few-runs", minN) : null
|
|
891
|
+
];
|
|
892
|
+
return {
|
|
893
|
+
primary: {
|
|
894
|
+
kind: "regression",
|
|
895
|
+
worse,
|
|
896
|
+
fixed,
|
|
897
|
+
lead
|
|
898
|
+
},
|
|
899
|
+
fairness,
|
|
900
|
+
foot: footParts.filter(Boolean).join(" "),
|
|
901
|
+
kicker
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
if (compare && arms.length >= 2) {
|
|
905
|
+
const baseline = arms.find((a) => a.role === "baseline");
|
|
906
|
+
const variants = arms.filter((a) => a.role === "variant");
|
|
907
|
+
const attemptsFor = (arm) => e.attempts.filter((a) => {
|
|
908
|
+
const cell = cellOf(e, a.cell);
|
|
909
|
+
return cell && (cell.environmentDigest ?? cell.environment) === arm.id;
|
|
910
|
+
});
|
|
911
|
+
const rateFor = (arm) => passRateForAttempts(attemptsFor(arm));
|
|
912
|
+
const baselineRate = rateFor(baseline);
|
|
913
|
+
const lifts = variants.map((v) => {
|
|
914
|
+
const rate = rateFor(v);
|
|
915
|
+
const nV = rate.n;
|
|
916
|
+
const nB = baselineRate.n;
|
|
917
|
+
const { points, low, high } = liftInterval({
|
|
918
|
+
pass: rate.k,
|
|
919
|
+
n: nV
|
|
920
|
+
}, {
|
|
921
|
+
pass: baselineRate.k,
|
|
922
|
+
n: nB
|
|
923
|
+
});
|
|
924
|
+
return {
|
|
925
|
+
variant: {
|
|
926
|
+
id: v.id,
|
|
927
|
+
label: v.label
|
|
928
|
+
},
|
|
929
|
+
points: ptsDelta(points),
|
|
930
|
+
interval: ptsInterval(low, high),
|
|
931
|
+
rate,
|
|
932
|
+
rawPoints: points,
|
|
933
|
+
lo: low,
|
|
934
|
+
hi: high,
|
|
935
|
+
n: Math.min(nV, nB)
|
|
936
|
+
};
|
|
937
|
+
});
|
|
938
|
+
lifts.sort((a, b) => Math.abs(b.rawPoints) - Math.abs(a.rawPoints));
|
|
939
|
+
const best = lifts[0];
|
|
940
|
+
const others = lifts.slice(1).map(({ variant, points, interval, rate }) => ({
|
|
941
|
+
variant,
|
|
942
|
+
points,
|
|
943
|
+
interval,
|
|
944
|
+
rate
|
|
945
|
+
}));
|
|
946
|
+
const fairness = fairnessOf(best.rate.n, baselineRate.n, {
|
|
947
|
+
lo: best.lo,
|
|
948
|
+
hi: best.hi
|
|
949
|
+
});
|
|
950
|
+
if (REPORT_POLICY.tiedCeiling && baselineRate.rate === 1 && best.rate.rate === 1 && assertions.length > 0) {
|
|
951
|
+
const baseAttempts = attemptsFor(baseline).filter((a) => countedOutcome(a.outcome));
|
|
952
|
+
const varAttempts = attemptsFor(variants.find((v) => v.id === best.variant.id)).filter((a) => countedOutcome(a.outcome));
|
|
953
|
+
const baseMeasures = measuresOf(baseAttempts);
|
|
954
|
+
const varMeasures = measuresOf(varAttempts);
|
|
955
|
+
let lead = null;
|
|
956
|
+
for (const key of MEASURE_KEYS) {
|
|
957
|
+
const b = baseMeasures[key]?.median;
|
|
958
|
+
const v = varMeasures[key]?.median;
|
|
959
|
+
if (!b || !v || b.value === null || v.value === null) continue;
|
|
960
|
+
const delta = pctDelta(v.value, b.value);
|
|
961
|
+
if (delta.value === null) continue;
|
|
962
|
+
const abs = Math.abs(delta.value);
|
|
963
|
+
if (abs < .1) continue;
|
|
964
|
+
if (!lead || abs > lead.abs) lead = {
|
|
965
|
+
measure: key,
|
|
966
|
+
baseline: b,
|
|
967
|
+
variant: v,
|
|
968
|
+
delta,
|
|
969
|
+
abs
|
|
970
|
+
};
|
|
971
|
+
}
|
|
972
|
+
if (lead) {
|
|
973
|
+
const n = baselineRate.n + best.rate.n;
|
|
974
|
+
return {
|
|
975
|
+
primary: {
|
|
976
|
+
kind: "tied-ceiling",
|
|
977
|
+
baseline: {
|
|
978
|
+
id: baseline.id,
|
|
979
|
+
label: baseline.label
|
|
980
|
+
},
|
|
981
|
+
variant: best.variant,
|
|
982
|
+
rate: baselineRate,
|
|
983
|
+
assertions: assertions.length,
|
|
984
|
+
lead: {
|
|
985
|
+
measure: lead.measure,
|
|
986
|
+
baseline: lead.baseline,
|
|
987
|
+
variant: lead.variant,
|
|
988
|
+
delta: lead.delta
|
|
989
|
+
}
|
|
990
|
+
},
|
|
991
|
+
fairness,
|
|
992
|
+
foot: `Every attempt passed all ${assertions.length} applicable assertions (${n} of ${n}).`,
|
|
993
|
+
kicker
|
|
994
|
+
};
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
const floor = floorSentence(fairness, best.n);
|
|
998
|
+
return {
|
|
999
|
+
primary: {
|
|
1000
|
+
kind: "lift",
|
|
1001
|
+
baseline: {
|
|
1002
|
+
id: baseline.id,
|
|
1003
|
+
label: baseline.label
|
|
1004
|
+
},
|
|
1005
|
+
variant: best.variant,
|
|
1006
|
+
points: best.points,
|
|
1007
|
+
interval: best.interval,
|
|
1008
|
+
baselineRate,
|
|
1009
|
+
variantRate: best.rate,
|
|
1010
|
+
others
|
|
1011
|
+
},
|
|
1012
|
+
fairness,
|
|
1013
|
+
foot: [`n = ${best.rate.n} vs ${baselineRate.n} counted attempts. 95% CI ${best.interval.text}.`, floor].filter(Boolean).join(" "),
|
|
1014
|
+
kicker
|
|
1015
|
+
};
|
|
1016
|
+
}
|
|
1017
|
+
if (assertions.length > 0) {
|
|
1018
|
+
const gated = e.attempts.filter((a) => {
|
|
1019
|
+
const cell = cellOf(e, a.cell);
|
|
1020
|
+
return cell && cell.gate !== "none" && countedOutcome(a.outcome);
|
|
1021
|
+
});
|
|
1022
|
+
const pool = gated.length ? gated : e.attempts.filter((a) => countedOutcome(a.outcome));
|
|
1023
|
+
let passedAll = 0;
|
|
1024
|
+
for (const a of pool) {
|
|
1025
|
+
const applicable = (a.assertions ?? []).filter((x) => x.status !== "not_applicable");
|
|
1026
|
+
if (applicable.length === 0) continue;
|
|
1027
|
+
if (applicable.every((x) => x.status === "pass")) passedAll++;
|
|
1028
|
+
}
|
|
1029
|
+
const rate = rateOf(passedAll, pool.length);
|
|
1030
|
+
const outcome = passRateOutcomeFromMatchers(assertions.map((a) => a.matcher)) ?? "passed";
|
|
1031
|
+
const outcomeWord = PASS_RATE_OUTCOME_LABEL[outcome];
|
|
1032
|
+
return {
|
|
1033
|
+
primary: {
|
|
1034
|
+
kind: "pass-rate",
|
|
1035
|
+
rate,
|
|
1036
|
+
assertions: assertions.length,
|
|
1037
|
+
harnesses: Math.max(1, harnesses),
|
|
1038
|
+
outcome
|
|
1039
|
+
},
|
|
1040
|
+
fairness: pool.length < 4 ? "too-few-runs" : "clear",
|
|
1041
|
+
foot: `${rate.k} of ${rate.n} counted attempts ${outcomeWord} on all ${assertions.length} assertions. ${harnesses} coding harness${harnesses === 1 ? "" : "es"}.`,
|
|
1042
|
+
kicker
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
const answers = attemptAnswerRows(e);
|
|
1046
|
+
const choiceRow = answers.find((r) => r.choices && r.choices.length >= 2);
|
|
1047
|
+
if (choiceRow?.choices) {
|
|
1048
|
+
const options = choiceRow.choices.map((c) => ({
|
|
1049
|
+
label: c.choice,
|
|
1050
|
+
count: c.n
|
|
1051
|
+
}));
|
|
1052
|
+
const noPick = choiceRow.answers.filter((a) => !a.choice).length;
|
|
1053
|
+
return {
|
|
1054
|
+
primary: {
|
|
1055
|
+
kind: "choice",
|
|
1056
|
+
question: choiceRow.question,
|
|
1057
|
+
options,
|
|
1058
|
+
attempts: choiceRow.answers.length,
|
|
1059
|
+
noPick
|
|
1060
|
+
},
|
|
1061
|
+
fairness: "clear",
|
|
1062
|
+
foot: `${choiceRow.answers.length} attempts, ${noPick} without a pick.`,
|
|
1063
|
+
kicker
|
|
1064
|
+
};
|
|
1065
|
+
}
|
|
1066
|
+
const judgeRows = e.attempts.flatMap((a) => labeledJudgments(a));
|
|
1067
|
+
if (judgeRows.length) {
|
|
1068
|
+
const first = judgeRows[0];
|
|
1069
|
+
const scores = judgeRows.filter((j) => j.label === first.label).map((j) => j.judgment.score);
|
|
1070
|
+
const spread = spreadOf(scores, (n) => n.toFixed(2));
|
|
1071
|
+
const threshold = (first.check ?? null) === null ? null : first.judgment.threshold ?? null;
|
|
1072
|
+
const atOrAbove = threshold === null ? null : countOf(scores.filter((s) => s >= threshold).length, scores.length);
|
|
1073
|
+
return {
|
|
1074
|
+
primary: {
|
|
1075
|
+
kind: "judge",
|
|
1076
|
+
label: first.label,
|
|
1077
|
+
question: first.judgment.question,
|
|
1078
|
+
scale: [0, 1],
|
|
1079
|
+
threshold,
|
|
1080
|
+
spread,
|
|
1081
|
+
atOrAbove
|
|
1082
|
+
},
|
|
1083
|
+
fairness: scores.length < 4 ? "too-few-runs" : "clear",
|
|
1084
|
+
foot: atOrAbove ? `${atOrAbove.text} at or above the ${threshold} threshold.` : "Diagnostic judgment, no threshold.",
|
|
1085
|
+
kicker
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
const observeNumeric = e.attempts.flatMap((a) => (a.observations ?? []).flatMap((o) => typeof o.value === "number" ? [{
|
|
1089
|
+
name: o.name,
|
|
1090
|
+
value: o.value
|
|
1091
|
+
}] : []));
|
|
1092
|
+
if (observeNumeric.length) {
|
|
1093
|
+
const name = observeNumeric[0].name;
|
|
1094
|
+
const values = observeNumeric.filter((o) => o.name === name).map((o) => o.value);
|
|
1095
|
+
const spread = spreadOf(values, formatCount);
|
|
1096
|
+
return {
|
|
1097
|
+
primary: {
|
|
1098
|
+
kind: "measure",
|
|
1099
|
+
label: name,
|
|
1100
|
+
unit: "count",
|
|
1101
|
+
spread
|
|
1102
|
+
},
|
|
1103
|
+
fairness: values.length < 4 ? "too-few-runs" : "clear",
|
|
1104
|
+
foot: `${spread.n} attempts, interquartile range ${spread.q1.text} to ${spread.q3.text}.`,
|
|
1105
|
+
kicker
|
|
1106
|
+
};
|
|
1107
|
+
}
|
|
1108
|
+
const textRow = answers.find((r) => !r.choices);
|
|
1109
|
+
if (textRow) {
|
|
1110
|
+
const answered = textRow.answers.filter((a) => a.text.trim());
|
|
1111
|
+
const leadAnswer = answered[0];
|
|
1112
|
+
let lead = null;
|
|
1113
|
+
if (leadAnswer) {
|
|
1114
|
+
const attempt = e.attempts.find((a) => a.id === leadAnswer.attempt);
|
|
1115
|
+
if (attempt) lead = {
|
|
1116
|
+
text: leadAnswer.text.slice(0, 200),
|
|
1117
|
+
attempt: {
|
|
1118
|
+
id: "A?",
|
|
1119
|
+
key: attempt.key,
|
|
1120
|
+
href: "#"
|
|
1121
|
+
},
|
|
1122
|
+
harness: cellOf(e, attempt.cell)?.harness ?? attempt.cell,
|
|
1123
|
+
arm: cellOf(e, attempt.cell)?.environment ?? null,
|
|
1124
|
+
case: attempt.case ?? null,
|
|
1125
|
+
input: leadAnswer.input
|
|
1126
|
+
};
|
|
1127
|
+
}
|
|
1128
|
+
const leadRole = answered.length > 1 && lead ? "representative" : answered.length === 1 && lead ? "sole" : null;
|
|
1129
|
+
return {
|
|
1130
|
+
primary: {
|
|
1131
|
+
kind: "text",
|
|
1132
|
+
question: textRow.question,
|
|
1133
|
+
answered: countOf(answered.length, textRow.answers.length),
|
|
1134
|
+
lead,
|
|
1135
|
+
leadRole
|
|
1136
|
+
},
|
|
1137
|
+
fairness: "clear",
|
|
1138
|
+
foot: leadRole === "representative" ? `${answered.length} of ${textRow.answers.length} attempts answered. Showing 1 representative answer.` : `${answered.length} of ${textRow.answers.length} attempts answered.`,
|
|
1139
|
+
kicker
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
const tokenValues = e.attempts.filter((a) => countedOutcome(a.outcome) && a.usage).map((a) => totalTokens(a.usage));
|
|
1143
|
+
const spread = spreadOf(tokenValues, tokensText) ?? {
|
|
1144
|
+
n: 0,
|
|
1145
|
+
median: tokensFigure(null),
|
|
1146
|
+
q1: tokensFigure(null),
|
|
1147
|
+
q3: tokensFigure(null)
|
|
1148
|
+
};
|
|
1149
|
+
return {
|
|
1150
|
+
primary: {
|
|
1151
|
+
kind: "measure",
|
|
1152
|
+
label: "tokens",
|
|
1153
|
+
unit: "tokens",
|
|
1154
|
+
spread
|
|
1155
|
+
},
|
|
1156
|
+
fairness: tokenValues.length < 4 ? "too-few-runs" : "clear",
|
|
1157
|
+
foot: spread.n === 0 ? "No attempt reported usage." : `${spread.n} attempts, interquartile range ${spread.q1.text} to ${spread.q3.text}.`,
|
|
1158
|
+
kicker
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
function buildTaskResults(e) {
|
|
1162
|
+
const labels = assertionLabels(e);
|
|
1163
|
+
if (!labels.length) return null;
|
|
1164
|
+
const harnesses = [...new Set(e.cells.map((c) => c.harness))];
|
|
1165
|
+
const columns = harnesses;
|
|
1166
|
+
const assertions = labels.map((row) => {
|
|
1167
|
+
const cells = {};
|
|
1168
|
+
for (const h of columns) cells[h] = checkCellOf(e.attempts.filter((a) => cellOf(e, a.cell)?.harness === h), row.label);
|
|
1169
|
+
const total = checkCellOf(e.attempts.filter((a) => countedOutcome(a.outcome) || a.outcome === "infra_error"), row.label);
|
|
1170
|
+
return {
|
|
1171
|
+
label: row.label,
|
|
1172
|
+
matcher: row.matcher,
|
|
1173
|
+
at: row.at,
|
|
1174
|
+
cells,
|
|
1175
|
+
total
|
|
1176
|
+
};
|
|
1177
|
+
});
|
|
1178
|
+
const axes = ["Coding harness"];
|
|
1179
|
+
const breakdown = harnesses.map((h) => {
|
|
1180
|
+
const attempts = e.attempts.filter((a) => cellOf(e, a.cell)?.harness === h);
|
|
1181
|
+
return {
|
|
1182
|
+
keys: [h],
|
|
1183
|
+
rate: passRateForAttempts(attempts),
|
|
1184
|
+
measures: measuresOf(attempts)
|
|
1185
|
+
};
|
|
1186
|
+
});
|
|
1187
|
+
return {
|
|
1188
|
+
kind: "task",
|
|
1189
|
+
columns,
|
|
1190
|
+
assertions,
|
|
1191
|
+
axes,
|
|
1192
|
+
breakdown,
|
|
1193
|
+
median: breakdown.length > 1 ? {
|
|
1194
|
+
keys: ["Median"],
|
|
1195
|
+
rate: passRateForAttempts(e.attempts),
|
|
1196
|
+
measures: measuresOf(e.attempts.filter((a) => countedOutcome(a.outcome)))
|
|
1197
|
+
} : null
|
|
1198
|
+
};
|
|
1199
|
+
}
|
|
1200
|
+
function buildVariantComparison(e, arms) {
|
|
1201
|
+
const baseline = arms.find((a) => a.role === "baseline");
|
|
1202
|
+
const attemptsFor = (arm) => e.attempts.filter((a) => {
|
|
1203
|
+
const cell = cellOf(e, a.cell);
|
|
1204
|
+
return cell && (cell.environmentDigest ?? cell.environment) === arm.id;
|
|
1205
|
+
});
|
|
1206
|
+
const baseAttempts = baseline ? attemptsFor(baseline).filter((a) => countedOutcome(a.outcome)) : [];
|
|
1207
|
+
const baseMeasures = measuresOf(baseAttempts);
|
|
1208
|
+
const baseRate = passRateForAttempts(baseAttempts);
|
|
1209
|
+
const summary = arms.map((arm) => {
|
|
1210
|
+
const attempts = attemptsFor(arm).filter((a) => countedOutcome(a.outcome));
|
|
1211
|
+
const rate = passRateForAttempts(attempts);
|
|
1212
|
+
const measures = measuresOf(attempts);
|
|
1213
|
+
const vs = emptyMeasureVsBaseline();
|
|
1214
|
+
for (const key of MEASURE_KEYS) vs[key] = measureVs(measures[key], arm.role === "baseline" ? null : baseMeasures[key], null);
|
|
1215
|
+
if (arm.role === "baseline") return {
|
|
1216
|
+
arm: arm.label,
|
|
1217
|
+
rate,
|
|
1218
|
+
lift: null,
|
|
1219
|
+
interval: null,
|
|
1220
|
+
verdict: "baseline",
|
|
1221
|
+
verdictScope: "pass-rate",
|
|
1222
|
+
measures: vs
|
|
1223
|
+
};
|
|
1224
|
+
const { points, low, high } = liftInterval({
|
|
1225
|
+
pass: rate.k,
|
|
1226
|
+
n: rate.n
|
|
1227
|
+
}, {
|
|
1228
|
+
pass: baseRate.k,
|
|
1229
|
+
n: baseRate.n
|
|
1230
|
+
});
|
|
1231
|
+
const fairness = fairnessOf(rate.n, baseRate.n, {
|
|
1232
|
+
lo: low,
|
|
1233
|
+
hi: high
|
|
1234
|
+
});
|
|
1235
|
+
const verdict = fairness === "too-few-runs" ? "too-few-runs" : fairness === "within-noise" ? "within-noise" : points > 0 ? "helped" : points < 0 ? "made-worse" : "within-noise";
|
|
1236
|
+
return {
|
|
1237
|
+
arm: arm.label,
|
|
1238
|
+
rate,
|
|
1239
|
+
lift: ptsDelta(points),
|
|
1240
|
+
interval: ptsInterval(low, high),
|
|
1241
|
+
verdict,
|
|
1242
|
+
verdictScope: "pass-rate",
|
|
1243
|
+
measures: vs
|
|
1244
|
+
};
|
|
1245
|
+
});
|
|
1246
|
+
const labels = assertionLabels(e);
|
|
1247
|
+
arms.map((a) => a.label);
|
|
1248
|
+
let uniform = labels.length > 0;
|
|
1249
|
+
const assertionsByArm = [];
|
|
1250
|
+
for (const row of labels) {
|
|
1251
|
+
const cells = {};
|
|
1252
|
+
for (const arm of arms) {
|
|
1253
|
+
cells[arm.label] = checkCellOf(attemptsFor(arm), row.label);
|
|
1254
|
+
const cell = cells[arm.label];
|
|
1255
|
+
if (cell.status !== "pass" && cell.status !== "na") uniform = false;
|
|
1256
|
+
}
|
|
1257
|
+
const total = checkCellOf(e.attempts, row.label);
|
|
1258
|
+
assertionsByArm.push({
|
|
1259
|
+
label: row.label,
|
|
1260
|
+
matcher: row.matcher,
|
|
1261
|
+
at: row.at,
|
|
1262
|
+
cells,
|
|
1263
|
+
total
|
|
1264
|
+
});
|
|
1265
|
+
}
|
|
1266
|
+
const uniformLead = uniform ? `Every arm passed every applicable assertion (${labels.length} checks).` : null;
|
|
1267
|
+
const contrasts = contrastsOf(e).flatMap((c) => {
|
|
1268
|
+
const aCell = cellOf(e, c.a);
|
|
1269
|
+
const bCell = cellOf(e, c.b);
|
|
1270
|
+
if (!aCell || !bCell || !c.lift) return [];
|
|
1271
|
+
const fairness = fairnessOf(c.lift.variant.counted, c.lift.control.counted, c.lift.interval);
|
|
1272
|
+
const verdict = fairness === "too-few-runs" ? "too-few-runs" : fairness === "within-noise" ? "within-noise" : c.lift.points > 0 ? "helped" : c.lift.points < 0 ? "made-worse" : "within-noise";
|
|
1273
|
+
return [{
|
|
1274
|
+
from: aCell.variant ?? aCell.environment,
|
|
1275
|
+
to: bCell.variant ?? bCell.environment,
|
|
1276
|
+
measure: "pass-rate",
|
|
1277
|
+
delta: ptsDelta(c.lift.points),
|
|
1278
|
+
interval: ptsInterval(c.lift.interval.lo, c.lift.interval.hi),
|
|
1279
|
+
verdict,
|
|
1280
|
+
verdictScope: "pass-rate"
|
|
1281
|
+
}];
|
|
1282
|
+
});
|
|
1283
|
+
const effortByHarness = [...new Set(e.cells.map((c) => c.harness))].flatMap((h) => arms.map((arm) => {
|
|
1284
|
+
const attempts = e.attempts.filter((a) => {
|
|
1285
|
+
const cell = cellOf(e, a.cell);
|
|
1286
|
+
return cell?.harness === h && cell && (cell.environmentDigest ?? cell.environment) === arm.id && countedOutcome(a.outcome);
|
|
1287
|
+
});
|
|
1288
|
+
const measures = measuresOf(attempts);
|
|
1289
|
+
const vs = emptyMeasureVsBaseline();
|
|
1290
|
+
const baseM = measuresOf(baseline ? e.attempts.filter((a) => {
|
|
1291
|
+
const cell = cellOf(e, a.cell);
|
|
1292
|
+
return cell?.harness === h && cell && (cell.environmentDigest ?? cell.environment) === baseline.id && countedOutcome(a.outcome);
|
|
1293
|
+
}) : []);
|
|
1294
|
+
for (const key of MEASURE_KEYS) vs[key] = measureVs(measures[key], arm.role === "baseline" ? null : baseM[key], null);
|
|
1295
|
+
return {
|
|
1296
|
+
harness: h,
|
|
1297
|
+
arm: arm.label,
|
|
1298
|
+
rate: passRateForAttempts(attempts),
|
|
1299
|
+
measures: vs
|
|
1300
|
+
};
|
|
1301
|
+
}));
|
|
1302
|
+
return {
|
|
1303
|
+
kind: "comparison",
|
|
1304
|
+
arms,
|
|
1305
|
+
uniformLead,
|
|
1306
|
+
summary,
|
|
1307
|
+
assertionsByArm: uniformLead ? [] : assertionsByArm,
|
|
1308
|
+
behavior: [],
|
|
1309
|
+
contrasts,
|
|
1310
|
+
effortByHarness
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1313
|
+
function compareSideText(row, side) {
|
|
1314
|
+
const cells = row.perCell;
|
|
1315
|
+
if (cells?.length) {
|
|
1316
|
+
const pass = cells.reduce((n, c) => n + c[side].pass, 0);
|
|
1317
|
+
const n = pass + cells.reduce((n, c) => n + c[side].fail, 0);
|
|
1318
|
+
if (n > 0) return `${pass} of ${n}`;
|
|
1319
|
+
}
|
|
1320
|
+
return row[side] ?? "n/a";
|
|
1321
|
+
}
|
|
1322
|
+
function compareSideDelta(row) {
|
|
1323
|
+
const cells = row.perCell;
|
|
1324
|
+
if (!cells?.length) return null;
|
|
1325
|
+
const basePass = cells.reduce((n, c) => n + c.base.pass, 0);
|
|
1326
|
+
const baseN = cells.reduce((n, c) => n + c.base.pass + c.base.fail, 0);
|
|
1327
|
+
const headPass = cells.reduce((n, c) => n + c.head.pass, 0);
|
|
1328
|
+
const headN = cells.reduce((n, c) => n + c.head.pass + c.head.fail, 0);
|
|
1329
|
+
if (!baseN || !headN) return null;
|
|
1330
|
+
return ptsDelta(headPass / headN - basePass / baseN);
|
|
1331
|
+
}
|
|
1332
|
+
function buildRegression(prior) {
|
|
1333
|
+
const rows = [];
|
|
1334
|
+
const unchanged = [];
|
|
1335
|
+
for (const row of prior.rows) {
|
|
1336
|
+
const change = {
|
|
1337
|
+
signal: row.id,
|
|
1338
|
+
kind: "outcome",
|
|
1339
|
+
harness: null,
|
|
1340
|
+
prior: compareSideText(row, "base"),
|
|
1341
|
+
current: compareSideText(row, "head"),
|
|
1342
|
+
delta: compareSideDelta(row),
|
|
1343
|
+
status: row.status,
|
|
1344
|
+
gate: Boolean(row.gate)
|
|
1345
|
+
};
|
|
1346
|
+
if (row.status === "same") unchanged.push(change);
|
|
1347
|
+
else rows.push(change);
|
|
1348
|
+
}
|
|
1349
|
+
const whatChanged = [...prior.changes.map((c) => `${c.label}: ${c.base} to ${c.head}`), ...prior.rows.flatMap((r) => (r.changes ?? []).map((c) => `${r.id}: ${c}`))];
|
|
1350
|
+
const priorRec = prior.record;
|
|
1351
|
+
return {
|
|
1352
|
+
kind: "regression",
|
|
1353
|
+
baseRunId: priorRec.runId,
|
|
1354
|
+
prior: {
|
|
1355
|
+
runId: priorRec.runId,
|
|
1356
|
+
startedAt: priorRec.startedAt,
|
|
1357
|
+
gitSha: priorRec.git?.sha ?? null,
|
|
1358
|
+
branch: priorRec.git?.branch ?? null,
|
|
1359
|
+
author: priorRec.author
|
|
1360
|
+
},
|
|
1361
|
+
whatChanged,
|
|
1362
|
+
rows,
|
|
1363
|
+
unchanged
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
function buildChoices(e) {
|
|
1367
|
+
return attemptAnswerRows(e).filter((r) => r.choices && r.choices.length >= 2).map((r) => {
|
|
1368
|
+
const options = [...r.choices ?? []].map((c) => ({
|
|
1369
|
+
label: c.choice,
|
|
1370
|
+
count: c.n
|
|
1371
|
+
})).sort((a, b) => b.count - a.count || a.label.localeCompare(b.label));
|
|
1372
|
+
const noPick = r.answers.filter((a) => !a.choice).length;
|
|
1373
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
1374
|
+
for (const ans of r.answers) {
|
|
1375
|
+
const attempt = e.attempts.find((a) => a.id === ans.attempt);
|
|
1376
|
+
if (!attempt) continue;
|
|
1377
|
+
const cell = cellOf(e, attempt.cell);
|
|
1378
|
+
const harness = cell?.harness ?? attempt.cell;
|
|
1379
|
+
const arm = cell?.environment ?? null;
|
|
1380
|
+
const key = `${arm ?? ""}\0${harness}`;
|
|
1381
|
+
const row = byKey.get(key) ?? {
|
|
1382
|
+
arm,
|
|
1383
|
+
prompt: null,
|
|
1384
|
+
harness,
|
|
1385
|
+
counts: Object.fromEntries(options.map((o) => [o.label, 0])),
|
|
1386
|
+
attempts: 0
|
|
1387
|
+
};
|
|
1388
|
+
row.attempts++;
|
|
1389
|
+
if (ans.choice && row.counts[ans.choice] !== void 0) row.counts[ans.choice]++;
|
|
1390
|
+
byKey.set(key, row);
|
|
1391
|
+
}
|
|
1392
|
+
const rows = [...byKey.values()].map((row) => ({
|
|
1393
|
+
...row,
|
|
1394
|
+
text: options.map((o) => `${o.label} ${row.counts[o.label] ?? 0}`).join(", ")
|
|
1395
|
+
}));
|
|
1396
|
+
return {
|
|
1397
|
+
question: r.question,
|
|
1398
|
+
options,
|
|
1399
|
+
noPick,
|
|
1400
|
+
rows,
|
|
1401
|
+
passByPick: options.map((o) => {
|
|
1402
|
+
const picked = r.answers.filter((a) => a.choice === o.label);
|
|
1403
|
+
const passed = picked.filter((a) => a.outcome === "pass").length;
|
|
1404
|
+
return {
|
|
1405
|
+
option: o.label,
|
|
1406
|
+
rate: rateOf(passed, picked.length)
|
|
1407
|
+
};
|
|
1408
|
+
})
|
|
1409
|
+
};
|
|
1410
|
+
});
|
|
1411
|
+
}
|
|
1412
|
+
const SKILL_SOURCES = [
|
|
1413
|
+
"placed",
|
|
1414
|
+
"skill-dir",
|
|
1415
|
+
"elsewhere"
|
|
1416
|
+
];
|
|
1417
|
+
/**
|
|
1418
|
+
* The skill an attempt's first passing `toHaveCalledSkill` (or `poll.`, `resolves.` … but not a
|
|
1419
|
+
* `not.`) checked, read from its default label: the matcher as recorded, the skill, then any ` #n`
|
|
1420
|
+
* that keeps the label unique. Undefined when every such call had a label of its own, which
|
|
1421
|
+
* doesn't name the skill.
|
|
1422
|
+
*/
|
|
1423
|
+
function checkedSkill(a) {
|
|
1424
|
+
for (const x of a.assertions ?? []) {
|
|
1425
|
+
const parts = x.matcher.split(".");
|
|
1426
|
+
if (parts.at(-1) !== "toHaveCalledSkill" || parts.includes("not")) continue;
|
|
1427
|
+
if (x.status !== "pass" || x.labelled || !x.label.startsWith(`${x.matcher} `)) continue;
|
|
1428
|
+
const name = x.label.slice(x.matcher.length + 1).replace(/(?: #\d+)+$/, "");
|
|
1429
|
+
if (name) return name;
|
|
1430
|
+
}
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
* How many of `attempts` (each found the skill) used the skill it checked from each source, by its
|
|
1434
|
+
* first use in their facts: one source an attempt, so the counts sum to at most the attempts.
|
|
1435
|
+
*/
|
|
1436
|
+
function skillSources(attempts, facts) {
|
|
1437
|
+
if (!facts) return [];
|
|
1438
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1439
|
+
for (const a of attempts) {
|
|
1440
|
+
const name = checkedSkill(a);
|
|
1441
|
+
if (name === void 0) continue;
|
|
1442
|
+
const first = facts.get(a.key)?.skills.filter((s) => s.skill === name || s.skill.endsWith(`:${name}`)).sort((x, y) => x.firstLine - y.firstLine)[0];
|
|
1443
|
+
if (first) counts.set(first.source, (counts.get(first.source) ?? 0) + 1);
|
|
1444
|
+
}
|
|
1445
|
+
return SKILL_SOURCES.flatMap((source) => {
|
|
1446
|
+
const count = counts.get(source) ?? 0;
|
|
1447
|
+
return count > 0 ? [{
|
|
1448
|
+
source,
|
|
1449
|
+
count
|
|
1450
|
+
}] : [];
|
|
1451
|
+
});
|
|
1452
|
+
}
|
|
1453
|
+
function buildDiscovery(e, facts) {
|
|
1454
|
+
if (e.compare) return [];
|
|
1455
|
+
const matchers = [
|
|
1456
|
+
{
|
|
1457
|
+
what: "skill",
|
|
1458
|
+
matcher: "toHaveCalledSkill"
|
|
1459
|
+
},
|
|
1460
|
+
{
|
|
1461
|
+
what: "MCP tool",
|
|
1462
|
+
matcher: "toHaveCalledMcp"
|
|
1463
|
+
},
|
|
1464
|
+
{
|
|
1465
|
+
what: "web fetch",
|
|
1466
|
+
matcher: "toHaveFetched"
|
|
1467
|
+
}
|
|
1468
|
+
];
|
|
1469
|
+
const out = [];
|
|
1470
|
+
for (const m of matchers) {
|
|
1471
|
+
const attempts = e.attempts.filter((a) => countedOutcome(a.outcome));
|
|
1472
|
+
const foundAttempts = attempts.filter((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher) && x.status === "pass"));
|
|
1473
|
+
if (!foundAttempts.length && !attempts.some((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher)))) continue;
|
|
1474
|
+
const applicable = attempts.filter((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher)));
|
|
1475
|
+
if (!applicable.length) continue;
|
|
1476
|
+
out.push({
|
|
1477
|
+
what: m.what,
|
|
1478
|
+
matcher: m.matcher,
|
|
1479
|
+
found: countOf(foundAttempts.length, applicable.length),
|
|
1480
|
+
rows: [...new Set(e.cells.map((c) => c.harness))].map((h) => {
|
|
1481
|
+
const pool = applicable.filter((a) => cellOf(e, a.cell)?.harness === h);
|
|
1482
|
+
const found = pool.filter((a) => (a.assertions ?? []).some((x) => x.matcher.includes(m.matcher) && x.status === "pass"));
|
|
1483
|
+
return {
|
|
1484
|
+
keys: [h],
|
|
1485
|
+
found: countOf(found.length, pool.length)
|
|
1486
|
+
};
|
|
1487
|
+
}),
|
|
1488
|
+
sources: m.what === "skill" ? skillSources(foundAttempts, facts) : []
|
|
1489
|
+
});
|
|
1490
|
+
}
|
|
1491
|
+
return out;
|
|
1492
|
+
}
|
|
1493
|
+
function buildJudgments(e) {
|
|
1494
|
+
const scores = e.attempts.flatMap((a) => {
|
|
1495
|
+
const cell = cellOf(e, a.cell);
|
|
1496
|
+
return labeledJudgments(a).map((j) => ({
|
|
1497
|
+
...j,
|
|
1498
|
+
keys: [cell?.harness ?? a.cell, cell?.environment ?? ""].filter(Boolean)
|
|
1499
|
+
}));
|
|
1500
|
+
});
|
|
1501
|
+
if (!scores.length && !(e.answers ?? []).length) return null;
|
|
1502
|
+
const byLabel = /* @__PURE__ */ new Map();
|
|
1503
|
+
for (const s of scores) {
|
|
1504
|
+
const list = byLabel.get(s.label) ?? [];
|
|
1505
|
+
list.push(s);
|
|
1506
|
+
byLabel.set(s.label, list);
|
|
1507
|
+
}
|
|
1508
|
+
const hasArms = e.cells.some((c) => Boolean(c.environment));
|
|
1509
|
+
const judgeRows = [...byLabel.entries()].map(([label, rows]) => {
|
|
1510
|
+
const values = rows.map((r) => r.judgment.score);
|
|
1511
|
+
const spread = spreadOf(values, (n) => n.toFixed(2));
|
|
1512
|
+
const assertion = rows[0].check ?? null;
|
|
1513
|
+
const threshold = assertion === null ? null : rows[0].judgment.threshold ?? null;
|
|
1514
|
+
return {
|
|
1515
|
+
label,
|
|
1516
|
+
question: rows[0].judgment.question,
|
|
1517
|
+
assertion,
|
|
1518
|
+
threshold,
|
|
1519
|
+
keys: hasArms ? ["Coding harness", "Arm"] : ["Coding harness"],
|
|
1520
|
+
spread,
|
|
1521
|
+
atOrAbove: threshold === null ? null : countOf(values.filter((v) => v >= threshold).length, values.length)
|
|
1522
|
+
};
|
|
1523
|
+
});
|
|
1524
|
+
const pairwise = [];
|
|
1525
|
+
for (const ans of e.answers ?? []) {
|
|
1526
|
+
const ties = ans.votes.filter((v) => v.winner === "tie").length;
|
|
1527
|
+
for (const cut of ["within", "across"]) {
|
|
1528
|
+
const rates = ans.rates.filter((r) => r.cut === cut);
|
|
1529
|
+
if (rates.length < 2) {
|
|
1530
|
+
for (const r of rates) pairwise.push({
|
|
1531
|
+
question: ans.question,
|
|
1532
|
+
cut,
|
|
1533
|
+
a: r.name,
|
|
1534
|
+
b: "rest",
|
|
1535
|
+
aWins: r.wins,
|
|
1536
|
+
bWins: Math.max(0, r.n - r.wins),
|
|
1537
|
+
ties,
|
|
1538
|
+
pairs: r.n,
|
|
1539
|
+
reasons: []
|
|
1540
|
+
});
|
|
1541
|
+
continue;
|
|
1542
|
+
}
|
|
1543
|
+
const [a, b] = rates;
|
|
1544
|
+
if (!a || !b) continue;
|
|
1545
|
+
pairwise.push({
|
|
1546
|
+
question: ans.question,
|
|
1547
|
+
cut,
|
|
1548
|
+
a: a.name,
|
|
1549
|
+
b: b.name,
|
|
1550
|
+
aWins: a.wins,
|
|
1551
|
+
bWins: b.wins,
|
|
1552
|
+
ties,
|
|
1553
|
+
pairs: Math.max(a.n, b.n),
|
|
1554
|
+
reasons: []
|
|
1555
|
+
});
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
return {
|
|
1559
|
+
axes: hasArms ? ["Coding harness", "Arm"] : ["Coding harness"],
|
|
1560
|
+
scores: judgeRows,
|
|
1561
|
+
pairwise
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
1564
|
+
function buildQualitative(e, ids, linkBase, hrefs, hrefByKey) {
|
|
1565
|
+
return attemptAnswerRows(e).filter((r) => !r.choices).map((r) => {
|
|
1566
|
+
const entries = [];
|
|
1567
|
+
for (const ans of r.answers) {
|
|
1568
|
+
if (!ans.text.trim()) continue;
|
|
1569
|
+
const attempt = e.attempts.find((a) => a.id === ans.attempt);
|
|
1570
|
+
if (!attempt) continue;
|
|
1571
|
+
const ref = attemptRef(ids, e, attempt, linkBase, hrefs, hrefByKey);
|
|
1572
|
+
if (!ref) continue;
|
|
1573
|
+
const cell = cellOf(e, attempt.cell);
|
|
1574
|
+
entries.push({
|
|
1575
|
+
text: ans.text.slice(0, 400),
|
|
1576
|
+
attempt: ref,
|
|
1577
|
+
harness: cell?.harness ?? attempt.cell,
|
|
1578
|
+
arm: cell?.environment ?? null,
|
|
1579
|
+
case: attempt.case ? caseName(e, {
|
|
1580
|
+
key: attempt.case,
|
|
1581
|
+
name: void 0
|
|
1582
|
+
}) : null,
|
|
1583
|
+
input: ans.input ? withBase(linkBase, ans.input) : null
|
|
1584
|
+
});
|
|
1585
|
+
}
|
|
1586
|
+
return {
|
|
1587
|
+
question: r.question,
|
|
1588
|
+
answered: countOf(entries.length, r.answers.length),
|
|
1589
|
+
themes: null,
|
|
1590
|
+
entries
|
|
1591
|
+
};
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
function buildCaptures(e, ids, linkBase, hrefs, hrefByKey, pages) {
|
|
1595
|
+
const rows = /* @__PURE__ */ new Map();
|
|
1596
|
+
for (const a of e.attempts) {
|
|
1597
|
+
const arts = a.files.artifacts ?? [];
|
|
1598
|
+
if (!arts.length) continue;
|
|
1599
|
+
const ref = attemptRef(ids, e, a, linkBase, hrefs, hrefByKey);
|
|
1600
|
+
if (!ref) continue;
|
|
1601
|
+
const cell = cellOf(e, a.cell);
|
|
1602
|
+
const key = `${cell?.harness ?? ""}\0${cell?.environment ?? ""}\0${a.case ?? ""}`;
|
|
1603
|
+
const row = rows.get(key) ?? {
|
|
1604
|
+
harness: cell?.harness ?? a.cell,
|
|
1605
|
+
arm: cell?.environment ?? null,
|
|
1606
|
+
case: a.case ?? null,
|
|
1607
|
+
shots: []
|
|
1608
|
+
};
|
|
1609
|
+
for (const art of arts) {
|
|
1610
|
+
const pagePath = art.capture?.page?.path;
|
|
1611
|
+
const page = pagePath ? pages?.get(pagePath) : void 0;
|
|
1612
|
+
row.shots.push({
|
|
1613
|
+
name: art.name,
|
|
1614
|
+
href: captureHref(art.path, linkBase),
|
|
1615
|
+
meta: page?.finalUrl ?? (art.capture?.instructions?.length ? art.capture.instructions.join("; ") : art.capture?.path ?? null),
|
|
1616
|
+
attempt: ref,
|
|
1617
|
+
status: art.capture?.status ?? page?.status ?? null,
|
|
1618
|
+
pageErrors: art.capture?.pageErrors ?? page?.errors.length ?? null,
|
|
1619
|
+
consoleErrors: art.capture?.consoleErrors ?? page?.console.filter((c) => c.level === "error").length ?? null
|
|
1620
|
+
});
|
|
1621
|
+
}
|
|
1622
|
+
rows.set(key, row);
|
|
1623
|
+
}
|
|
1624
|
+
if (!rows.size) return null;
|
|
1625
|
+
return { rows: [...rows.values()] };
|
|
1626
|
+
}
|
|
1627
|
+
function buildFailures(e, ids, linkBase, hrefs, hrefByKey, remarks) {
|
|
1628
|
+
const counted = e.attempts.filter((a) => countedOutcome(a.outcome) || a.outcome === "infra_error");
|
|
1629
|
+
const clusters = /* @__PURE__ */ new Map();
|
|
1630
|
+
const add = (kind, label, at, attempt) => {
|
|
1631
|
+
const ref = attemptRef(ids, e, attempt, linkBase, hrefs, hrefByKey);
|
|
1632
|
+
if (!ref) return;
|
|
1633
|
+
const key = `${kind}\0${label}\0${at ?? ""}`;
|
|
1634
|
+
const existing = clusters.get(key);
|
|
1635
|
+
if (existing) {
|
|
1636
|
+
existing.attempts.push(ref);
|
|
1637
|
+
existing.count = countOf(existing.attempts.length, counted.length);
|
|
1638
|
+
return;
|
|
1639
|
+
}
|
|
1640
|
+
clusters.set(key, {
|
|
1641
|
+
kind,
|
|
1642
|
+
label,
|
|
1643
|
+
at,
|
|
1644
|
+
count: countOf(1, counted.length),
|
|
1645
|
+
attempts: [ref]
|
|
1646
|
+
});
|
|
1647
|
+
};
|
|
1648
|
+
for (const a of e.attempts) {
|
|
1649
|
+
for (const x of a.assertions ?? []) if (x.status === "fail") add("assertion", x.label, x.at ?? null, a);
|
|
1650
|
+
for (const t of a.turns) if (t.status === "error" && t.error) add("turn-error", messageLine(t.error), null, a);
|
|
1651
|
+
if (a.outcome === "infra_error" && a.error) add("infra-error", messageLine(a.error.message), null, a);
|
|
1652
|
+
else if (a.error && a.outcome === "fail") add("error", messageLine(a.error.message), null, a);
|
|
1653
|
+
}
|
|
1654
|
+
const evalPath = factSegment(e.id);
|
|
1655
|
+
const evalRemarks = remarks?.remarks.filter((r) => r.severity === "concern" && (r.evalKey === e.key || r.evalId === e.id || r.evalId === e.key || r.evalId === evalPath)).map((r) => ({
|
|
1656
|
+
kind: r.kind,
|
|
1657
|
+
severity: r.severity,
|
|
1658
|
+
text: r.text,
|
|
1659
|
+
fix: r.fix
|
|
1660
|
+
})) ?? [];
|
|
1661
|
+
if (!clusters.size && !evalRemarks.length) return null;
|
|
1662
|
+
return {
|
|
1663
|
+
clusters: [...clusters.values()].sort((a, b) => b.count.k - a.count.k),
|
|
1664
|
+
remarks: evalRemarks
|
|
1665
|
+
};
|
|
1666
|
+
}
|
|
1667
|
+
function fillTextLeadCites(primary, e, ids, linkBase, hrefs, hrefByKey) {
|
|
1668
|
+
if (primary.kind !== "text" || !primary.lead) return primary;
|
|
1669
|
+
const first = attemptAnswerRows(e).find((r) => r.question === primary.question)?.answers.find((a) => a.text.trim());
|
|
1670
|
+
const attempt = first && e.attempts.find((a) => a.id === first.attempt);
|
|
1671
|
+
const ref = attempt && attemptRef(ids, e, attempt, linkBase, hrefs, hrefByKey);
|
|
1672
|
+
if (!ref) return {
|
|
1673
|
+
...primary,
|
|
1674
|
+
lead: null,
|
|
1675
|
+
leadRole: null
|
|
1676
|
+
};
|
|
1677
|
+
return {
|
|
1678
|
+
...primary,
|
|
1679
|
+
lead: {
|
|
1680
|
+
...primary.lead,
|
|
1681
|
+
attempt: ref
|
|
1682
|
+
}
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
function buildEvalSection(ctx, e, displayName) {
|
|
1686
|
+
const arms = armsOf(e);
|
|
1687
|
+
const shapes = shapesOf(e, Boolean(ctx.prior));
|
|
1688
|
+
const design = designOf(e);
|
|
1689
|
+
const regression = ctx.prior ? buildRegression(ctx.prior) : null;
|
|
1690
|
+
const chosen = chooseHeadline(e, arms, ctx.prior, regression);
|
|
1691
|
+
const primaryHeadline = fillTextLeadCites(chosen.primary, e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey);
|
|
1692
|
+
const headline = {
|
|
1693
|
+
task: displayName,
|
|
1694
|
+
kicker: chosen.kicker,
|
|
1695
|
+
fairness: chosen.fairness,
|
|
1696
|
+
primary: primaryHeadline,
|
|
1697
|
+
foot: chosen.foot
|
|
1698
|
+
};
|
|
1699
|
+
let primary = null;
|
|
1700
|
+
if (regression) primary = regression;
|
|
1701
|
+
else if (e.compare) primary = buildVariantComparison(e, arms);
|
|
1702
|
+
else primary = buildTaskResults(e);
|
|
1703
|
+
const insights = [];
|
|
1704
|
+
if (ctx.analysis) {
|
|
1705
|
+
const body = ctx.analysis.analysis;
|
|
1706
|
+
for (const claim of [
|
|
1707
|
+
...body.whatHappened,
|
|
1708
|
+
...body.results,
|
|
1709
|
+
...body.why
|
|
1710
|
+
]) insights.push({
|
|
1711
|
+
role: "finding",
|
|
1712
|
+
text: claim.text,
|
|
1713
|
+
confidence: claim.confidence,
|
|
1714
|
+
cites: [],
|
|
1715
|
+
action: null
|
|
1716
|
+
});
|
|
1717
|
+
for (const anomaly of body.anomalies) insights.push({
|
|
1718
|
+
role: "caveat",
|
|
1719
|
+
text: anomaly.text,
|
|
1720
|
+
confidence: "medium",
|
|
1721
|
+
cites: [],
|
|
1722
|
+
action: null
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
return {
|
|
1726
|
+
id: e.key,
|
|
1727
|
+
anchor: gfmAnchor(displayName),
|
|
1728
|
+
name: displayName,
|
|
1729
|
+
file: e.file,
|
|
1730
|
+
shapes,
|
|
1731
|
+
design,
|
|
1732
|
+
headline,
|
|
1733
|
+
insights,
|
|
1734
|
+
primary,
|
|
1735
|
+
choices: buildChoices(e),
|
|
1736
|
+
discovery: buildDiscovery(e, ctx.facts),
|
|
1737
|
+
judgments: buildJudgments(e),
|
|
1738
|
+
qualitative: buildQualitative(e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey),
|
|
1739
|
+
captures: buildCaptures(e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey, ctx.pages),
|
|
1740
|
+
failures: buildFailures(e, ctx.ids, ctx.linkBase, ctx.hrefs, ctx.hrefByKey, ctx.remarks)
|
|
1741
|
+
};
|
|
1742
|
+
}
|
|
1743
|
+
function buildIntegrity(ctx) {
|
|
1744
|
+
const issues = [];
|
|
1745
|
+
const { record, evals, ids, linkBase } = ctx;
|
|
1746
|
+
if (record.status !== "complete") issues.push({
|
|
1747
|
+
kind: "run-incomplete",
|
|
1748
|
+
severity: "error",
|
|
1749
|
+
text: `Run stopped early (${record.status}); pass rates and medians cover only the attempts that finished.`,
|
|
1750
|
+
evalId: null,
|
|
1751
|
+
cites: []
|
|
1752
|
+
});
|
|
1753
|
+
if (record.git?.dirty) issues.push({
|
|
1754
|
+
kind: "dirty-worktree",
|
|
1755
|
+
severity: "warn",
|
|
1756
|
+
text: "The worktree was dirty; an exact rerun needs the saved worktree patch.",
|
|
1757
|
+
evalId: null,
|
|
1758
|
+
cites: []
|
|
1759
|
+
});
|
|
1760
|
+
for (const e of evals) {
|
|
1761
|
+
const cites = [];
|
|
1762
|
+
for (const a of e.attempts) {
|
|
1763
|
+
if (!a.turns.some((t) => t.status === "error")) continue;
|
|
1764
|
+
const ref = attemptRef(ids, e, a, linkBase, ctx.hrefs, ctx.hrefByKey);
|
|
1765
|
+
if (ref) cites.push(ref);
|
|
1766
|
+
}
|
|
1767
|
+
if (cites.length) issues.push({
|
|
1768
|
+
kind: "turn-error",
|
|
1769
|
+
severity: "error",
|
|
1770
|
+
text: `${cites.length} attempt${cites.length === 1 ? "" : "s"} had a mid-turn error; those outcomes may not reflect a finished agent turn.`,
|
|
1771
|
+
evalId: e.key,
|
|
1772
|
+
cites
|
|
1773
|
+
});
|
|
1774
|
+
const envNames = /* @__PURE__ */ new Map();
|
|
1775
|
+
for (const c of e.cells) {
|
|
1776
|
+
const set = envNames.get(c.environment) ?? /* @__PURE__ */ new Set();
|
|
1777
|
+
set.add(c.environmentDigest ?? c.environment);
|
|
1778
|
+
envNames.set(c.environment, set);
|
|
1779
|
+
}
|
|
1780
|
+
for (const [name, digests] of envNames) if (digests.size > 1) issues.push({
|
|
1781
|
+
kind: "duplicate-environment",
|
|
1782
|
+
severity: "warn",
|
|
1783
|
+
text: `Environment name "${name}" maps to ${digests.size} digests; arm labels include the digest prefix.`,
|
|
1784
|
+
evalId: e.key,
|
|
1785
|
+
cites: []
|
|
1786
|
+
});
|
|
1787
|
+
if (e.compare) {
|
|
1788
|
+
for (const c of e.cells.filter((x) => x.role === "control")) for (const check of c.checks ?? []) if (check.pass + check.fail > 0 && check.pass === 0) {
|
|
1789
|
+
if (e.cells.find((v) => v.role === "variant" && v.harness === c.harness && v.sandbox === c.sandbox && (v.checks ?? []).some((k) => k.label === check.label && k.pass === k.pass + k.fail && k.pass > 0))) issues.push({
|
|
1790
|
+
kind: "baseline-never-passes",
|
|
1791
|
+
severity: "warn",
|
|
1792
|
+
text: `Check "${check.label}" never passes on the baseline and always passes on a variant; it may not apply to the baseline.`,
|
|
1793
|
+
evalId: e.key,
|
|
1794
|
+
cites: []
|
|
1795
|
+
});
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
for (const a of e.attempts) if (!a.usage && countedOutcome(a.outcome)) {}
|
|
1799
|
+
const unreported = e.attempts.filter((a) => countedOutcome(a.outcome) && !a.usage);
|
|
1800
|
+
if (unreported.length) issues.push({
|
|
1801
|
+
kind: "unreported-usage",
|
|
1802
|
+
severity: "warn",
|
|
1803
|
+
text: `${unreported.length} counted attempt${unreported.length === 1 ? "" : "s"} reported no token usage; token medians omit them.`,
|
|
1804
|
+
evalId: e.key,
|
|
1805
|
+
cites: unreported.flatMap((a) => {
|
|
1806
|
+
const ref = attemptRef(ids, e, a, linkBase, ctx.hrefs, ctx.hrefByKey);
|
|
1807
|
+
return ref ? [ref] : [];
|
|
1808
|
+
})
|
|
1809
|
+
});
|
|
1810
|
+
const preHits = e.attempts.flatMap((a) => (a.assertions ?? []).filter((x) => x.preSatisfied).map((x) => ({
|
|
1811
|
+
attempt: a,
|
|
1812
|
+
label: x.label
|
|
1813
|
+
})));
|
|
1814
|
+
if (preHits.length) {
|
|
1815
|
+
const labels = [...new Set(preHits.map((h) => h.label))];
|
|
1816
|
+
issues.push({
|
|
1817
|
+
kind: "pre-satisfied",
|
|
1818
|
+
severity: "warn",
|
|
1819
|
+
text: `${preHits.length} pre-satisfied check${preHits.length === 1 ? "" : "s"} (${labels.join(", ")}); existing workspace state may satisfy them without agent work.`,
|
|
1820
|
+
evalId: e.key,
|
|
1821
|
+
cites: preHits.flatMap((h) => {
|
|
1822
|
+
const ref = attemptRef(ids, e, h.attempt, linkBase, ctx.hrefs, ctx.hrefByKey);
|
|
1823
|
+
return ref ? [ref] : [];
|
|
1824
|
+
})
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
}
|
|
1828
|
+
if (ctx.prior) {
|
|
1829
|
+
const sampleNs = ctx.prior.rows.flatMap((row) => (row.perCell ?? []).map((c) => Math.min(c.base.pass + c.base.fail, c.head.pass + c.head.fail)));
|
|
1830
|
+
if (ctx.prior.rows.some((row) => (row.status === "regressed" || row.status === "fixed") && Boolean(row.gate) && (row.perCell ?? []).some((c) => Math.min(c.base.pass + c.base.fail, c.head.pass + c.head.fail) < 4)) || sampleNs.length && Math.min(...sampleNs) < 4) {
|
|
1831
|
+
const minN = sampleNs.length ? Math.min(...sampleNs) : 0;
|
|
1832
|
+
issues.push({
|
|
1833
|
+
kind: "fairness-floor",
|
|
1834
|
+
severity: "warn",
|
|
1835
|
+
text: `A gated regression claim rests on fewer than 4 counted attempts per side${minN ? ` (as low as ${minN})` : ""}; treat worse/fixed as provisional.`,
|
|
1836
|
+
evalId: null,
|
|
1837
|
+
cites: []
|
|
1838
|
+
});
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
return issues;
|
|
1842
|
+
}
|
|
1843
|
+
function buildRoster(evals, record) {
|
|
1844
|
+
if (evals.length < 2) return [];
|
|
1845
|
+
const harnesses = record.stack.harnesses;
|
|
1846
|
+
if (harnesses.length < 2) return [];
|
|
1847
|
+
return harnesses.map((h) => {
|
|
1848
|
+
const attempts = evals.flatMap((e) => e.attempts.filter((a) => cellOf(e, a.cell)?.harness === h.id && a.seq !== null));
|
|
1849
|
+
const gateways = [...new Set(evals.flatMap((e) => e.cells.filter((c) => c.harness === h.id).map((c) => c.gateway).filter(Boolean)))];
|
|
1850
|
+
return {
|
|
1851
|
+
harness: h.id,
|
|
1852
|
+
model: h.model ?? null,
|
|
1853
|
+
effort: h.effort ?? null,
|
|
1854
|
+
servedModels: h.resolvedModels ?? [],
|
|
1855
|
+
gateway: gateways[0] ?? null,
|
|
1856
|
+
attempts: attempts.length
|
|
1857
|
+
};
|
|
1858
|
+
});
|
|
1859
|
+
}
|
|
1860
|
+
function buildRollup(sections, analysis) {
|
|
1861
|
+
if (sections.length < 2 && !analysis?.analysis.tldr) return null;
|
|
1862
|
+
const lead = analysis?.analysis.tldr ? {
|
|
1863
|
+
role: "finding",
|
|
1864
|
+
text: analysis.analysis.tldr,
|
|
1865
|
+
confidence: analysis.analysis.confidence.overall,
|
|
1866
|
+
cites: [],
|
|
1867
|
+
action: null
|
|
1868
|
+
} : null;
|
|
1869
|
+
const findings = analysis ? [...analysis.analysis.whatHappened, ...analysis.analysis.results].map((c) => ({
|
|
1870
|
+
role: "finding",
|
|
1871
|
+
text: c.text,
|
|
1872
|
+
confidence: c.confidence,
|
|
1873
|
+
cites: [],
|
|
1874
|
+
action: null
|
|
1875
|
+
})) : [];
|
|
1876
|
+
return {
|
|
1877
|
+
lead,
|
|
1878
|
+
rows: sections.map((s) => ({
|
|
1879
|
+
evalId: s.id,
|
|
1880
|
+
name: s.name,
|
|
1881
|
+
anchor: s.anchor,
|
|
1882
|
+
shapes: s.shapes,
|
|
1883
|
+
design: s.design,
|
|
1884
|
+
primary: headlinePrimaryText(s.headline.primary),
|
|
1885
|
+
medianTokens: s.headline.primary.kind === "measure" && s.headline.primary.unit === "tokens" ? s.headline.primary.spread.median : tokensFigure(s.primary && "breakdown" in s.primary ? s.primary.median?.measures.tokens?.median.value ?? s.primary.breakdown[0]?.measures.tokens?.median.value ?? null : null)
|
|
1886
|
+
})),
|
|
1887
|
+
findings
|
|
1888
|
+
};
|
|
1889
|
+
}
|
|
1890
|
+
function buildActions(analysis) {
|
|
1891
|
+
if (!analysis) return [];
|
|
1892
|
+
return analysis.analysis.actions.map((a) => ({
|
|
1893
|
+
role: "action",
|
|
1894
|
+
text: a.title,
|
|
1895
|
+
confidence: analysis.analysis.confidence.overall,
|
|
1896
|
+
cites: [],
|
|
1897
|
+
action: {
|
|
1898
|
+
kind: a.kind,
|
|
1899
|
+
target: a.target,
|
|
1900
|
+
change: a.change,
|
|
1901
|
+
verify: a.verify
|
|
1902
|
+
}
|
|
1903
|
+
}));
|
|
1904
|
+
}
|
|
1905
|
+
function buildAttemptIndex(ctx) {
|
|
1906
|
+
const { evals, ids, linkBase, record, hrefByKey } = ctx;
|
|
1907
|
+
const rows = [];
|
|
1908
|
+
for (const e of evals) for (const a of e.attempts) {
|
|
1909
|
+
if (a.seq === null) continue;
|
|
1910
|
+
const id = ids.get(a.key);
|
|
1911
|
+
if (!id) continue;
|
|
1912
|
+
const cell = cellOf(e, a.cell);
|
|
1913
|
+
const href = hrefByKey.get(a.key) ?? attemptHref(e, a, linkBase, ctx.hrefs);
|
|
1914
|
+
const diffStats = attemptDiffStats(a);
|
|
1915
|
+
const reclassified = a.outcome === "infra_error" && (a.turnStatusRead === false || a.turnStatusRead === void 0 && a.turns.some((t) => t.status === "error"));
|
|
1916
|
+
const canceled = a.outcome === "fail" && a.error?.kind === "aborted" && record.status === "interrupted";
|
|
1917
|
+
rows.push({
|
|
1918
|
+
id,
|
|
1919
|
+
key: a.key,
|
|
1920
|
+
evalId: e.key,
|
|
1921
|
+
harness: cell?.harness ?? a.cell,
|
|
1922
|
+
arm: cell?.environment ?? null,
|
|
1923
|
+
case: a.case ?? null,
|
|
1924
|
+
outcome: canceled ? "canceled" : a.outcome,
|
|
1925
|
+
reclassified,
|
|
1926
|
+
turnError: turnErrorMessage(a),
|
|
1927
|
+
tokens: tokensFigure(a.usage ? totalTokens(a.usage) : null),
|
|
1928
|
+
agentMs: durationFigure(a.timings.agentMs),
|
|
1929
|
+
toolCalls: countFigure(a.activity.tools),
|
|
1930
|
+
...ctx.facts?.has(a.key) ? { facts: ctx.facts.get(a.key) } : {},
|
|
1931
|
+
diff: diffStats.files === 0 && diffStats.added === null ? null : {
|
|
1932
|
+
files: diffStats.files,
|
|
1933
|
+
added: diffStats.added,
|
|
1934
|
+
removed: diffStats.removed,
|
|
1935
|
+
text: diffStats.added === null || diffStats.removed === null ? `${diffStats.files} files (lines not recorded)` : `${diffStats.files} files, ${diffLinesText(diffStats.added, diffStats.removed)}`
|
|
1936
|
+
},
|
|
1937
|
+
links: {
|
|
1938
|
+
transcript: a.files.transcript ? withBase(linkBase, a.files.transcript.path) : null,
|
|
1939
|
+
diff: a.files.diff ? withBase(linkBase, a.files.diff.path) : null,
|
|
1940
|
+
captures: (a.files.artifacts ?? []).map((art) => ({
|
|
1941
|
+
name: art.name,
|
|
1942
|
+
href: withBase(linkBase, art.path)
|
|
1943
|
+
})),
|
|
1944
|
+
questions: (a.answers ?? []).map((q) => q.input?.path).filter((p) => Boolean(p)).map((p) => withBase(linkBase, p))
|
|
1945
|
+
},
|
|
1946
|
+
href
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
return rows.sort((a, b) => Number(a.id.slice(1)) - Number(b.id.slice(1)));
|
|
1950
|
+
}
|
|
1951
|
+
/** The only reader of a `RunRecord` for a report. Pure and deterministic. */
|
|
1952
|
+
function buildRunReport(record, options = {}) {
|
|
1953
|
+
const evals = selectedEvals(record, options.evals);
|
|
1954
|
+
if (evals.length === 0) throw new Error("buildRunReport: no evals selected");
|
|
1955
|
+
const names = uniqueEvalNames(evals);
|
|
1956
|
+
const ctx = {
|
|
1957
|
+
record,
|
|
1958
|
+
evals,
|
|
1959
|
+
ids: attemptIds(record, new Set(evals.map((e) => e.key))),
|
|
1960
|
+
linkBase: options.links?.base,
|
|
1961
|
+
remarks: options.remarks ?? null,
|
|
1962
|
+
analysis: options.analysis ?? null,
|
|
1963
|
+
pages: options.pages,
|
|
1964
|
+
facts: options.facts,
|
|
1965
|
+
prior: options.prior ?? null,
|
|
1966
|
+
hrefs: /* @__PURE__ */ new Set(),
|
|
1967
|
+
hrefByKey: /* @__PURE__ */ new Map()
|
|
1968
|
+
};
|
|
1969
|
+
assignAttemptHrefs(ctx);
|
|
1970
|
+
const header = buildHeader(ctx);
|
|
1971
|
+
if (options.title) header.title = options.title;
|
|
1972
|
+
else header.title = runTitle(record, evals);
|
|
1973
|
+
const sections = evals.map((e) => buildEvalSection(ctx, e, names.get(e.key) ?? e.name));
|
|
1974
|
+
const single = sections.length === 1;
|
|
1975
|
+
const rollup = buildRollup(sections, ctx.analysis);
|
|
1976
|
+
const roster = buildRoster(evals, record);
|
|
1977
|
+
return {
|
|
1978
|
+
reportVersion: 1,
|
|
1979
|
+
runId: record.runId,
|
|
1980
|
+
single,
|
|
1981
|
+
header,
|
|
1982
|
+
integrity: buildIntegrity(ctx),
|
|
1983
|
+
rollup: single && !ctx.analysis?.analysis.tldr ? null : rollup,
|
|
1984
|
+
roster,
|
|
1985
|
+
evals: sections,
|
|
1986
|
+
actions: buildActions(ctx.analysis),
|
|
1987
|
+
provenance: provenanceState(record, options.links?.record),
|
|
1988
|
+
attempts: buildAttemptIndex(ctx)
|
|
1989
|
+
};
|
|
1990
|
+
}
|
|
1991
|
+
// src/view/report/split.ts
|
|
1992
|
+
/** The newest `reportVersion` this reeval writes and reads. */
|
|
1993
|
+
const REPORT_VERSION = 1;
|
|
1994
|
+
/** The attempt index's order: by the number in each row's display id, `A1..AN`. */
|
|
1995
|
+
const inIdOrder = (a, b) => Number(a.id.slice(1)) - Number(b.id.slice(1));
|
|
1996
|
+
/**
|
|
1997
|
+
* `report` as its index and one detail per eval, which `joinRunReport` puts back together exactly.
|
|
1998
|
+
* Each eval's `id` is its own, each attempt index row names its eval (`AttemptIndexRow.evalId`,
|
|
1999
|
+
* the section's `id`) and the rows are in their display ids' order, as `buildRunReport` writes
|
|
2000
|
+
* them; a report that isn't throws.
|
|
2001
|
+
*/
|
|
2002
|
+
function splitRunReport(report) {
|
|
2003
|
+
const { evals, attempts, ...top } = report;
|
|
2004
|
+
const rows = /* @__PURE__ */ new Map();
|
|
2005
|
+
for (const e of evals) {
|
|
2006
|
+
if (rows.has(e.id)) throw new Error(`Eval ${e.id} is in the report twice`);
|
|
2007
|
+
rows.set(e.id, []);
|
|
2008
|
+
}
|
|
2009
|
+
attempts.forEach((row, i) => {
|
|
2010
|
+
const of = rows.get(row.evalId);
|
|
2011
|
+
if (!of) throw new Error(`Attempt ${row.key} names eval ${row.evalId}, which the report lacks`);
|
|
2012
|
+
const prev = attempts[i - 1];
|
|
2013
|
+
if (prev && inIdOrder(prev, row) >= 0) throw new Error(`Attempt ${row.id} is out of the index's order`);
|
|
2014
|
+
of.push(row);
|
|
2015
|
+
});
|
|
2016
|
+
const details = evals.map((e) => ({
|
|
2017
|
+
reportVersion: report.reportVersion,
|
|
2018
|
+
runId: report.runId,
|
|
2019
|
+
eval: e,
|
|
2020
|
+
attempts: rows.get(e.id) ?? []
|
|
2021
|
+
}));
|
|
2022
|
+
return {
|
|
2023
|
+
index: {
|
|
2024
|
+
...top,
|
|
2025
|
+
evals: details.map(({ eval: e, attempts }) => ({
|
|
2026
|
+
id: e.id,
|
|
2027
|
+
anchor: e.anchor,
|
|
2028
|
+
name: e.name,
|
|
2029
|
+
file: e.file,
|
|
2030
|
+
shapes: e.shapes,
|
|
2031
|
+
design: e.design,
|
|
2032
|
+
headline: e.headline,
|
|
2033
|
+
attempts: attempts.length
|
|
2034
|
+
}))
|
|
2035
|
+
},
|
|
2036
|
+
details
|
|
2037
|
+
};
|
|
2038
|
+
}
|
|
2039
|
+
/** The `RunReport` that `splitRunReport` split into `index` and `details`, keys in its order. */
|
|
2040
|
+
function joinRunReport(index, details) {
|
|
2041
|
+
const byId = new Map(details.map((d) => [d.eval.id, d]));
|
|
2042
|
+
const detailOf = (id) => {
|
|
2043
|
+
const d = byId.get(id);
|
|
2044
|
+
if (!d) throw new Error(`No detail for eval ${id}`);
|
|
2045
|
+
return d;
|
|
2046
|
+
};
|
|
2047
|
+
const { evals, actions, provenance, ...head } = index;
|
|
2048
|
+
return {
|
|
2049
|
+
...head,
|
|
2050
|
+
evals: evals.map((e) => detailOf(e.id).eval),
|
|
2051
|
+
actions,
|
|
2052
|
+
provenance,
|
|
2053
|
+
attempts: evals.flatMap((e) => detailOf(e.id).attempts).sort(inIdOrder)
|
|
2054
|
+
};
|
|
2055
|
+
}
|
|
2056
|
+
export { worktreePatchLink as _, provenanceState as a, REPORT_POLICY as c, captureHref as d, evalAnchorHref as f, withBase as g, recordHref as h, buildRunReport as i, attemptHref as l, questionHref as m, joinRunReport as n, stateRows as o, gfmAnchor as p, splitRunReport as r, warnings as s, REPORT_VERSION as t, attemptRef as u, runScope as v };
|
|
2057
|
+
|
|
2058
|
+
//# sourceMappingURL=core-BXYzOEyQ.js.map
|