reeval 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.md +17 -17
- package/THIRD_PARTY_NOTICES.md +0 -2068
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +65 -50
- package/dist/{adapter-Dlq3-4OS.js → adapter-vLxVtLdk.js} +6 -6
- package/dist/{adapter-Dlq3-4OS.js.map → adapter-vLxVtLdk.js.map} +1 -1
- package/dist/{agent-OsAfayRf.js → agent-DN36i629.js} +9 -9
- package/dist/{agent-OsAfayRf.js.map → agent-DN36i629.js.map} +1 -1
- package/dist/{agents-BztvTkfM.js → agents-Bp40Y7Lk.js} +47 -59
- package/dist/agents-Bp40Y7Lk.js.map +1 -0
- package/dist/{analysis-BnG_1x_O.js → analysis-8vvMnUJB.js} +4 -5
- package/dist/analysis-8vvMnUJB.js.map +1 -0
- package/dist/api-ZeGnedB3.js +141 -0
- package/dist/api-ZeGnedB3.js.map +1 -0
- package/dist/assemble-ChNVeWNr.js +1 -0
- package/dist/{assemble-Doa1m-HW.js → assemble-DsllweEG.js} +132 -68
- package/dist/assemble-DsllweEG.js.map +1 -0
- package/dist/{attach-BH2HdwH_.js → attach-DFf2hFl7.js} +18 -10
- package/dist/attach-DFf2hFl7.js.map +1 -0
- package/dist/{attempt-_dUsblVx.js → attempt-CfwXgF2q.js} +3 -3
- package/dist/{attempt-_dUsblVx.js.map → attempt-CfwXgF2q.js.map} +1 -1
- package/dist/{attempt-host-8mQV8Uee.js → attempt-host-XyyFR6jc.js} +37 -14
- package/dist/attempt-host-XyyFR6jc.js.map +1 -0
- package/dist/boat-BCNNEJni.js +298 -0
- package/dist/boat-BCNNEJni.js.map +1 -0
- package/dist/{boat-DKjqo_ie.js → boat-CNUnhvWH.js} +117 -197
- package/dist/boat-CNUnhvWH.js.map +1 -0
- package/dist/boat-DRTQHXlj.js +1 -0
- package/dist/{boat-images-CRN_U_eG.js → boat-images-B0MsRBFx.js} +18 -22
- package/dist/boat-images-B0MsRBFx.js.map +1 -0
- package/dist/{bundle-DU2aQVeq.js → bundle-B1v8DEEX.js} +26 -15
- package/dist/bundle-B1v8DEEX.js.map +1 -0
- package/dist/{capture-By1byHaX.js → capture-BAtaw-L-.js} +20 -19
- package/dist/{capture-By1byHaX.js.map → capture-BAtaw-L-.js.map} +1 -1
- package/dist/capture-DOHR_KXl.js +1 -0
- package/dist/cases-Bsp-G-ZL.js +1 -0
- package/dist/{cases-CM-MQRQp.js → cases-C6vnV-gn.js} +61 -43
- package/dist/cases-C6vnV-gn.js.map +1 -0
- package/dist/classify-BIJQ2Xqo.js +163 -0
- package/dist/classify-BIJQ2Xqo.js.map +1 -0
- package/dist/{cli-DMHOwdVr.js → cli-BFfZy9Ax.js} +385 -178
- package/dist/cli-BFfZy9Ax.js.map +1 -0
- package/dist/{cli-spec-D0wuUaj9.js → cli-spec-Bj2z-4Pa.js} +242 -39
- package/dist/cli-spec-Bj2z-4Pa.js.map +1 -0
- package/dist/cli.js +6 -2
- package/dist/cli.js.map +1 -1
- package/dist/client-Dj65gXG3.js +105 -0
- package/dist/client-Dj65gXG3.js.map +1 -0
- package/dist/cloud-DgMSxpCk.js +154 -0
- package/dist/cloud-DgMSxpCk.js.map +1 -0
- package/dist/{hosted-5Zz2peTR.js → cloud-TuNUlZ0L.js} +31 -31
- package/dist/cloud-TuNUlZ0L.js.map +1 -0
- package/dist/cloud-files-BjoNhDJs.js +301 -0
- package/dist/cloud-files-BjoNhDJs.js.map +1 -0
- package/dist/cloud-project-B1lN8D9m.js +1 -0
- package/dist/cloud-project-Dh17iadw.js +110 -0
- package/dist/cloud-project-Dh17iadw.js.map +1 -0
- package/dist/cloud-push-CdYqsM34.js +817 -0
- package/dist/cloud-push-CdYqsM34.js.map +1 -0
- package/dist/cloud-push-GXvEYHoa.js +1 -0
- package/dist/cloud-run-BfFCs6Fm.js +1 -0
- package/dist/cloud-run-CZOpabN1.js +517 -0
- package/dist/cloud-run-CZOpabN1.js.map +1 -0
- package/dist/cloud-secrets-BskXM_2n.js +1 -0
- package/dist/cloud-secrets-DrrgGOF_.js +345 -0
- package/dist/cloud-secrets-DrrgGOF_.js.map +1 -0
- package/dist/{command-CXxckfsq.js → command-Bic6f84Y.js} +28 -130
- package/dist/command-Bic6f84Y.js.map +1 -0
- package/dist/{command-CAmnWLFL.js → command-C7m3ZoXS.js} +93 -27
- package/dist/command-C7m3ZoXS.js.map +1 -0
- package/dist/command-DHrT6YVl.js +183 -0
- package/dist/command-DHrT6YVl.js.map +1 -0
- package/dist/command-DMwSLfJT.js +141 -0
- package/dist/command-DMwSLfJT.js.map +1 -0
- package/dist/{command-ao0NZ8h0.js → command-DP8sQ-dk.js} +29 -23
- package/dist/command-DP8sQ-dk.js.map +1 -0
- package/dist/{commands-ttChCsAy.js → commands-BPLexk6R.js} +62 -41
- package/dist/commands-BPLexk6R.js.map +1 -0
- package/dist/commands-D8D7ZJ0I.js +1 -0
- package/dist/{config-myJT1Qjc.js → config-DQqcGmz7.js} +67 -47
- package/dist/config-DQqcGmz7.js.map +1 -0
- package/dist/core-BXYzOEyQ.js +2058 -0
- package/dist/core-BXYzOEyQ.js.map +1 -0
- package/dist/{slug-IHh3f770.js → cost-DQ0ayy6h.js} +37 -2
- package/dist/cost-DQ0ayy6h.js.map +1 -0
- package/dist/{cost-C21gGrju.js → cost-Dp-xFGJ1.js} +2 -2
- package/dist/{cost-C21gGrju.js.map → cost-Dp-xFGJ1.js.map} +1 -1
- package/dist/{cost-CdRX1s2p.js → cost-a_rUSdsf.js} +1 -1
- package/dist/{cursor-catalog-CUQrabJ_.js → cursor-catalog-DQCmwsX3.js} +4 -4
- package/dist/{cursor-catalog-CUQrabJ_.js.map → cursor-catalog-DQCmwsX3.js.map} +1 -1
- package/dist/{daytona-Dfhwtr9P.js → daytona-DOeJzXl_.js} +27 -19
- package/dist/daytona-DOeJzXl_.js.map +1 -0
- package/dist/{daytona-image-CnG6G00x.js → daytona-image-B9JpVeP5.js} +5 -4
- package/dist/daytona-image-B9JpVeP5.js.map +1 -0
- package/dist/daytona-image-BsCbvDGn.js +1 -0
- package/dist/db-BqFlDMmS.js +1 -0
- package/dist/{db-5H95sFii.js → db-DU3fgU0h.js} +32 -120
- package/dist/db-DU3fgU0h.js.map +1 -0
- package/dist/debug-CGg7KHKq.js +1 -0
- package/dist/{debug-CIXzB4wK.js → debug-T29p6HOX.js} +2 -10
- package/dist/debug-T29p6HOX.js.map +1 -0
- package/dist/{default-effort-DYpCkJRk.js → default-effort-BFl6WLBi.js} +3 -3
- package/dist/{default-effort-DYpCkJRk.js.map → default-effort-BFl6WLBi.js.map} +1 -1
- package/dist/{default-effort-CtNY9CGW.js → default-effort-C6Gcd-Fy.js} +1 -1
- package/dist/{dist-DeqeCUZe.js → dist-DCG7vZGQ.js} +3 -3
- package/dist/{dist-DeqeCUZe.js.map → dist-DCG7vZGQ.js.map} +1 -1
- package/dist/{docker-kpudeSxV.js → docker-B5YBmpPz.js} +1 -1
- package/dist/{docker-CR0JZ1FE.js → docker-hIo_H0W0.js} +4 -4
- package/dist/{docker-CR0JZ1FE.js.map → docker-hIo_H0W0.js.map} +1 -1
- package/dist/{doctor-aE-RMbB_.js → doctor-DGWbvYtu.js} +29 -29
- package/dist/{doctor-aE-RMbB_.js.map → doctor-DGWbvYtu.js.map} +1 -1
- package/dist/duration-Bz2H7pXb.js +34 -0
- package/dist/duration-Bz2H7pXb.js.map +1 -0
- package/dist/{environment-C8pUCD_A.js → environment-WC8PrLoe.js} +14 -11
- package/dist/environment-WC8PrLoe.js.map +1 -0
- package/dist/errors-BjzEWESY.js +93 -0
- package/dist/errors-BjzEWESY.js.map +1 -0
- package/dist/{errors-DPUoUTdK.js → errors-Cda7UzRA.js} +2 -2
- package/dist/{errors-DPUoUTdK.js.map → errors-Cda7UzRA.js.map} +1 -1
- package/dist/{eval-source-CaLIGH3U.js → eval-source-DL1HOi8U.js} +6 -6
- package/dist/{eval-source-CaLIGH3U.js.map → eval-source-DL1HOi8U.js.map} +1 -1
- package/dist/{evaluate-4kBVtwZZ.js → evaluate-DHnD8yzq.js} +280 -59
- package/dist/evaluate-DHnD8yzq.js.map +1 -0
- package/dist/{evaluate-DHaNtgVf.d.ts → evaluate-Nnh0iPuo.d.ts} +2 -3
- package/dist/{events-CXxFvVMK.js → events-oe84EOLK.js} +153 -34
- package/dist/{events-CXxFvVMK.js.map → events-oe84EOLK.js.map} +1 -1
- package/dist/{fake-bootstrap-CzxrCqO3.js → fake-bootstrap-Ct1qEs-Z.js} +2 -2
- package/dist/{fake-bootstrap-CzxrCqO3.js.map → fake-bootstrap-Ct1qEs-Z.js.map} +1 -1
- package/dist/files-2fO6iaJG.js +1 -0
- package/dist/{files-DpwJW3w6.js → files-4Qt4KUyR.js} +3 -3
- package/dist/{files-DpwJW3w6.js.map → files-4Qt4KUyR.js.map} +1 -1
- package/dist/{files-DgHeTdCZ.js → files-4XWe_gow.js} +27 -41
- package/dist/files-4XWe_gow.js.map +1 -0
- package/dist/files-63CNzRBF.js +1 -0
- package/dist/files-BDkCNIEh.js +1 -0
- package/dist/{files-CDfG8swM.js → files-Bv6ipFjc.js} +27 -18
- package/dist/files-Bv6ipFjc.js.map +1 -0
- package/dist/{files-Cs2dtb7-.js → files-ClUbQPj9.js} +2 -2
- package/dist/{files-Cs2dtb7-.js.map → files-ClUbQPj9.js.map} +1 -1
- package/dist/files-CzgXLYGq.js +1609 -0
- package/dist/files-CzgXLYGq.js.map +1 -0
- package/dist/files-DCcRkJN4.js +1 -0
- package/dist/{files-BkvxIqfI.js → files-DW8y3WCx.js} +5 -5
- package/dist/{files-BkvxIqfI.js.map → files-DW8y3WCx.js.map} +1 -1
- package/dist/gateway.d.ts +2 -3
- package/dist/{git-BnnNEXqh.js → git-CiCya2yZ.js} +3 -3
- package/dist/{git-BnnNEXqh.js.map → git-CiCya2yZ.js.map} +1 -1
- package/dist/{harness-FcTyyPco.js → harness-Dcd3ffkY.js} +3 -2
- package/dist/harness-Dcd3ffkY.js.map +1 -0
- package/dist/{harness-nwnyghJZ.js → harness-DvqNGwl6.js} +3 -3
- package/dist/{harness-nwnyghJZ.js.map → harness-DvqNGwl6.js.map} +1 -1
- package/dist/{harness-install-BfMBYDfP.js → harness-install-BSet5fE5.js} +4 -3
- package/dist/harness-install-BSet5fE5.js.map +1 -0
- package/dist/harness.d.ts +2 -3
- package/dist/harness.js +1 -1
- package/dist/http-DtlSzw4A.js +96 -0
- package/dist/http-DtlSzw4A.js.map +1 -0
- package/dist/{ids-a6cr243_.js → ids-CtxRn13a.js} +3 -2
- package/dist/ids-CtxRn13a.js.map +1 -0
- package/dist/{images-DtXwj2wL.js → images-BkJn7NHU.js} +13 -13
- package/dist/{images-DtXwj2wL.js.map → images-BkJn7NHU.js.map} +1 -1
- package/dist/index.d.ts +9 -5
- package/dist/index.js +4 -5
- package/dist/init-BOYNr3Ql.js +1 -0
- package/dist/{init-Ci_IAp8p.js → init-CAJvEKEt.js} +6 -6
- package/dist/{init-Ci_IAp8p.js.map → init-CAJvEKEt.js.map} +1 -1
- package/dist/{ink-list-Dnh6XD3C.js → ink-list-BJIKlKpN.js} +3 -3
- package/dist/{ink-list-Dnh6XD3C.js.map → ink-list-BJIKlKpN.js.map} +1 -1
- package/dist/{inputs-z4JqnQ5g.js → inputs-CjhDEzr4.js} +6 -6
- package/dist/{inputs-z4JqnQ5g.js.map → inputs-CjhDEzr4.js.map} +1 -1
- package/dist/inputs-load-C8HEtAej.js +1 -0
- package/dist/{inputs-load-DPUP_IFL.js → inputs-load-DQRC3dUA.js} +3 -3
- package/dist/{inputs-load-DPUP_IFL.js.map → inputs-load-DQRC3dUA.js.map} +1 -1
- package/dist/internal/vm-api/boat.d.ts +46 -0
- package/dist/internal/vm-api/boat.js +1 -0
- package/dist/internal/vm-api/daytona.d.ts +26 -0
- package/dist/internal/vm-api/daytona.js +220 -0
- package/dist/internal/vm-api/daytona.js.map +1 -0
- package/dist/internal/vm-api.d.ts +86 -0
- package/dist/internal/vm-api.js +3 -0
- package/dist/internal/vm.d.ts +7 -8
- package/dist/internal/vm.js +6 -6
- package/dist/internal/vm.js.map +1 -1
- package/dist/{invocations-C1gY3E0J.js → invocations-DUFiIb_S.js} +6 -6
- package/dist/{invocations-C1gY3E0J.js.map → invocations-DUFiIb_S.js.map} +1 -1
- package/dist/issue-BKMnnit3.js +120 -0
- package/dist/issue-BKMnnit3.js.map +1 -0
- package/dist/issue-esBvB5Df.js +1 -0
- package/dist/{jiti-ILwyxt93.js → jiti-aNfS_lSj.js} +2 -2
- package/dist/{jiti-ILwyxt93.js.map → jiti-aNfS_lSj.js.map} +1 -1
- package/dist/{judge-DwzqucLS.js → judge-_fsdF4YB.js} +7 -4
- package/dist/judge-_fsdF4YB.js.map +1 -0
- package/dist/judge.d.ts +5 -6
- package/dist/judge.js +1 -1
- package/dist/{judgments-Cj8bFaDG.js → judgments-CVCUWT-g.js} +8 -5
- package/dist/{judgments-Cj8bFaDG.js.map → judgments-CVCUWT-g.js.map} +1 -1
- package/dist/{kinds-DEKXOWVJ.js → kinds-5utDFJ15.js} +37 -12
- package/dist/kinds-5utDFJ15.js.map +1 -0
- package/dist/labels-Dtyf5-pd.js +251 -0
- package/dist/labels-Dtyf5-pd.js.map +1 -0
- package/dist/{layout-nCnU9Vu2.js → layout-Co4tPHWI.js} +3 -3
- package/dist/{layout-nCnU9Vu2.js.map → layout-Co4tPHWI.js.map} +1 -1
- package/dist/link-Dat_O7Gn.js +128 -0
- package/dist/link-Dat_O7Gn.js.map +1 -0
- package/dist/{list-MK4L8tfW.js → list-CjcNH-5e.js} +13 -8
- package/dist/list-CjcNH-5e.js.map +1 -0
- package/dist/live-BElpkw5b.js +56 -0
- package/dist/live-BElpkw5b.js.map +1 -0
- package/dist/{live-Dlq-S2yy.js → live-BRF0ogR6.js} +3 -5
- package/dist/live-BRF0ogR6.js.map +1 -0
- package/dist/{machine-service-Byl0BlcT.js → machine-service-Cymfk7E-.js} +1 -1
- package/dist/{machine-service-CZiXiVsS.js → machine-service-DYdTM_rW.js} +146 -13
- package/dist/machine-service-DYdTM_rW.js.map +1 -0
- package/dist/{machines-Ch5bhgvk.js → machines-B6rS9-Xu.js} +38 -38
- package/dist/{machines-Ch5bhgvk.js.map → machines-B6rS9-Xu.js.map} +1 -1
- package/dist/markdown-COts-jxf.js +653 -0
- package/dist/markdown-COts-jxf.js.map +1 -0
- package/dist/{model-lLaUxxno.js → model-BdLvfSQO.js} +1 -1
- package/dist/{model-DRa7yv6K.js → model-CIMBjBS7.js} +5 -5
- package/dist/{model-DRa7yv6K.js.map → model-CIMBjBS7.js.map} +1 -1
- package/dist/model-CrsA2Evf.js +1 -0
- package/dist/{model-CRh8aiYS.js → model-oBFmfzES.js} +4 -4
- package/dist/{model-CRh8aiYS.js.map → model-oBFmfzES.js.map} +1 -1
- package/dist/{model-LoLnq9y1.js → model-xemZQ7DK.js} +251 -65
- package/dist/model-xemZQ7DK.js.map +1 -0
- package/dist/{node-BQjSQw0d.js → node-DyRPGAT5.js} +19 -8
- package/dist/{node-BQjSQw0d.js.map → node-DyRPGAT5.js.map} +1 -1
- package/dist/options-CeIRZslO.js +1 -0
- package/dist/otlp-ehLWBhVL.js +123 -0
- package/dist/otlp-ehLWBhVL.js.map +1 -0
- package/dist/outcome-1LBk2e3w.js +38 -0
- package/dist/outcome-1LBk2e3w.js.map +1 -0
- package/dist/{patch-D8X6VB-e.js → patch-BgSw3TXR.js} +3 -3
- package/dist/{patch-D8X6VB-e.js.map → patch-BgSw3TXR.js.map} +1 -1
- package/dist/{patch-C7yBztZj.js → patch-uwo6t4Zy.js} +26 -2
- package/dist/patch-uwo6t4Zy.js.map +1 -0
- package/dist/{pi-providers-HygmL1y-.js → pi-providers-BjcQv2A1.js} +23 -9
- package/dist/{pi-providers-HygmL1y-.js.map → pi-providers-BjcQv2A1.js.map} +1 -1
- package/dist/{plain-Ctn8oUdp.js → plain-DXUNwqpO.js} +2 -2
- package/dist/{plain-Ctn8oUdp.js.map → plain-DXUNwqpO.js.map} +1 -1
- package/dist/{plan-DgIVIrXJ.js → plan-x-_vZMQH.js} +11 -11
- package/dist/{plan-DgIVIrXJ.js.map → plan-x-_vZMQH.js.map} +1 -1
- package/dist/{plugins-DvEbow6a.js → plugins-0DuNw_c5.js} +7 -7
- package/dist/{plugins-DvEbow6a.js.map → plugins-0DuNw_c5.js.map} +1 -1
- package/dist/{preflight-dJqs5x3r.js → preflight-0Wt3wufx.js} +1 -1
- package/dist/preflight-CKrvQ-of.js +1 -0
- package/dist/{preflight-BKGbxN67.js → preflight-CUc-Y1d3.js} +9 -9
- package/dist/{preflight-BKGbxN67.js.map → preflight-CUc-Y1d3.js.map} +1 -1
- package/dist/{preflight-Qm4wE094.js → preflight-Cce8YEPE.js} +4 -4
- package/dist/{preflight-Qm4wE094.js.map → preflight-Cce8YEPE.js.map} +1 -1
- package/dist/{prepare-CRPPfRJf.js → prepare-Bj8KzeBU.js} +2 -2
- package/dist/{prepare-BAjRSoku.js → prepare-FXc1YcxO.js} +8 -8
- package/dist/{prepare-BAjRSoku.js.map → prepare-FXc1YcxO.js.map} +1 -1
- package/dist/prices-CfnkJOth.js +1 -0
- package/dist/{prices-DCokI_Ka.js → prices-CoLymMRd.js} +3 -3
- package/dist/{prices-DCokI_Ka.js.map → prices-CoLymMRd.js.map} +1 -1
- package/dist/process-B86Nvm4J.js +118 -0
- package/dist/process-B86Nvm4J.js.map +1 -0
- package/dist/provenance-Cylt_YxT.d.ts +222 -0
- package/dist/{provenance-CSyRCc9B.js → provenance-DDZGWE4O.js} +6 -6
- package/dist/{provenance-CSyRCc9B.js.map → provenance-DDZGWE4O.js.map} +1 -1
- package/dist/provider-B24hZX95.js +1 -0
- package/dist/{provider-ieWOn4YS.js → provider-Cn7lmECy.js} +2 -2
- package/dist/{provider-ieWOn4YS.js.map → provider-Cn7lmECy.js.map} +1 -1
- package/dist/{provider-stop-CxzWW7Ft.js → provider-stop-DjBMfZW5.js} +2 -2
- package/dist/{provider-stop-CxzWW7Ft.js.map → provider-stop-DjBMfZW5.js.map} +1 -1
- package/dist/{providers-DxZ26YoG.js → providers-BPujoqQM.js} +1 -1
- package/dist/{providers-DkaEJvF5.js → providers-Cqb25Aq1.js} +35 -35
- package/dist/{providers-DkaEJvF5.js.map → providers-Cqb25Aq1.js.map} +1 -1
- package/dist/{query-BL66BOqL.js → query-C_lwW-ok.js} +2 -2
- package/dist/{query-BL66BOqL.js.map → query-C_lwW-ok.js.map} +1 -1
- package/dist/query-DzS1Z4bw.js +1 -0
- package/dist/{questions-boVPzGoC.js → questions-DRPM7pky.js} +19 -13
- package/dist/questions-DRPM7pky.js.map +1 -0
- package/dist/{recipe-qyIpP96r.js → recipe-CV_EifJk.js} +4 -4
- package/dist/{recipe-qyIpP96r.js.map → recipe-CV_EifJk.js.map} +1 -1
- package/dist/{record-D1_D1KFY.js → record-Cj3yWR0E.js} +50 -13
- package/dist/record-Cj3yWR0E.js.map +1 -0
- package/dist/{redact-t-L3VGy8.js → redact-DMueYS-v.js} +28 -6
- package/dist/redact-DMueYS-v.js.map +1 -0
- package/dist/{remarks-DqriPKxm.js → remarks-BPKGorBW.js} +4 -2
- package/dist/remarks-BPKGorBW.js.map +1 -0
- package/dist/{remarks-DEvk5f8v.js → remarks-Bded6YeV.js} +22 -12
- package/dist/{remarks-DEvk5f8v.js.map → remarks-Bded6YeV.js.map} +1 -1
- package/dist/{remote-CQoBRpOv.js → remote-DMuAUedR.js} +9 -7
- package/dist/{remote-CQoBRpOv.js.map → remote-DMuAUedR.js.map} +1 -1
- package/dist/renderer/ascii.d.ts +6 -0
- package/dist/renderer/ascii.js +7 -0
- package/dist/renderer/ascii.js.map +1 -0
- package/dist/renderer/markdown.d.ts +31 -0
- package/dist/renderer/markdown.js +1 -0
- package/dist/report-BSi-R8hC.js +941 -0
- package/dist/report-BSi-R8hC.js.map +1 -0
- package/dist/report-inputs-JhiwKAZH.js +113 -0
- package/dist/report-inputs-JhiwKAZH.js.map +1 -0
- package/dist/{resolve-Bfo8sjd-.js → resolve-BEltmYhD.js} +23 -7
- package/dist/resolve-BEltmYhD.js.map +1 -0
- package/dist/resolve-s6JzFZu0.js +1 -0
- package/dist/{resolve-D2RqmybX.js → resolve-usf6ejs_.js} +64 -19
- package/dist/resolve-usf6ejs_.js.map +1 -0
- package/dist/{results-BpxzyoMd.js → results-rYLTGa1F.js} +11 -7
- package/dist/{results-BpxzyoMd.js.map → results-rYLTGa1F.js.map} +1 -1
- package/dist/{resume-2ekm_9OB.js → resume-aGIZhM58.js} +7 -7
- package/dist/{resume-2ekm_9OB.js.map → resume-aGIZhM58.js.map} +1 -1
- package/dist/{retry-CAhpW1hH.js → retry-BNFAHTES.js} +5 -5
- package/dist/{retry-CAhpW1hH.js.map → retry-BNFAHTES.js.map} +1 -1
- package/dist/{root-BzeoyfGd.js → root-Co0jhOT5.js} +2 -2
- package/dist/{root-BzeoyfGd.js.map → root-Co0jhOT5.js.map} +1 -1
- package/dist/run-DSKRllpd.js +127 -0
- package/dist/run-DSKRllpd.js.map +1 -0
- package/dist/{client-B41UK150.js → runtime-B2zv9GPL.js} +525 -105
- package/dist/runtime-B2zv9GPL.js.map +1 -0
- package/dist/sandbox/boat.d.ts +4 -5
- package/dist/sandbox/boat.js +2 -2
- package/dist/sandbox/daytona.d.ts +4 -5
- package/dist/sandbox/daytona.js +1 -1
- package/dist/{sandbox-t-Q-8lA_.js → sandbox-DbTHBA7g.js} +3 -3
- package/dist/{sandbox-t-Q-8lA_.js.map → sandbox-DbTHBA7g.js.map} +1 -1
- package/dist/sandbox.d.ts +4 -5
- package/dist/sandbox.js +1 -1
- package/dist/secrets-file-BEScYC3S.js +100 -0
- package/dist/secrets-file-BEScYC3S.js.map +1 -0
- package/dist/secrets-file-take-BXfYMTM-.js +33 -0
- package/dist/secrets-file-take-BXfYMTM-.js.map +1 -0
- package/dist/server-BIqAFNjF.js +1 -0
- package/dist/server-Dc-V_-C1.js +204 -0
- package/dist/server-Dc-V_-C1.js.map +1 -0
- package/dist/{services-BG8xuU82.js → services-CHh83UDC.js} +4 -2
- package/dist/services-CHh83UDC.js.map +1 -0
- package/dist/shared-B3iLKAiu.d.ts +1 -0
- package/dist/{shared-DtZwIAfy.js → shared-bXaqVxk-.js} +5 -34
- package/dist/shared-bXaqVxk-.js.map +1 -0
- package/dist/skill-dirs-B7o83G5I.js +63 -0
- package/dist/skill-dirs-B7o83G5I.js.map +1 -0
- package/dist/{source-CrXM19HD.js → source-BodZTD70.js} +33 -18
- package/dist/source-BodZTD70.js.map +1 -0
- package/dist/source-BtdFbZLW.js +1 -0
- package/dist/spans-C3FlFC22.js +1 -0
- package/dist/spans-Dq3fo6HM.js +6 -0
- package/dist/spans-Dq3fo6HM.js.map +1 -0
- package/dist/spans-WvAmJF2z.js +72 -0
- package/dist/spans-WvAmJF2z.js.map +1 -0
- package/dist/{src-C6MsGyD6.js → src-AAdLbAn2.js} +1 -1
- package/dist/{src-VG2IRoT4.js → src-BbaLTc-p.js} +2 -2
- package/dist/src-BjiNaGf5.js +2 -0
- package/dist/{src-Cf4vRyea.js → src-BkkdN2w0.js} +2 -2
- package/dist/{src-C5rrQlt8.js → src-BzeiF5Lh.js} +197 -138
- package/dist/src-BzeiF5Lh.js.map +1 -0
- package/dist/{src-Ds1pmXTg.js → src-CfPSD_GS.js} +2 -2
- package/dist/{src-C-ChdnG6.js → src-ClMhIgi3.js} +2 -2
- package/dist/{src-C-ChdnG6.js.map → src-ClMhIgi3.js.map} +1 -1
- package/dist/{src-io_SKoZ5.js → src-D5i8gCa3.js} +2 -2
- package/dist/{src-Bb3k9Kn_.js → src-DDIV_RfE.js} +65 -10
- package/dist/{src-Bb3k9Kn_.js.map → src-DDIV_RfE.js.map} +1 -1
- package/dist/src-DXZXofCJ.js +1 -0
- package/dist/src-Db01fVzc.js +2 -0
- package/dist/{src-DwcTn4JY.js → src-DhZuOcgq.js} +5 -5
- package/dist/{src-DwcTn4JY.js.map → src-DhZuOcgq.js.map} +1 -1
- package/dist/{src-DwcCKsV0.js → src-DiBd7r1i.js} +3 -3
- package/dist/{src-DNWiU__5.js → src-Dm2YIeXn.js} +8 -8
- package/dist/{src-DNWiU__5.js.map → src-Dm2YIeXn.js.map} +1 -1
- package/dist/{src-c3ZQu7Uu.js → src-NYxoDS3b.js} +5 -5
- package/dist/{src-c3ZQu7Uu.js.map → src-NYxoDS3b.js.map} +1 -1
- package/dist/{src-C3OhA5pw.js → src-Y4AiZxLt.js} +5 -5
- package/dist/{src-C3OhA5pw.js.map → src-Y4AiZxLt.js.map} +1 -1
- package/dist/{src-Bf5RZlPi.js → src-YpmRtJmP.js} +2 -2
- package/dist/{src-Bf5RZlPi.js.map → src-YpmRtJmP.js.map} +1 -1
- package/dist/{ssh-pool-Dvfx5GuV.js → ssh-pool-B2c2vxzx.js} +20 -16
- package/dist/ssh-pool-B2c2vxzx.js.map +1 -0
- package/dist/ssh-pool-mcCYe6RO.js +1 -0
- package/dist/{start-COB07NpH.js → start-DyXCyHGX.js} +64 -12
- package/dist/start-DyXCyHGX.js.map +1 -0
- package/dist/{state-B319V1p8.js → state-CiR8Ema7.js} +21 -38
- package/dist/state-CiR8Ema7.js.map +1 -0
- package/dist/state-DBuFimCA.js +1 -0
- package/dist/store/local.d.ts +2 -3
- package/dist/store/s3.d.ts +2 -3
- package/dist/studio-share-C05twqpy.js +209 -0
- package/dist/studio-share-C05twqpy.js.map +1 -0
- package/dist/suite-fO-u8jlS.js +2 -0
- package/dist/{suite-D1tgXZxM.js → suite-oH7uiCRq.js} +108 -107
- package/dist/suite-oH7uiCRq.js.map +1 -0
- package/dist/summary-CTMWfnOF.js +75 -0
- package/dist/summary-CTMWfnOF.js.map +1 -0
- package/dist/{sync-mOG74RXH.js → sync-r8EvW6Hr.js} +6 -6
- package/dist/{sync-mOG74RXH.js.map → sync-r8EvW6Hr.js.map} +1 -1
- package/dist/{tart-DMppfbEm.js → tart-BUh58nxA.js} +37 -10
- package/dist/tart-BUh58nxA.js.map +1 -0
- package/dist/{tart-DmVuVsmX.js → tart-DNrumWZu.js} +1 -1
- package/dist/{tart-CzViW3WZ.js → tart-Duqp33Zh.js} +2 -2
- package/dist/{tart-CzViW3WZ.js.map → tart-Duqp33Zh.js.map} +1 -1
- package/dist/{tart-bake-Cva_qdir.js → tart-bake-ChrVpjuF.js} +8 -7
- package/dist/tart-bake-ChrVpjuF.js.map +1 -0
- package/dist/{tart-bake-DVIoXWr_.js → tart-bake-LwdUVOzS.js} +1 -1
- package/dist/{tart-cli-LzDud2Z6.js → tart-cli-Dv3dNC13.js} +3 -2
- package/dist/tart-cli-Dv3dNC13.js.map +1 -0
- package/dist/{tart-cli-D88c5YYJ.js → tart-cli-S388NU-r.js} +1 -1
- package/dist/{tart-hosts-CTRY5kQ2.js → tart-hosts-B-dQEGpc.js} +7 -6
- package/dist/tart-hosts-B-dQEGpc.js.map +1 -0
- package/dist/{tart-hosts-8N6Tt7Yb.js → tart-hosts-_nHGB4mC.js} +1 -1
- package/dist/{tart-images-JfHirU8W.js → tart-images-DAEYoC_M.js} +3 -3
- package/dist/{tart-images-JfHirU8W.js.map → tart-images-DAEYoC_M.js.map} +1 -1
- package/dist/testing.d.ts +13 -5
- package/dist/testing.js +4 -4
- package/dist/{tool-class-CncfCw03.js → tool-class-CB-m4PiF.js} +2 -2
- package/dist/{tool-class-CncfCw03.js.map → tool-class-CB-m4PiF.js.map} +1 -1
- package/dist/{trace-BpXByD7W.js → trace-CJQLlAbB.js} +2 -2
- package/dist/{trace-BpXByD7W.js.map → trace-CJQLlAbB.js.map} +1 -1
- package/dist/{traits-CCD8XsmI.js → traits-DPIffsN8.js} +2 -2
- package/dist/{traits-CCD8XsmI.js.map → traits-DPIffsN8.js.map} +1 -1
- package/dist/{transcript-BXWZiMuC.js → transcript-CW02A4s9.js} +3 -3
- package/dist/{transcript-BXWZiMuC.js.map → transcript-CW02A4s9.js.map} +1 -1
- package/dist/{resources-DqdwihhT.js → turn-error-Dmv_Muj7.js} +15 -2
- package/dist/turn-error-Dmv_Muj7.js.map +1 -0
- package/dist/{types-BDo8ttQo.d.ts → types-BZaItiui.d.ts} +209 -28
- package/dist/types-Buut5yGe.js +29 -0
- package/dist/types-Buut5yGe.js.map +1 -0
- package/dist/types-C48zuZbT.d.ts +156 -0
- package/dist/types-CYz4dqAd.d.ts +1366 -0
- package/dist/{ui-DUiVPbQ2.js → ui-CJvbt2Cc.js} +15 -14
- package/dist/ui-CJvbt2Cc.js.map +1 -0
- package/dist/vertex-D4H9bKHD.d.ts +2 -0
- package/dist/{vertex-BEWxHoxH.js → vertex-DQQGhUn7.js} +2 -2
- package/dist/{vertex-BEWxHoxH.js.map → vertex-DQQGhUn7.js.map} +1 -1
- package/dist/vertex-DtWe0Xf6.js +1 -0
- package/dist/view/index.d.ts +142 -810
- package/dist/view/index.js +17 -12
- package/dist/view/report/index.d.ts +217 -0
- package/dist/view/report/index.js +4 -0
- package/dist/{views-CiZGh3ws.js → views-CtnOlABJ.js} +6 -6
- package/dist/{views-CiZGh3ws.js.map → views-CtnOlABJ.js.map} +1 -1
- package/dist/views-crppDW_B.js +2 -0
- package/dist/{vm-container-BVn7pXfD.js → vm-container-DwroMAXW.js} +13 -10
- package/dist/vm-container-DwroMAXW.js.map +1 -0
- package/dist/{vm-images-CNk-Zl2a.js → vm-images-y0BcVyv3.js} +19 -21
- package/dist/vm-images-y0BcVyv3.js.map +1 -0
- package/dist/wait-CixM6W0G.js +65 -0
- package/dist/wait-CixM6W0G.js.map +1 -0
- package/dist/{worker-entry-DmXQ2Znu.js → worker-entry-8KHmPzr4.js} +7 -7
- package/dist/{worker-entry-DmXQ2Znu.js.map → worker-entry-8KHmPzr4.js.map} +1 -1
- package/dist/workspace-JlgoGyR0.js +1 -0
- package/dist/{workspace-wwvwEHzP.js → workspace-xCQ6VDR-.js} +9 -8
- package/dist/workspace-xCQ6VDR-.js.map +1 -0
- package/package.json +33 -9
- package/skills/reeval/SKILL.md +2 -2
- package/skills/reeval/references/cli.md +1 -1
- package/skills/reeval/references/config.md +1 -1
- package/skills/reeval/references/environments.md +1 -1
- package/skills/setup/SKILL.md +1 -1
- package/dist/agents-BztvTkfM.js.map +0 -1
- package/dist/analysis-BnG_1x_O.js.map +0 -1
- package/dist/app-files-6NHzghym.js +0 -10
- package/dist/app-files-6NHzghym.js.map +0 -1
- package/dist/assemble-Doa1m-HW.js.map +0 -1
- package/dist/assemble-XrZskSNj.js +0 -1
- package/dist/attach-BH2HdwH_.js.map +0 -1
- package/dist/attempt-host-8mQV8Uee.js.map +0 -1
- package/dist/boat-DKRH0Gv7.js +0 -1
- package/dist/boat-DKjqo_ie.js.map +0 -1
- package/dist/boat-images-CRN_U_eG.js.map +0 -1
- package/dist/bundle-DU2aQVeq.js.map +0 -1
- package/dist/capture-BzjNv0qP.js +0 -1
- package/dist/cases-CM-MQRQp.js.map +0 -1
- package/dist/cases-FvhbM8M5.js +0 -1
- package/dist/cli-DMHOwdVr.js.map +0 -1
- package/dist/cli-spec-D0wuUaj9.js.map +0 -1
- package/dist/client-B41UK150.js.map +0 -1
- package/dist/command-8GIHYGZf.js +0 -298
- package/dist/command-8GIHYGZf.js.map +0 -1
- package/dist/command-CAmnWLFL.js.map +0 -1
- package/dist/command-CXxckfsq.js.map +0 -1
- package/dist/command-ao0NZ8h0.js.map +0 -1
- package/dist/commands-B3lnIPj5.js +0 -1
- package/dist/commands-ttChCsAy.js.map +0 -1
- package/dist/config-myJT1Qjc.js.map +0 -1
- package/dist/context-DnzR9TNK.js +0 -120
- package/dist/context-DnzR9TNK.js.map +0 -1
- package/dist/daytona-Dfhwtr9P.js.map +0 -1
- package/dist/daytona-image-CnG6G00x.js.map +0 -1
- package/dist/daytona-image-yJK3c4EV.js +0 -1
- package/dist/db-5H95sFii.js.map +0 -1
- package/dist/db-Bbg1n_2M.js +0 -2
- package/dist/debug-CF4p4Lp_.js +0 -1
- package/dist/debug-CIXzB4wK.js.map +0 -1
- package/dist/environment-C8pUCD_A.js.map +0 -1
- package/dist/evaluate-4kBVtwZZ.js.map +0 -1
- package/dist/files-BFnkOAhn.js +0 -1
- package/dist/files-BgQxoDlN.js +0 -1
- package/dist/files-CDfG8swM.js.map +0 -1
- package/dist/files-CSe4VaMj.js +0 -1
- package/dist/files-CX_mmvCc.js +0 -775
- package/dist/files-CX_mmvCc.js.map +0 -1
- package/dist/files-D6M7J3iX.js +0 -1
- package/dist/files-DgHeTdCZ.js.map +0 -1
- package/dist/gate-CgYdaprG.js +0 -49
- package/dist/gate-CgYdaprG.js.map +0 -1
- package/dist/harness-FcTyyPco.js.map +0 -1
- package/dist/harness-install-BfMBYDfP.js.map +0 -1
- package/dist/hosted-5Zz2peTR.js.map +0 -1
- package/dist/hosted-Co-waHe0.js +0 -131
- package/dist/hosted-Co-waHe0.js.map +0 -1
- package/dist/hosted-run-202QicZ-.js +0 -1
- package/dist/hosted-run-BfAPIMyT.js +0 -230
- package/dist/hosted-run-BfAPIMyT.js.map +0 -1
- package/dist/ids-a6cr243_.js.map +0 -1
- package/dist/init-BiL6B3S4.js +0 -1
- package/dist/inputs-load-M-BGe7gd.js +0 -1
- package/dist/judge-DwzqucLS.js.map +0 -1
- package/dist/kinds-DEKXOWVJ.js.map +0 -1
- package/dist/list-MK4L8tfW.js.map +0 -1
- package/dist/live-BkPNXVmN.js +0 -120
- package/dist/live-BkPNXVmN.js.map +0 -1
- package/dist/live-Dlq-S2yy.js.map +0 -1
- package/dist/live-gZtkBwuo.js +0 -1
- package/dist/machine-service-CZiXiVsS.js.map +0 -1
- package/dist/model-C6aoJLg_.js +0 -1
- package/dist/model-LoLnq9y1.js.map +0 -1
- package/dist/options-I9I3FwMl.js +0 -1
- package/dist/pairs-BGb134Vu.js +0 -230
- package/dist/pairs-BGb134Vu.js.map +0 -1
- package/dist/patch-C7yBztZj.js.map +0 -1
- package/dist/paths-Dwmi7L4X.js +0 -75
- package/dist/paths-Dwmi7L4X.js.map +0 -1
- package/dist/preflight-fyV5nnea.js +0 -1
- package/dist/prices-C0gKS_8G.js +0 -1
- package/dist/provider-DHT20Gl8.js +0 -1
- package/dist/query-C9OKU1Vz.js +0 -1
- package/dist/questions-boVPzGoC.js.map +0 -1
- package/dist/record-D1_D1KFY.js.map +0 -1
- package/dist/redact-t-L3VGy8.js.map +0 -1
- package/dist/remarks-DqriPKxm.js.map +0 -1
- package/dist/resolve-BdpO8BN1.js +0 -1
- package/dist/resolve-Bfo8sjd-.js.map +0 -1
- package/dist/resolve-D2RqmybX.js.map +0 -1
- package/dist/resources-DqdwihhT.js.map +0 -1
- package/dist/runtime-C5fKSFCz.js +0 -457
- package/dist/runtime-C5fKSFCz.js.map +0 -1
- package/dist/server-BcLg9WzE.js +0 -284
- package/dist/server-BcLg9WzE.js.map +0 -1
- package/dist/server-Buav2-CG.js +0 -1
- package/dist/services-BG8xuU82.js.map +0 -1
- package/dist/shared-CHvO7eLi.d.ts +0 -1
- package/dist/shared-DtZwIAfy.js.map +0 -1
- package/dist/slug-IHh3f770.js.map +0 -1
- package/dist/source-CrXM19HD.js.map +0 -1
- package/dist/source-dvxkqYvk.js +0 -1
- package/dist/src-5dCrqbFd.js +0 -2
- package/dist/src-C5rrQlt8.js.map +0 -1
- package/dist/src-DEbNp-rx.js +0 -1
- package/dist/src-DrLbG1HM.js +0 -2
- package/dist/ssh-pool-BorCizYw.js +0 -1
- package/dist/ssh-pool-Dvfx5GuV.js.map +0 -1
- package/dist/start-COB07NpH.js.map +0 -1
- package/dist/state-B319V1p8.js.map +0 -1
- package/dist/state-C5oBvmoZ.js +0 -1
- package/dist/suite-BwJGOhCZ.js +0 -2
- package/dist/suite-D1tgXZxM.js.map +0 -1
- package/dist/summary-ClLns_wv.js +0 -169
- package/dist/summary-ClLns_wv.js.map +0 -1
- package/dist/tart-DMppfbEm.js.map +0 -1
- package/dist/tart-bake-Cva_qdir.js.map +0 -1
- package/dist/tart-cli-LzDud2Z6.js.map +0 -1
- package/dist/tart-hosts-CTRY5kQ2.js.map +0 -1
- package/dist/types-CbcnJxCy.js +0 -15
- package/dist/types-CbcnJxCy.js.map +0 -1
- package/dist/ui-DUiVPbQ2.js.map +0 -1
- package/dist/vertex-C7PI-WPc.d.ts +0 -3
- package/dist/vertex-DXfcXYKn.js +0 -1
- package/dist/views-B0qZR5Ef.js +0 -2
- package/dist/vm-container-BVn7pXfD.js.map +0 -1
- package/dist/vm-images-CNk-Zl2a.js.map +0 -1
- package/dist/workspace-CRgFdmgW.js +0 -1
- package/dist/workspace-wwvwEHzP.js.map +0 -1
package/dist/view/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import { $n as
|
|
1
|
+
import { $n as UpstreamCount, Bn as FamilyRate, Cr as formatCount, Dr as splitOf, Er as splitLabel, Fn as CellRecord, Gn as HistoryEntry, H as CostUsd, Hn as FamilyVerdict, In as CheckTally, Jn as RunDebug, Jt as Outcome, Kn as JudgmentRecord, Ln as EvalEntry, Nn as ArtifactRef, Or as tokenValues, Pn as AttemptRecord, Qn as StackSummary, Rn as FamilyLift, Sr as filesChanged, Tr as formatSeconds, Un as FileRef, V as CostTotal, Vn as FamilyRecord, Wn as HistoryCell, Xn as RunRecord, Yn as RunInvocation, Zn as RunScope, _r as effortLabel, ar as EffortMetric, b as CapturePage, br as effortSummary, cr as TOKENS_UNREPORTED, dr as cellContrast, er as CellEffort, fn as SandboxWarning, fr as cellEffort, g as CaptureFailedRequest, gr as effortChangeLabel, h as CaptureConsoleMessage, hr as directional, ir as EffortContrast, lr as VERDICT_TEXT, mr as contrastEffort, nr as EFFORT_ALPHA, or as EffortSample, ot as ErrorKind, pr as cellSamples, rr as EFFORT_MIN_RUNS, sr as EffortVerdict, tr as DIRECTIONAL_MIN, ur as bothUnderATenth, vr as effortOf, wr as formatRatio, x as CapturePageFile, xn as TranscriptEvent, xr as effortVerdict, yr as effortSample } from "../types-BZaItiui.js";
|
|
2
|
+
import { $ as Remark, $t as checkLabels, A as IntegrityKind, An as AnalysisClaim, At as formatUsage, B as Quote, Bn as DIGEST_VERSION, Bt as reasoningLabel, C as ExecutiveRollup, Cn as stackChanges, Ct as evalMarkOutcome, D as Figure, Dn as AnalysisAction, Dt as findEval, E as Fairness, En as ActionKind, Et as findAttempt, F as MeasureKey, Fn as AnalysisNote, Ft as isUngatedMiss, G as RunHeader, Gt as shortSha, H as RegressionChanges, Hn as isSuperseded, Ht as runOutcome, I as MeasureVsBaseline, In as AnalysisScope, It as languageLabel, J as Spread, Jt as transcriptItems, K as RunReport, Kt as splitEvalId, L as PairwiseRow, Ln as AnalysisValidation, Lt as notRunYet, M as JudgeRow, Mn as AnalysisDoc, Mt as hasUngatedCells, N as JudgmentBreakdown, Nn as AnalysisEntry, Nt as isMatrix, O as HeadlinePrimary, On as AnalysisAnomaly, Ot as formatDuration, P as MeasureCells, Pn as AnalysisIndex, Pt as isNotGatedPass, Q as REMARK_KINDS, Qt as cellUsage, R as ProvenanceState, Rn as AnomalyKind, Rt as parseTranscript, S as EvalShape, Sn as sourceChanges, St as evalHistory, T as FailureClusters, Tn as ANALYSIS_PROMPT_VERSION, Tt as failedJudgeLabels, U as ReportClaim, Un as pinnedAnalysis, Ut as runRows, V as Rate, Vn as Level, Vt as runLead, W as RosterRow, Wn as sortAnalyses, Wt as runtimeLabel, X as VariantComparison, Xt as JudgeKeyRate, Y as TaskResults, Yt as CollapsedJudge, Z as REMARKS_FILE, Zt as cellJudgeRates, _ as Delta, _n as StackSource, _t as answerRows, a as AttemptIndexRow, an as judgeLabel, at as AttemptAnswerItem, b as EvalHeadline, bn as compare, bt as compareView, c as BlockMarkdown, cn as summaryJudgeRates, ct as CellRow, d as CaptureShot, dn as CompareCell, dt as RunRow, en as checkTries, et as RunRemarks, f as ChangeRow, fn as CompareFamilyLift, ft as Tally, g as Count, gn as StackChange, gt as activityLabel, h as ContrastRow, hn as CompareStatus, ht as TreeNode, i as AssertionRow, in as commandLine, it as AnswerRow, j as Interval, jn as AnalysisClaimRole, jt as harnessLabel, k as IntegrityIssue, kn as AnalysisBody, kt as formatPercent, l as BreakdownRow, ln as triesNote, lt as CompareViewRow, m as ChoiceBreakdown, mn as CompareRow, mt as TranscriptLine, n as ArmRef, nn as collapsedJudgeLabel, nt as attemptFacts, o as AttemptRef, on as judgeRatesLabel, ot as AttemptAnswerRow, p as CheckCell, pn as CompareFamilyRate, pt as TranscriptItem, q as RunTally, qt as transcriptByLine, r as ArmRow, rn as collapsedJudges, rt as AnswerRate, s as BehaviorRow, sn as labeledJudgments, st as COMPARE_ORDER, t as Arm, tn as checksByLabel, tt as AttemptFacts, u as CaptureGallery, un as uniqueJudgeLabels, ut as EvalHistoryPoint, v as DiscoveryRate, vn as addedRow, vt as attemptAnswerRows, w as FailureCluster, wn as whyChanged, wt as evalTree, x as EvalSection, xn as newRows, xt as defaultAttempt, y as EffortPivotRow, yn as cellsChanged, yt as cellRows, z as QualitativeAnswers, zn as Confidence, zt as parseTranscriptLine } from "../types-CYz4dqAd.js";
|
|
3
|
+
import { $ as REPORT_VERSION, A as attemptIds, B as pctDelta, C as discoverySourceLabel, D as AttemptIdMap, E as verdictLabel, F as diffLinesText, G as tokensFigure, H as ptsInterval, I as durationFigure, J as ReportLinks, K as tokensText, L as figure, M as selectedEvals, N as countFigure, O as attemptAnchor, P as countOf, Q as EvalSummary, R as judgeScoreText, S as contrastMeasureLabel, T as passRateOutcomeFromMatchers, U as rateOf, V as ptsDelta, W as spreadOf, X as buildRunReport, Y as RunReportOptions, Z as EvalDetail, _ as MEASURE_LABEL, a as ReportPolicy, b as VERDICT_LABEL, c as captureHref, d as questionHref, et as RunReportIndex, f as recordHref, g as DISCOVERY_SOURCE_LABEL, h as COMPARE_STATUS_LABEL, i as REPORT_POLICY, j as idOf, k as attemptAnchorId, l as evalAnchorHref, m as worktreePatchLink, n as stateRows, nt as joinRunReport, o as attemptHref, p as withBase, q as usdFigure, r as warnings, rt as splitRunReport, s as attemptRef, t as provenanceState, tt as SplitRunReport, u as gfmAnchor, v as PASS_RATE_OUTCOME_LABEL, w as measurePhrase, x as compareStatusLabel, y as PassRateOutcome, z as medianWithDelta } from "../provenance-Cylt_YxT.js";
|
|
2
4
|
/** `pass@1 2/3 · 67% (95% 21…94)`, or `pass@1 67% over 3 cases (95% 30…100)` at several runs. */
|
|
3
5
|
export declare function familyRateLabel(rate: FamilyRate): string;
|
|
4
6
|
/**
|
|
@@ -29,293 +31,139 @@ export declare function templateName(template: string): string;
|
|
|
29
31
|
export declare function caseEntries(e: EvalEntry): EvalEntry[];
|
|
30
32
|
/** An eval with `cases` as `foldCases` read it: its cells' rates and lifts, as a family. */
|
|
31
33
|
export declare function familyOf(e: EvalEntry): FamilyRecord | undefined;
|
|
32
|
-
/**
|
|
33
|
-
export declare const
|
|
34
|
-
|
|
35
|
-
export declare const
|
|
36
|
-
export
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
export type
|
|
57
|
-
export type
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
overall: Confidence;
|
|
80
|
-
why: string;
|
|
81
|
-
};
|
|
82
|
-
};
|
|
83
|
-
export type AnalysisValidation = {
|
|
84
|
-
/** Citations checked, and those that named nothing in the digest. */
|
|
85
|
-
citations: number;
|
|
86
|
-
invalid: string[];
|
|
87
|
-
/** Claims, anomalies and actions dropped for a bad citation, invented quote, wrong judge score, or rejected verify. */
|
|
88
|
-
dropped: number;
|
|
89
|
-
commands: number;
|
|
90
|
-
invalidCommands: string[];
|
|
91
|
-
unverifiedNumbers: number;
|
|
92
|
-
};
|
|
93
|
-
export type AnalysisScope = {
|
|
94
|
-
kind: 'run';
|
|
95
|
-
head: string;
|
|
96
|
-
/** Resolved once, so a nearer ancestor recorded later doesn't change what this analyzed. */
|
|
97
|
-
base: string | null;
|
|
98
|
-
/** An eval filter (substring of the eval id), or null for every eval. */
|
|
99
|
-
evals: string | null;
|
|
100
|
-
};
|
|
101
|
-
export type AnalysisDoc = {
|
|
102
|
-
kind: 'reeval-analysis';
|
|
103
|
-
schemaVersion: 1;
|
|
104
|
-
scope: AnalysisScope;
|
|
105
|
-
/** The run the analysis belongs to: the scope's head. */
|
|
106
|
-
anchor: string;
|
|
107
|
-
inputs: {
|
|
108
|
-
records: Array<{
|
|
109
|
-
runId: string;
|
|
110
|
-
sha256: string;
|
|
111
|
-
}>;
|
|
112
|
-
digestVersion: number;
|
|
113
|
-
digestHash: string;
|
|
114
|
-
};
|
|
115
|
-
promptVersion: string;
|
|
116
|
-
model: string;
|
|
117
|
-
resolvedModel: string | null;
|
|
118
|
-
route: 'direct' | 'gateway' | 'openrouter' | 'vertex' | 'fake';
|
|
119
|
-
usage: {
|
|
120
|
-
inputTokens: number | null;
|
|
121
|
-
outputTokens: number | null;
|
|
122
|
-
};
|
|
123
|
-
latencyMs: number;
|
|
124
|
-
createdAt: string;
|
|
125
|
-
author: string;
|
|
126
|
-
validation: AnalysisValidation;
|
|
127
|
-
analysis: AnalysisBody;
|
|
128
|
-
/** Short attempt ids used in citations, and where each attempt is. */
|
|
129
|
-
attempts: Record<string, {
|
|
130
|
-
runId: string;
|
|
131
|
-
evalSlug: string;
|
|
132
|
-
attempt: string;
|
|
133
|
-
}>;
|
|
134
|
-
/** Each cited fact or evidence id, summarized in one line. */
|
|
135
|
-
cited: Record<string, string>;
|
|
136
|
-
/**
|
|
137
|
-
* Why the analysis failed; its `analysis` is then empty. `reeval analyze` treats a cached one as
|
|
138
|
-
* a miss and calls the model again.
|
|
139
|
-
*/
|
|
140
|
-
error?: string;
|
|
34
|
+
/** The routes' version, in their prefix: a breaking change to a route or shape bumps it. */
|
|
35
|
+
export declare const STUDIO_API_VERSION = 1;
|
|
36
|
+
export declare const STUDIO_API_PREFIX = "/api/v1";
|
|
37
|
+
export declare const RUN_ID: RegExp;
|
|
38
|
+
export declare const RUN_LIST_LIMIT: {
|
|
39
|
+
default: number;
|
|
40
|
+
max: number;
|
|
41
|
+
};
|
|
42
|
+
export declare const TRANSCRIPT_LIMIT: {
|
|
43
|
+
default: number;
|
|
44
|
+
max: number;
|
|
45
|
+
};
|
|
46
|
+
/** The route names, each with what it answers. */
|
|
47
|
+
export type StudioRoutes = {
|
|
48
|
+
pair: PairResponse;
|
|
49
|
+
handshake: Handshake;
|
|
50
|
+
runs: RunList;
|
|
51
|
+
report: RunReportIndex;
|
|
52
|
+
eval: EvalDetail;
|
|
53
|
+
artifact: Uint8Array | TranscriptPage;
|
|
54
|
+
};
|
|
55
|
+
export type StudioRoute = keyof StudioRoutes;
|
|
56
|
+
export declare const STUDIO_ROUTES: readonly StudioRoute[];
|
|
57
|
+
/** An opaque position in a list: pass a page's `next` back as `cursor` for the page after it. */
|
|
58
|
+
export type Cursor = string;
|
|
59
|
+
export type Page<T> = {
|
|
60
|
+
items: T[];
|
|
61
|
+
next: Cursor | null;
|
|
62
|
+
};
|
|
63
|
+
export type PairRequest = {
|
|
64
|
+
token: string;
|
|
65
|
+
};
|
|
66
|
+
export type PairResponse = {
|
|
67
|
+
session: string;
|
|
68
|
+
handshake: Handshake;
|
|
69
|
+
};
|
|
70
|
+
/** What a server of runs is and serves, so a client knows which routes and report it can read. */
|
|
71
|
+
export type Handshake = {
|
|
72
|
+
api: typeof STUDIO_API_VERSION;
|
|
73
|
+
/** The newest `RunReport.reportVersion` the server answers with. */
|
|
74
|
+
reportVersion: typeof REPORT_VERSION;
|
|
75
|
+
/** The reeval that serves the runs. */
|
|
76
|
+
reeval: string;
|
|
77
|
+
project: string;
|
|
78
|
+
routes: StudioRoute[];
|
|
79
|
+
/** The run the user asked to open, when they named one, else the newest. */
|
|
80
|
+
focus: string | null;
|
|
141
81
|
};
|
|
82
|
+
export type RunListItem = HistoryEntry;
|
|
83
|
+
export type RunList = Page<RunListItem>;
|
|
142
84
|
/**
|
|
143
|
-
* A
|
|
144
|
-
*
|
|
85
|
+
* A page of a `.jsonl` artifact's transcript lines. `next`, the cursor and `totalLines` count the
|
|
86
|
+
* lines that parse as transcript lines; a line that isn't JSON, or has no `type`, is skipped.
|
|
145
87
|
*/
|
|
146
|
-
export type
|
|
147
|
-
|
|
148
|
-
schemaVersion: 1;
|
|
149
|
-
runId: string;
|
|
150
|
-
evalId: string;
|
|
151
|
-
/** The harness id, as in the cell key. */
|
|
152
|
-
harness: string;
|
|
153
|
-
model: string;
|
|
154
|
-
resolvedModel: string | null;
|
|
155
|
-
createdAt: string;
|
|
156
|
-
/** The attempt ids (`<cell key>#<n>`) the note covers. */
|
|
157
|
-
attempts: string[];
|
|
158
|
-
/** Empty when the call failed. */
|
|
159
|
-
text: string;
|
|
160
|
-
/** Why the call failed; the run's outcomes don't change. */
|
|
161
|
-
error?: string;
|
|
162
|
-
usage: {
|
|
163
|
-
inputTokens: number | null;
|
|
164
|
-
outputTokens: number | null;
|
|
165
|
-
};
|
|
166
|
-
};
|
|
167
|
-
/** One analysis file the studio can show: saved in the run's folder, or in the local cache. */
|
|
168
|
-
export type AnalysisEntry = {
|
|
169
|
-
file: string;
|
|
170
|
-
source: 'saved' | 'cached';
|
|
171
|
-
doc: AnalysisDoc;
|
|
88
|
+
export type TranscriptPage = Page<TranscriptLine> & {
|
|
89
|
+
totalLines: number;
|
|
172
90
|
};
|
|
173
|
-
export
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
91
|
+
export declare const studioPaths: {
|
|
92
|
+
pair: () => string;
|
|
93
|
+
handshake: () => string;
|
|
94
|
+
runs: () => string;
|
|
95
|
+
report: (runId: string) => string;
|
|
96
|
+
eval: (runId: string, evalId: string) => string;
|
|
97
|
+
artifact: (runId: string, path: string) => string;
|
|
178
98
|
};
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
export declare function pinnedAnalysis(entries: readonly AnalysisEntry[]): AnalysisEntry | undefined;
|
|
182
|
-
/** Newest first, saved before cached at the same time. */
|
|
183
|
-
export declare function sortAnalyses(entries: AnalysisEntry[]): AnalysisEntry[];
|
|
184
|
-
/** Superseded by a newer prompt or digest version: the studio says so. */
|
|
185
|
-
export declare const isSuperseded: (doc: AnalysisDoc) => boolean;
|
|
186
|
-
/** `GET /missing.js → 404`, or `→ net::ERR_…` when no response came. */
|
|
187
|
-
export declare const failedRequestLine: (r: CaptureFailedRequest) => string;
|
|
188
|
-
/** A capture's page facts as a line each: its page errors, console messages, then failed requests. */
|
|
189
|
-
export declare function pageLines(page: CapturePage): string[];
|
|
190
|
-
export type CompareStatus = 'regressed' | 'fixed' | 'same' | 'gate' | 'drift' | 'added' | 'removed' | 'infra';
|
|
191
|
-
export type CompareRow = {
|
|
192
|
-
id: string;
|
|
193
|
-
status: CompareStatus;
|
|
194
|
-
base: Outcome | null;
|
|
195
|
-
head: Outcome | null;
|
|
196
|
-
/**
|
|
197
|
-
* For `drift`: what changed, e.g. `body changed`, `revision 2 → 3` or `harness config changed`. On
|
|
198
|
-
* any other row, `gate changed` when the cells' gate did.
|
|
199
|
-
*/
|
|
200
|
-
changes?: string[];
|
|
201
|
-
/**
|
|
202
|
-
* When the cells' gate changed but nothing else did: each side's gate, and base's outcome with
|
|
203
|
-
* its recorded attempts scored under head's gate. `status` compares that with head, so a flip
|
|
204
|
-
* the gate alone explains is `gate`, not `regressed` or `fixed`.
|
|
205
|
-
*/
|
|
206
|
-
gate?: {
|
|
207
|
-
base: string;
|
|
208
|
-
head: string;
|
|
209
|
-
rescored: Outcome;
|
|
210
|
-
};
|
|
211
|
-
/**
|
|
212
|
-
* When the runs ran different cells of the eval: the cells only head or only base has. Then
|
|
213
|
-
* `status`, `base` and `head` are over the cells both have.
|
|
214
|
-
*/
|
|
215
|
-
cells?: {
|
|
216
|
-
added: string[];
|
|
217
|
-
removed: string[];
|
|
218
|
-
};
|
|
219
|
-
/** Cells in both runs, with each side's lift (variants) and effort. */
|
|
220
|
-
perCell?: CompareCell[];
|
|
221
|
-
/**
|
|
222
|
-
* The sides whose `fail` is only attempts the run's interrupt canceled, which `reeval show`
|
|
223
|
-
* reads as `canceled`. Such an eval is never `regressed` or `fixed`: it's `drift`.
|
|
224
|
-
*/
|
|
225
|
-
canceled?: Array<'base' | 'head'>;
|
|
226
|
-
/** An eval with `cases`: each case's row, its `id` the case key, in head's order then base's. */
|
|
227
|
-
cases?: CompareRow[];
|
|
228
|
-
/**
|
|
229
|
-
* An eval with `cases` when the runs ran different cases: the case keys only head or only base
|
|
230
|
-
* has. Then `base` and `head` are over the cases both have.
|
|
231
|
-
*/
|
|
232
|
-
caseScope?: {
|
|
233
|
-
added: string[];
|
|
234
|
-
removed: string[];
|
|
235
|
-
};
|
|
236
|
-
/** An eval with `cases`: each variant's lift paired by case on each side, null where that run has none. */
|
|
237
|
-
lifts?: CompareFamilyLift[];
|
|
238
|
-
/** An eval with `cases`: each cell's rate over the cases that a `passRate` gates, on each side. */
|
|
239
|
-
rates?: CompareFamilyRate[];
|
|
240
|
-
/**
|
|
241
|
-
* `family` on a case row whose cell's rate over the cases gates it (`CaseRecord.verdict`):
|
|
242
|
-
* `base` and `head` are what it did, and the eval's row is the verdict, so
|
|
243
|
-
* `--fail-on-regression` leaves it to that row.
|
|
244
|
-
*/
|
|
245
|
-
verdict?: 'family';
|
|
246
|
-
};
|
|
247
|
-
export type CompareFamilyLift = {
|
|
248
|
-
key: string;
|
|
249
|
-
vs: string;
|
|
250
|
-
base: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
|
|
251
|
-
head: Omit<FamilyLiftSummary, 'variant' | 'vs'> | null;
|
|
252
|
-
};
|
|
253
|
-
export type CompareFamilyRate = {
|
|
254
|
-
key: string;
|
|
255
|
-
base: {
|
|
256
|
-
rate: number;
|
|
257
|
-
passRate: number;
|
|
258
|
-
outcome: 'pass' | 'fail';
|
|
259
|
-
} | null;
|
|
260
|
-
head: {
|
|
261
|
-
rate: number;
|
|
262
|
-
passRate: number;
|
|
263
|
-
outcome: 'pass' | 'fail';
|
|
264
|
-
} | null;
|
|
265
|
-
};
|
|
266
|
-
export type CompareCell = {
|
|
267
|
-
key: string;
|
|
268
|
-
base: Pick<HistoryCell, 'lift' | 'effort' | 'pass' | 'fail'>;
|
|
269
|
-
head: Pick<HistoryCell, 'lift' | 'effort' | 'pass' | 'fail'>;
|
|
270
|
-
};
|
|
271
|
-
export type StackChange = {
|
|
272
|
-
field: keyof StackSummary;
|
|
273
|
-
label: string;
|
|
99
|
+
/** Where a run's report links point: its files at the artifacts route, its record beside them. */
|
|
100
|
+
export declare const reportLinks: (runId: string) => {
|
|
274
101
|
base: string;
|
|
275
|
-
|
|
276
|
-
/** Not covered by eval fingerprints, so outcome flips across it are drift, not regressions. */
|
|
277
|
-
outsideFingerprint: boolean;
|
|
102
|
+
record: string;
|
|
278
103
|
};
|
|
279
|
-
type
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
104
|
+
export type StudioTarget = {
|
|
105
|
+
route: 'pair';
|
|
106
|
+
} | {
|
|
107
|
+
route: 'handshake';
|
|
108
|
+
} | {
|
|
109
|
+
route: 'runs';
|
|
110
|
+
} | {
|
|
111
|
+
route: 'report';
|
|
112
|
+
runId: string;
|
|
113
|
+
} | {
|
|
114
|
+
route: 'eval';
|
|
115
|
+
runId: string;
|
|
116
|
+
evalId: string;
|
|
117
|
+
} | {
|
|
118
|
+
route: 'artifact';
|
|
119
|
+
runId: string;
|
|
120
|
+
segments: string[];
|
|
286
121
|
};
|
|
287
122
|
/**
|
|
288
|
-
* What
|
|
289
|
-
*
|
|
290
|
-
*
|
|
123
|
+
* What a request's pathname (as sent, still percent-encoded) names, or null. Run ids and
|
|
124
|
+
* artifact segments must match an allowlist, so `.`, `..`, empty segments, backslashes and NUL
|
|
125
|
+
* never get through; an eval id is decoded once and may hold any character but NUL.
|
|
291
126
|
*/
|
|
292
|
-
export declare function
|
|
127
|
+
export declare function parseStudioPath(pathname: string): StudioTarget | null;
|
|
128
|
+
/** A list's `limit` query value, clamped to the route's maximum; the default when absent or bad. */
|
|
129
|
+
export declare function pageLimit(raw: string | null | undefined, bounds: {
|
|
130
|
+
default: number;
|
|
131
|
+
max: number;
|
|
132
|
+
}): number;
|
|
293
133
|
/**
|
|
294
|
-
*
|
|
295
|
-
*
|
|
134
|
+
* One page of runs, newest first, from every header oldest first as the history lists them. The
|
|
135
|
+
* cursor is the last run id of the page before, so a run recorded meanwhile never shifts a page.
|
|
296
136
|
*/
|
|
297
|
-
export declare function
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
137
|
+
export declare function pageRuns(oldestFirst: readonly RunListItem[], query: {
|
|
138
|
+
cursor?: string | null;
|
|
139
|
+
limit?: string | null;
|
|
140
|
+
}): RunList;
|
|
141
|
+
/** Where a `.jsonl` page starts and how many lines it holds, as `transcriptQuery` reads them. */
|
|
142
|
+
export type TranscriptRange = {
|
|
143
|
+
start: number;
|
|
144
|
+
limit: number;
|
|
301
145
|
};
|
|
302
|
-
type Comparable = Pick<HistoryEntry, 'stack'> & {
|
|
303
|
-
evals: ComparableEval[];
|
|
304
|
-
};
|
|
305
|
-
/** Which source files were added, removed or changed. */
|
|
306
|
-
export declare function sourceChanges(before: ComparableEval['sources'], after: ComparableEval['sources']): string[];
|
|
307
146
|
/**
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
147
|
+
* A `.jsonl` page's query, read strictly: the cursor is a whole number (0 when absent), the limit
|
|
148
|
+
* a positive whole number clamped to `TRANSCRIPT_LIMIT.max` (the default when absent), neither
|
|
149
|
+
* with a leading zero. Anything else is the sentence saying why, which the server answers with a
|
|
150
|
+
* 400.
|
|
311
151
|
*/
|
|
312
|
-
export declare function
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
152
|
+
export declare function transcriptQuery(query: {
|
|
153
|
+
cursor?: string | null;
|
|
154
|
+
limit?: string | null;
|
|
155
|
+
}): TranscriptRange | string;
|
|
156
|
+
/**
|
|
157
|
+
* One page of a `.jsonl` file's lines, each read as a transcript line (`parseTranscript`), from
|
|
158
|
+
* the line `range.start` counts to.
|
|
159
|
+
*/
|
|
160
|
+
export declare function pageTranscript(transcript: string, range: TranscriptRange): TranscriptPage;
|
|
161
|
+
/** Artifacts are untrusted agent output: JSON stays JSON, PNGs are images, else plain text. */
|
|
162
|
+
export declare const artifactType: (name: string) => string;
|
|
163
|
+
/** `GET /missing.js → 404`, or `→ net::ERR_…` when no response came. */
|
|
164
|
+
export declare const failedRequestLine: (r: CaptureFailedRequest) => string;
|
|
165
|
+
/** A capture's page facts as a line each: its page errors, console messages, then failed requests. */
|
|
166
|
+
export declare function pageLines(page: CapturePage): string[];
|
|
319
167
|
/** The parts' costs added up; undefined when none has one. `used` parts with no cost are `unpriced`. */
|
|
320
168
|
export declare function sumCosts(parts: {
|
|
321
169
|
cost?: CostUsd | CostTotal;
|
|
@@ -400,14 +248,6 @@ export type DebugView = {
|
|
|
400
248
|
/** Whether `trace.ndjson` was kept. */
|
|
401
249
|
trace: boolean;
|
|
402
250
|
};
|
|
403
|
-
/**
|
|
404
|
-
* The studio server's debug API (`reeval studio` only): an attempt's DebugView (null when it has no
|
|
405
|
-
* local data), and its raw trace.
|
|
406
|
-
*/
|
|
407
|
-
export declare const debugApiPath: (runId: string, attemptKey: string) => string;
|
|
408
|
-
export declare const traceApiPath: (runId: string, attemptKey: string) => string;
|
|
409
|
-
/** A snapshot from the run's directory in `.reeval/runs`, as a tar.gz (`reeval studio` only). */
|
|
410
|
-
export declare const snapshotApiPath: (runId: string, path: string) => string;
|
|
411
251
|
/** `12:50:01`, or `12:50:01.234` with milliseconds, in UTC like every reeval timestamp. */
|
|
412
252
|
export declare const debugTime: (at: number, ms: boolean) => string;
|
|
413
253
|
/** A line of debug.log: `2026-09-26T12:50:01.234Z plan text`. */
|
|
@@ -424,91 +264,6 @@ export declare function formatBytes(n: number): string;
|
|
|
424
264
|
export declare function requestLine(r: Omit<DebugRequest, 'at'>): string;
|
|
425
265
|
/** The derived header of one run: what listings, indexes and comparisons need without the record. */
|
|
426
266
|
export declare function runHeader(record: RunRecord, recorded: boolean): HistoryEntry;
|
|
427
|
-
/** Short name for a judge question, e.g. `login` from `login?`. */
|
|
428
|
-
export declare function judgeLabel(question: string): string;
|
|
429
|
-
/**
|
|
430
|
-
* One short label per key. Shared prefixes take the distinctive tail so
|
|
431
|
-
* `Does this change…` does not collapse several checks; a tail that still
|
|
432
|
-
* collides falls back to the key.
|
|
433
|
-
*/
|
|
434
|
-
export declare function uniqueJudgeLabels(items: readonly {
|
|
435
|
-
key: string;
|
|
436
|
-
question: string;
|
|
437
|
-
}[]): Map<string, string>;
|
|
438
|
-
export type JudgeKeyRate = {
|
|
439
|
-
key: string;
|
|
440
|
-
/** The `expect` label when the judge had one, else a short name from its question. */
|
|
441
|
-
label: string;
|
|
442
|
-
/** Named by an `expect` label, so the cell's checks carry it too. */
|
|
443
|
-
labelled: boolean;
|
|
444
|
-
question: string;
|
|
445
|
-
pass: number;
|
|
446
|
-
counted: number;
|
|
447
|
-
/** Compact lift vs the paired control, when both sides counted an attempt. */
|
|
448
|
-
lift: Pick<Lift, 'points' | 'control' | 'variant' | 'withinNoise'> | null;
|
|
449
|
-
};
|
|
450
|
-
/** Per-key pass/counted for one cell's counted attempts, with lift on a variant when both sides counted an attempt. */
|
|
451
|
-
export declare function cellJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cellKey: string): JudgeKeyRate[];
|
|
452
|
-
/**
|
|
453
|
-
* The judge rates a cell's summary line adds to its checks: a labeled judge whose label is one
|
|
454
|
-
* of the cell's checks against its control is already there.
|
|
455
|
-
*/
|
|
456
|
-
export declare function summaryJudgeRates(e: Pick<EvalEntry, 'attempts' | 'cells'>, cell: Pick<CellRecord, 'key' | 'checks'>): JudgeKeyRate[];
|
|
457
|
-
/** `login 4/5 · checkout 0/5`, with `(+80 pts)` on a key that has lift. */
|
|
458
|
-
export declare function judgeRatesLabel(rates: readonly JudgeKeyRate[]): string;
|
|
459
|
-
export type CollapsedJudge = {
|
|
460
|
-
verdict: 'pass' | 'fail';
|
|
461
|
-
score: number;
|
|
462
|
-
threshold: number;
|
|
463
|
-
label: string;
|
|
464
|
-
key: string;
|
|
465
|
-
};
|
|
466
|
-
/** Each key, or the lowest score plus how many judges there were. */
|
|
467
|
-
export declare function collapsedJudges(judgments: readonly JudgmentRecord[]): {
|
|
468
|
-
shown: CollapsedJudge[];
|
|
469
|
-
extra: number;
|
|
470
|
-
};
|
|
471
|
-
/** `fail 0.78/0.80`, or `fail 0.33/0.80 · 3 judges` when there are many. */
|
|
472
|
-
export declare function collapsedJudgeLabel(judgments: readonly JudgmentRecord[]): string | null;
|
|
473
|
-
/**
|
|
474
|
-
* Each judgment's `expect` label, by position: the attempt's `toPass` assertions (less those on a
|
|
475
|
-
* failed judge call, which recorded no judgment) in call order, when they are exactly as many as
|
|
476
|
-
* its judgments. `check` is that assertion's label, the key of its check; a judgment no label
|
|
477
|
-
* reaches is named by its question.
|
|
478
|
-
*/
|
|
479
|
-
export declare function labeledJudgments(a: Pick<AttemptRecord, 'judgements' | 'assertions'>): Array<{
|
|
480
|
-
judgment: JudgmentRecord;
|
|
481
|
-
label: string;
|
|
482
|
-
labelled: boolean;
|
|
483
|
-
check?: string;
|
|
484
|
-
}>;
|
|
485
|
-
/** Sum of counted attempts' usage; null when none of them reported any. */
|
|
486
|
-
export declare function cellUsage(e: Pick<EvalEntry, 'attempts'>, cellKey: string): TokenUsage | null;
|
|
487
|
-
/**
|
|
488
|
-
* Each `c.exec` check's `expect` label, by position: the attempt's `toSucceed` assertions in call
|
|
489
|
-
* order, when they are exactly as many as its checks; undefined where no label reaches.
|
|
490
|
-
*/
|
|
491
|
-
export declare function checkLabels(a: Pick<AttemptRecord, 'checks' | 'assertions'>): Array<string | undefined>;
|
|
492
|
-
/**
|
|
493
|
-
* Each `toSucceed` label's `c.exec` check, by position, when the attempt's `toSucceed` assertions
|
|
494
|
-
* are exactly as many as its checks; empty otherwise.
|
|
495
|
-
*/
|
|
496
|
-
export declare function checksByLabel<C>(a: {
|
|
497
|
-
checks: readonly C[];
|
|
498
|
-
} & Pick<AttemptRecord, 'assertions'>): Map<string, C>;
|
|
499
|
-
/**
|
|
500
|
-
* How a `c.exec` given `retries` ended: `passed on try 2 of 3`, `failed all 3 tries`, or null
|
|
501
|
-
* when it had none or passed on its first try.
|
|
502
|
-
*/
|
|
503
|
-
export declare function triesNote(exitCode: number, tried: number, of: number): string | null;
|
|
504
|
-
/** {@link triesNote} for a recorded check. */
|
|
505
|
-
export declare const checkTries: (c: {
|
|
506
|
-
exitCode: number;
|
|
507
|
-
retries?: number;
|
|
508
|
-
tries?: readonly unknown[];
|
|
509
|
-
}) => string | null;
|
|
510
|
-
/** A check's command on one line: the first non-empty line, clipped; its log has the rest. */
|
|
511
|
-
export declare function commandLine(command: string, max?: number): string;
|
|
512
267
|
/** The progress file's name inside the run's staging directory. */
|
|
513
268
|
export declare const LIVE_FILE = "live.json";
|
|
514
269
|
export declare const LIVE_SCHEMA_VERSION = 1;
|
|
@@ -566,8 +321,6 @@ export type LiveRun = Omit<LiveProgress, 'schemaVersion' | 'pid' | 'host' | 'hea
|
|
|
566
321
|
export type LiveIndex = {
|
|
567
322
|
runs: LiveRun[];
|
|
568
323
|
};
|
|
569
|
-
export declare const liveApiPath = "api/live";
|
|
570
|
-
export declare const liveTranscriptPath: (runId: string, attemptKey: string) => string;
|
|
571
324
|
export declare const FINISHED: readonly LiveStatus[];
|
|
572
325
|
export declare const isFinished: (status: LiveStatus) => boolean;
|
|
573
326
|
export declare function liveCounts(attempts: readonly Pick<LiveAttempt, 'status'>[]): LiveCounts;
|
|
@@ -578,341 +331,6 @@ export declare function readableInvocation(v: unknown): RunInvocation | undefine
|
|
|
578
331
|
* corrupt, or from a newer reeval. Counts are recomputed from the attempts, never trusted.
|
|
579
332
|
*/
|
|
580
333
|
export declare function readableLiveProgress(value: unknown): LiveProgress | null;
|
|
581
|
-
export type EvalCatalogEntry = {
|
|
582
|
-
slug: string;
|
|
583
|
-
id: string;
|
|
584
|
-
file: string;
|
|
585
|
-
/** Group names, then the eval name. */
|
|
586
|
-
path: string[];
|
|
587
|
-
runs: number;
|
|
588
|
-
latest: {
|
|
589
|
-
runId: string;
|
|
590
|
-
outcome: Outcome;
|
|
591
|
-
};
|
|
592
|
-
};
|
|
593
|
-
export type IndexManifest = {
|
|
594
|
-
generatedAt: string;
|
|
595
|
-
/** Newest first. */
|
|
596
|
-
shards: Array<{
|
|
597
|
-
id: string;
|
|
598
|
-
runs: number;
|
|
599
|
-
}>;
|
|
600
|
-
latest: string | null;
|
|
601
|
-
evals: EvalCatalogEntry[];
|
|
602
|
-
};
|
|
603
|
-
export type StudioInfo = {
|
|
604
|
-
mode: 'live' | 'static';
|
|
605
|
-
reeval: string;
|
|
606
|
-
project: string;
|
|
607
|
-
generatedAt: string;
|
|
608
|
-
/** `debug`: local debug data (`.reeval/debug`) can be read; never in a static build. */
|
|
609
|
-
features: {
|
|
610
|
-
live: boolean;
|
|
611
|
-
scratch: boolean;
|
|
612
|
-
artifacts: boolean;
|
|
613
|
-
debug: boolean;
|
|
614
|
-
};
|
|
615
|
-
};
|
|
616
|
-
/** Splits `file › group › … › name` back into its parts. */
|
|
617
|
-
export declare function splitEvalId(id: string): {
|
|
618
|
-
file: string;
|
|
619
|
-
path: string[];
|
|
620
|
-
};
|
|
621
|
-
/** Month shards (newest run first) and the manifest over them, from every run's header. */
|
|
622
|
-
export declare function buildIndex(entries: HistoryEntry[], generatedAt?: string): {
|
|
623
|
-
manifest: IndexManifest;
|
|
624
|
-
shards: Map<string, HistoryEntry[]>;
|
|
625
|
-
};
|
|
626
|
-
/** Every file the index consists of, by relative path. */
|
|
627
|
-
export declare function indexFiles(entries: HistoryEntry[], generatedAt?: string): Map<string, unknown>;
|
|
628
|
-
export declare function formatDuration(ms: number | null | undefined): string;
|
|
629
|
-
export declare const formatPercent: (rate: number | null) => string;
|
|
630
|
-
/**
|
|
631
|
-
* A harness of the run's stack as `cursor (model composer-2.5, served composer-2.5[fast=true])`:
|
|
632
|
-
* the configured model and reasoning effort, then what the harness says it resolved.
|
|
633
|
-
*/
|
|
634
|
-
export declare function harnessLabel(h: RunRecord['stack']['harnesses'][number]): string;
|
|
635
|
-
/**
|
|
636
|
-
* The runtime reeval ran on, as `node 22.14.0`, or `bun 1.4.2 (node 26.3.0)` with the Node version
|
|
637
|
-
* it's compatible with. A run recorded before `stack.runtime` ran on Node.
|
|
638
|
-
*/
|
|
639
|
-
export declare function runtimeLabel(stack: Pick<RunRecord['stack'], 'node' | 'runtime'>): string;
|
|
640
|
-
/** The evals' language as `typescript evals` or `dart 3.13.5 evals`; null when not recorded. */
|
|
641
|
-
export declare function languageLabel(stack: Pick<RunRecord['stack'], 'language'>): string | null;
|
|
642
|
-
/** Harness-reported tokens as `127k in (98k cached) · 776 out`; null when none were reported. */
|
|
643
|
-
export declare function formatUsage(usage: TokenUsage | null | undefined): string | null;
|
|
644
|
-
export declare const shortSha: (sha: string | null | undefined) => string;
|
|
645
|
-
/** Reasoning time, or `n/a` when the harness couldn't measure it: never `0s` for unknown. */
|
|
646
|
-
export declare const reasoningLabel: (ms: number | null) => string;
|
|
647
|
-
/**
|
|
648
|
-
* An attempt's effort: tool calls, then the files the workspace diff changed, which counts a file
|
|
649
|
-
* the shell wrote, then the calls to the agent's edit tools, which doesn't.
|
|
650
|
-
*/
|
|
651
|
-
export declare function activityLabel(a: Pick<AttemptRecord['activity'], 'tools' | 'fileEdits'>, changed: number): string;
|
|
652
|
-
/**
|
|
653
|
-
* A run's outcome, which `reeval run`'s exit code is read from: an error outside any attempt, a
|
|
654
|
-
* failed gated cell or a budget stop fails the run, while failures in ungated cells (controls, by
|
|
655
|
-
* default) don't. An interrupted run is `skipped`, whatever ran; a resume its filters left
|
|
656
|
-
* `interrupted` exits on its outcome as if it were complete.
|
|
657
|
-
*/
|
|
658
|
-
export declare function runOutcome(run: {
|
|
659
|
-
status: RunRecord['status'];
|
|
660
|
-
evals: ReadonlyArray<Pick<EvalEntry, 'outcome'>>;
|
|
661
|
-
errors?: readonly string[];
|
|
662
|
-
}): Outcome;
|
|
663
|
-
/**
|
|
664
|
-
* The eval hasn't run yet: its run stopped (interrupted, or out of budget) or a filtered resume
|
|
665
|
-
* left it out, and it's `skipped` without being a todo or declared skipped. A resume runs it.
|
|
666
|
-
*/
|
|
667
|
-
export declare const notRunYet: (run: Pick<RunRecord, 'status'>, e: Pick<EvalEntry, 'outcome' | 'todo' | 'skip'>) => boolean;
|
|
668
|
-
/** Distinct judge labels that failed on a counted attempt, in first-seen order. */
|
|
669
|
-
export declare function failedJudgeLabels(record: Pick<RunRecord, 'evals'>): string[];
|
|
670
|
-
/**
|
|
671
|
-
* One sentence for the studio run header: counted pass/fail, which judges failed, and infra.
|
|
672
|
-
* Works without `reeval analyze`.
|
|
673
|
-
*/
|
|
674
|
-
export declare function runLead(record: Pick<RunRecord, 'summary' | 'evals'>): string;
|
|
675
|
-
export type RunRow = {
|
|
676
|
-
entry: HistoryEntry;
|
|
677
|
-
passed: number;
|
|
678
|
-
/** Attempts that reached a verdict or errored; skips excluded. */
|
|
679
|
-
total: number;
|
|
680
|
-
/** Passes and fails, the denominator of the run's pass@1. */
|
|
681
|
-
counted: number;
|
|
682
|
-
/** Stack changes in the cells it shares with earlier runs, and in the judge since the previous complete run. */
|
|
683
|
-
drift: StackChange[];
|
|
684
|
-
/** Evals with a cell whose fingerprint changed since the last run that had that cell. */
|
|
685
|
-
changedEvals: number;
|
|
686
|
-
};
|
|
687
|
-
/**
|
|
688
|
-
* Newest first. Drift is measured per cell, against the last run that had it: a run over fewer
|
|
689
|
-
* cells (a filtered run) is no drift, and neither is the full run after it. The judge's settings
|
|
690
|
-
* compare with the next older complete run.
|
|
691
|
-
*/
|
|
692
|
-
export declare function runRows(entries: HistoryEntry[]): RunRow[];
|
|
693
|
-
/** Attempt counts; `missed` is ungated cells' failures, which `fail` leaves out. */
|
|
694
|
-
export type Tally = {
|
|
695
|
-
pass: number;
|
|
696
|
-
fail: number;
|
|
697
|
-
missed: number;
|
|
698
|
-
infra: number;
|
|
699
|
-
skipped: number;
|
|
700
|
-
};
|
|
701
|
-
export type TreeNode = {
|
|
702
|
-
kind: 'group';
|
|
703
|
-
key: string;
|
|
704
|
-
name: string;
|
|
705
|
-
tally: Tally;
|
|
706
|
-
children: TreeNode[];
|
|
707
|
-
} | {
|
|
708
|
-
kind: 'eval';
|
|
709
|
-
key: string;
|
|
710
|
-
name: string;
|
|
711
|
-
tally: Tally;
|
|
712
|
-
eval: EvalEntry;
|
|
713
|
-
};
|
|
714
|
-
/** Files, then groups, then evals, each with attempt counts; failing branches sort first. */
|
|
715
|
-
export declare function evalTree(record: Pick<RunRecord, 'evals'>): TreeNode[];
|
|
716
|
-
export type CellRow = {
|
|
717
|
-
key: string;
|
|
718
|
-
harness: string;
|
|
719
|
-
environment: string;
|
|
720
|
-
sandbox: string;
|
|
721
|
-
role: CellRecord['role'];
|
|
722
|
-
/** Whether the cell's failures fail the eval: its gate isn't `'none'`. */
|
|
723
|
-
gated: boolean;
|
|
724
|
-
gate: CellRecord['gate'];
|
|
725
|
-
overrides: string[];
|
|
726
|
-
/** The reasoning effort the harness factory asked for; null when it left it unset. */
|
|
727
|
-
reasoningEffort: string | null;
|
|
728
|
-
runs: number;
|
|
729
|
-
pass: number;
|
|
730
|
-
fail: number;
|
|
731
|
-
infraError: number;
|
|
732
|
-
skipped: number;
|
|
733
|
-
/** pass@1: passes over passes and fails. */
|
|
734
|
-
passAt1: number | null;
|
|
735
|
-
outcome: Outcome;
|
|
736
|
-
/** A variant's lift over its control, in whole points (from the record), else null. */
|
|
737
|
-
lift: number | null;
|
|
738
|
-
/** Its 95% interval in whole points, and whether that spans 0. */
|
|
739
|
-
liftInterval: {
|
|
740
|
-
lo: number;
|
|
741
|
-
hi: number;
|
|
742
|
-
withinNoise: boolean;
|
|
743
|
-
} | null;
|
|
744
|
-
/** The interval's lower bound is below the compare's `minLift`. */
|
|
745
|
-
belowMinLift: boolean;
|
|
746
|
-
/** Median tool calls, agent time and tokens over the cell's counted attempts. */
|
|
747
|
-
effort: CellEffort | null;
|
|
748
|
-
/** A variant's effort against its control, with the rank test's verdict. */
|
|
749
|
-
effortVsControl: EffortContrast | null;
|
|
750
|
-
/** Per-key judge pass/counted over this cell's counted attempts. */
|
|
751
|
-
judgeRates: JudgeKeyRate[];
|
|
752
|
-
/** Sum of counted attempts' tokens; null when none reported. */
|
|
753
|
-
usage: TokenUsage | null;
|
|
754
|
-
attempts: AttemptRecord[];
|
|
755
|
-
};
|
|
756
|
-
/** One row per cell: controls and variants, with each variant's recorded lift. */
|
|
757
|
-
export declare function cellRows(e: EvalEntry): CellRow[];
|
|
758
|
-
/** One side's wins over the decided votes it was in; `rate` is null when none decided. */
|
|
759
|
-
export type AnswerRate = {
|
|
760
|
-
name: string;
|
|
761
|
-
wins: number;
|
|
762
|
-
n: number;
|
|
763
|
-
rate: number | null;
|
|
764
|
-
};
|
|
765
|
-
export type AnswerRow = {
|
|
766
|
-
question: string;
|
|
767
|
-
/** The model that answered: as served, when the provider said, else as configured. */
|
|
768
|
-
model: string;
|
|
769
|
-
votes: number;
|
|
770
|
-
ties: number;
|
|
771
|
-
/** Arms against each other on one harness, then harnesses against each other on one arm. */
|
|
772
|
-
within: AnswerRate[];
|
|
773
|
-
across: AnswerRate[];
|
|
774
|
-
/** Why some or all of its votes are missing: the model call failed. */
|
|
775
|
-
error: string | null;
|
|
776
|
-
/** No votes and no error: no pair had both sides pass their checks. */
|
|
777
|
-
noPairs: boolean;
|
|
778
|
-
/** Distinct vote reasons, in vote order, at most three. */
|
|
779
|
-
reasons: string[];
|
|
780
|
-
/** `questions/<n>.json` under the run's directory: what the model was sent; null when not kept. */
|
|
781
|
-
input: string | null;
|
|
782
|
-
};
|
|
783
|
-
/** One row per pairwise question, with each cut's win rates in the record's order. */
|
|
784
|
-
export declare function answerRows(e: Pick<EvalEntry, 'answers'>): AnswerRow[];
|
|
785
|
-
/** One attempt's answer to a question asked of it on its own. */
|
|
786
|
-
export type AttemptAnswerItem = {
|
|
787
|
-
/** The attempt's id, `<cell key>#<n>`. */
|
|
788
|
-
attempt: string;
|
|
789
|
-
outcome: Outcome;
|
|
790
|
-
choice: string | null;
|
|
791
|
-
text: string;
|
|
792
|
-
error: string | null;
|
|
793
|
-
/** `questions/<n>.json` in the attempt's directory: what the model was sent; null when not kept. */
|
|
794
|
-
input: string | null;
|
|
795
|
-
};
|
|
796
|
-
export type AttemptAnswerRow = {
|
|
797
|
-
question: string;
|
|
798
|
-
/** The model that answered: as served, when the provider said, else as configured. */
|
|
799
|
-
model: string;
|
|
800
|
-
/** How many attempts picked each choice, in the question's order; null on a free-text question. */
|
|
801
|
-
choices: Array<{
|
|
802
|
-
choice: string;
|
|
803
|
-
n: number;
|
|
804
|
-
}> | null;
|
|
805
|
-
answers: AttemptAnswerItem[];
|
|
806
|
-
};
|
|
807
|
-
/** One row per question asked of each attempt on its own, in first-asked order. */
|
|
808
|
-
export declare function attemptAnswerRows(e: Pick<EvalEntry, 'attempts'>): AttemptAnswerRow[];
|
|
809
|
-
/** Whether an eval needs the cells table: several cells, or several runs of one. */
|
|
810
|
-
export declare const isMatrix: (e: Pick<EvalEntry, 'cells'>) => boolean;
|
|
811
|
-
export type EvalHistoryPoint = {
|
|
812
|
-
run: HistoryEntry;
|
|
813
|
-
outcome: Outcome;
|
|
814
|
-
/** pass@1 over the gated cells' attempts. */
|
|
815
|
-
passAt1: number | null;
|
|
816
|
-
cells: HistoryCell[];
|
|
817
|
-
/** A cell's fingerprint differs from the last run that had that cell. */
|
|
818
|
-
changed: boolean;
|
|
819
|
-
};
|
|
820
|
-
/** One eval across runs, oldest first. */
|
|
821
|
-
export declare function evalHistory(entries: HistoryEntry[], slug: string): EvalHistoryPoint[];
|
|
822
|
-
export type CompareViewRow = {
|
|
823
|
-
id: string;
|
|
824
|
-
slug: string | null;
|
|
825
|
-
status: CompareStatus;
|
|
826
|
-
base: Outcome | null;
|
|
827
|
-
head: Outcome | null;
|
|
828
|
-
basePassAt1: number | null;
|
|
829
|
-
headPassAt1: number | null;
|
|
830
|
-
/** See `CompareRow`: then pass@1 is over the cells both runs have too. */
|
|
831
|
-
cells?: CompareRow['cells'];
|
|
832
|
-
changes?: string[];
|
|
833
|
-
gate?: CompareRow['gate'];
|
|
834
|
-
/** Mean judge score per run; null without judgments or without the record. */
|
|
835
|
-
baseScore: number | null;
|
|
836
|
-
headScore: number | null;
|
|
837
|
-
/** Cells in both runs, with lift and effort on each side. */
|
|
838
|
-
perCell: CompareCell[];
|
|
839
|
-
/** See `CompareRow`: an eval with `cases`, each case's row, and its gates over the cases. */
|
|
840
|
-
cases?: CompareRow['cases'];
|
|
841
|
-
caseScope?: CompareRow['caseScope'];
|
|
842
|
-
lifts?: CompareRow['lifts'];
|
|
843
|
-
rates?: CompareRow['rates'];
|
|
844
|
-
};
|
|
845
|
-
export declare const COMPARE_ORDER: CompareStatus[];
|
|
846
|
-
/**
|
|
847
|
-
* Whether some of an eval's cells aren't gated (`gate: 'none'`, e.g. controls under
|
|
848
|
-
* `compare.gate: 'variants'`). Its pass@1 then covers fewer attempts than its pass count, so
|
|
849
|
-
* the eval line labels it `gated`: `2/4 · pass@1 100% gated`.
|
|
850
|
-
*/
|
|
851
|
-
export declare const hasUngatedCells: (cells: readonly Pick<HistoryCell, 'gate'>[]) => boolean;
|
|
852
|
-
/** A study's pass: every cell `gate: 'none'`, so nothing it saw passed or failed it. */
|
|
853
|
-
export declare const isNotGatedPass: (e: {
|
|
854
|
-
outcome: Outcome;
|
|
855
|
-
cells: readonly Pick<HistoryCell, 'gate'>[];
|
|
856
|
-
}) => boolean;
|
|
857
|
-
/**
|
|
858
|
-
* An expected miss: a failed attempt in an ungated cell (a control, by default). It's reported
|
|
859
|
-
* apart from failures, since it fails neither its eval nor the run.
|
|
860
|
-
*/
|
|
861
|
-
export declare const isUngatedMiss: (e: {
|
|
862
|
-
cells: readonly Pick<HistoryCell, 'key' | 'gate'>[];
|
|
863
|
-
}, a: Pick<AttemptRecord, 'cell' | 'outcome' | 'error'>) => boolean;
|
|
864
|
-
/**
|
|
865
|
-
* The outcome an eval's mark shows. A pass needs a counted (passed or failed) attempt and no infra
|
|
866
|
-
* error: a gate that lets those through, such as `gate: 'none'`, still shows `!`, not `✓`.
|
|
867
|
-
*/
|
|
868
|
-
export declare function evalMarkOutcome(e: Pick<EvalEntry, 'outcome' | 'attempts'>): Outcome;
|
|
869
|
-
/** `reeval compare`'s classification, plus pass@1 and judge-score deltas from the records. */
|
|
870
|
-
export declare function compareView(base: HistoryEntry, head: HistoryEntry, records?: {
|
|
871
|
-
base?: RunRecord;
|
|
872
|
-
head?: RunRecord;
|
|
873
|
-
}): {
|
|
874
|
-
rows: CompareViewRow[];
|
|
875
|
-
counts: Record<CompareStatus, number>;
|
|
876
|
-
changes: StackChange[];
|
|
877
|
-
};
|
|
878
|
-
export type TranscriptLine = ({
|
|
879
|
-
turn: number;
|
|
880
|
-
} & TranscriptEvent) | {
|
|
881
|
-
turn: number;
|
|
882
|
-
ts: string;
|
|
883
|
-
atMs: number;
|
|
884
|
-
type: 'prompt';
|
|
885
|
-
text: string;
|
|
886
|
-
} | {
|
|
887
|
-
turn: number;
|
|
888
|
-
ts: string;
|
|
889
|
-
atMs: number;
|
|
890
|
-
type: 'turn-end';
|
|
891
|
-
status: TurnStatus;
|
|
892
|
-
durationMs: number;
|
|
893
|
-
};
|
|
894
|
-
export type TranscriptItem = {
|
|
895
|
-
kind: 'turn';
|
|
896
|
-
turn: number;
|
|
897
|
-
prompt: string;
|
|
898
|
-
ts: string;
|
|
899
|
-
} | {
|
|
900
|
-
kind: 'event';
|
|
901
|
-
turn: number;
|
|
902
|
-
event: TranscriptLine;
|
|
903
|
-
} | {
|
|
904
|
-
kind: 'turn-end';
|
|
905
|
-
turn: number;
|
|
906
|
-
status: TurnStatus;
|
|
907
|
-
durationMs: number;
|
|
908
|
-
};
|
|
909
|
-
/** Parses `transcript.jsonl`, skipping lines that aren't JSON (a truncated tail, say). */
|
|
910
|
-
export declare function parseTranscript(text: string): TranscriptLine[];
|
|
911
|
-
export declare function transcriptItems(lines: TranscriptLine[]): TranscriptItem[];
|
|
912
|
-
export declare const findEval: (record: Pick<RunRecord, 'evals'>, slug: string) => EvalEntry | undefined;
|
|
913
|
-
export declare const findAttempt: (e: EvalEntry, attemptId: string) => AttemptRecord | undefined;
|
|
914
|
-
/** The attempt to open by default: the first failure, else the first attempt. */
|
|
915
|
-
export declare const defaultAttempt: (e: EvalEntry) => AttemptRecord | undefined;
|
|
916
334
|
/** One observed name over one cell's attempts: its values tallied, or its numbers summarized. Never gates. */
|
|
917
335
|
export type ObservedTally = {
|
|
918
336
|
name: string;
|
|
@@ -958,6 +376,10 @@ export type PatchFile = {
|
|
|
958
376
|
removed: number;
|
|
959
377
|
text: string;
|
|
960
378
|
};
|
|
379
|
+
/** Dependency lockfiles excluded from report line-count totals. */
|
|
380
|
+
export declare const LOCKFILES: readonly ['pnpm-lock.yaml', 'package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock', 'bun.lock', 'bun.lockb', 'pubspec.lock', 'Cargo.lock', 'go.sum', 'poetry.lock', 'uv.lock', 'Pipfile.lock', 'Gemfile.lock', 'composer.lock', 'Podfile.lock', 'gradle.lockfile'];
|
|
381
|
+
/** True when `path` is a known dependency lockfile (basename match). */
|
|
382
|
+
export declare function isLockfile(path: string): boolean;
|
|
961
383
|
/**
|
|
962
384
|
* The path one file's part of a git patch changes: the new path, or the old one for a deletion.
|
|
963
385
|
* Read from its header lines, before the first `@@`, where git C-quotes an awkward name.
|
|
@@ -965,112 +387,12 @@ export type PatchFile = {
|
|
|
965
387
|
export declare function patchPath(part: string): string | null;
|
|
966
388
|
/** Splits a git patch into one entry per file, with line counts. */
|
|
967
389
|
export declare function splitPatch(patch: string): PatchFile[];
|
|
968
|
-
/**
|
|
969
|
-
* The studio's data API is a file layout: `reeval studio` serves these relative paths and
|
|
970
|
-
* `reeval studio build` writes them to disk, so the web client reads both the same way.
|
|
971
|
-
*
|
|
972
|
-
* studio.json mode, project and features
|
|
973
|
-
* data/index/manifest.json shard list and eval catalog
|
|
974
|
-
* data/index/<YYYY-MM>.json run headers started that UTC month, newest first
|
|
975
|
-
* data/runs/<runId>.json the record, byte for byte
|
|
976
|
-
* data/runs/<runId>/<artifact path> artifacts, raw
|
|
977
|
-
* data/analysis/<runId>.json the run's saved analyses (and, live, cached ones)
|
|
978
|
-
*/
|
|
979
|
-
export declare const RUN_ID: RegExp;
|
|
980
|
-
export declare const studioPath = "studio.json";
|
|
981
|
-
export declare const manifestPath = "data/index/manifest.json";
|
|
982
|
-
export declare const shardPath: (id: string) => string;
|
|
983
|
-
export declare const recordPath: (runId: string) => string;
|
|
984
|
-
export declare const artifactPath: (runId: string, path: string) => string;
|
|
985
|
-
/** A shard id: the UTC month a run started in. */
|
|
986
|
-
export declare const shardOf: (startedAt: string) => string;
|
|
987
|
-
export type DataTarget = {
|
|
988
|
-
kind: 'studio';
|
|
989
|
-
} | {
|
|
990
|
-
kind: 'manifest';
|
|
991
|
-
} | {
|
|
992
|
-
kind: 'shard';
|
|
993
|
-
id: string;
|
|
994
|
-
} | {
|
|
995
|
-
kind: 'record';
|
|
996
|
-
runId: string;
|
|
997
|
-
} | {
|
|
998
|
-
kind: 'analysis';
|
|
999
|
-
runId: string;
|
|
1000
|
-
} | {
|
|
1001
|
-
kind: 'artifact';
|
|
1002
|
-
runId: string;
|
|
1003
|
-
segments: string[];
|
|
1004
|
-
};
|
|
1005
|
-
/**
|
|
1006
|
-
* Maps a request path (relative, without a leading slash, already percent-decoded once) to what
|
|
1007
|
-
* it names, or null. Every segment must match an allowlist, so `.`, `..`, empty segments,
|
|
1008
|
-
* backslashes and NUL never get through.
|
|
1009
|
-
*/
|
|
1010
|
-
export declare function parseDataPath(path: string): DataTarget | null;
|
|
1011
|
-
/** Artifacts are untrusted agent output: JSON stays JSON, PNGs are images, else plain text. */
|
|
1012
|
-
export declare const artifactType: (name: string) => string;
|
|
1013
|
-
/** `<history>/<run>/remarks.json`, beside the record. */
|
|
1014
|
-
export declare const REMARKS_FILE = "remarks.json";
|
|
1015
|
-
/** A run's remarks, `<history>/<run>/remarks.json`. Written only when the run raised one. */
|
|
1016
|
-
export type RunRemarks = {
|
|
1017
|
-
kind: 'reeval-remarks';
|
|
1018
|
-
schemaVersion: 1;
|
|
1019
|
-
runId: string;
|
|
1020
|
-
createdAt: string;
|
|
1021
|
-
/**
|
|
1022
|
-
* Every remark this run raised, ranked, including those an earlier run printed first
|
|
1023
|
-
* (`firstSeenRun`), which don't print again.
|
|
1024
|
-
*/
|
|
1025
|
-
remarks: Remark[];
|
|
1026
|
-
};
|
|
1027
|
-
/** Each remark kind's severity. */
|
|
1028
|
-
export declare const REMARK_KINDS: Record<Remark['kind'], {
|
|
1029
|
-
severity: 'concern' | 'suggestion';
|
|
1030
|
-
}>;
|
|
1031
|
-
/** A remark about the eval's quality, not about the agent. */
|
|
1032
|
-
export type Remark = {
|
|
1033
|
-
/**
|
|
1034
|
-
* sha256(kind, evalId, the eval's sources hash, and for `slow-mcp-start` the server): one remark
|
|
1035
|
-
* per kind (and server) per eval version.
|
|
1036
|
-
*/
|
|
1037
|
-
id: string;
|
|
1038
|
-
evalId: string;
|
|
1039
|
-
kind: 'too-few-tries' | 'cant-tell-apart' | 'check-never-fails' | 'flaky-check' | 'slow-mcp-start' | 'proxy-only-egress' | 'setup-process-secret' | 'setup-tunnel-closed' | 'snapshot-too-large' | 'observe-throws';
|
|
1040
|
-
/** Fixed by kind (`REMARK_KINDS`). */
|
|
1041
|
-
severity: 'concern' | 'suggestion';
|
|
1042
|
-
/** One plain sentence: what's wrong. */
|
|
1043
|
-
text: string;
|
|
1044
|
-
/**
|
|
1045
|
-
* The remark as one short line, with its fix and where (`1 run per side; set runs: 4 at
|
|
1046
|
-
* evals/a.eval.ts:12 to confirm these differences.`), which `reeval run`'s list prints without
|
|
1047
|
-
* `-v`; absent when the kind has none.
|
|
1048
|
-
*/
|
|
1049
|
-
brief?: string;
|
|
1050
|
-
/** Facts from this run (or history facts) showing it mattered now. */
|
|
1051
|
-
evidence: string[];
|
|
1052
|
-
/** An edit at a source location (`source:<file>#L<n>`) in the eval. */
|
|
1053
|
-
fix: {
|
|
1054
|
-
at: string;
|
|
1055
|
-
edit: string;
|
|
1056
|
-
};
|
|
1057
|
-
/**
|
|
1058
|
-
* The first run that printed this `id`, or this run when none has: it prints only in that run,
|
|
1059
|
-
* except the kinds about the environment, which print in every run they happen.
|
|
1060
|
-
*/
|
|
1061
|
-
firstSeenRun: string;
|
|
1062
|
-
/**
|
|
1063
|
-
* Whether this run's `reeval run` output printed it: false when an earlier run printed it, or
|
|
1064
|
-
* when the cap or a run without `-v` left it out, so a later run prints it.
|
|
1065
|
-
*/
|
|
1066
|
-
printed?: boolean;
|
|
1067
|
-
};
|
|
1068
|
-
/** A file from a newer reeval, whose schemaVersion this reeval doesn't know. */
|
|
390
|
+
/** A file from a newer reeval, whose version (`field`, `schemaVersion` by default) this reeval doesn't know. */
|
|
1069
391
|
export declare class NewerSchemaError extends Error {
|
|
1070
392
|
name: string;
|
|
1071
393
|
/** The message without its subject: `written by a newer reeval (…). Upgrade reeval to read it.` */
|
|
1072
394
|
readonly reason: string;
|
|
1073
|
-
constructor(what: string, version: number, newest: number);
|
|
395
|
+
constructor(what: string, version: number, newest: number, field?: string);
|
|
1074
396
|
}
|
|
1075
397
|
/**
|
|
1076
398
|
* A record as every reader sees it: migrated to this reeval's schemaVersion, with each field added
|
|
@@ -1087,6 +409,8 @@ export declare const readableAnalysis: (doc: AnalysisDoc, what?: string) => Anal
|
|
|
1087
409
|
export declare const readableRemarks: (doc: RunRemarks, what?: string) => RunRemarks;
|
|
1088
410
|
/** A capture's `<name>.capture.json` as the studio sees it; one from a newer reeval throws. */
|
|
1089
411
|
export declare const readableCapturePage: (page: CapturePageFile, what?: string) => CapturePageFile;
|
|
412
|
+
/** The run's scope, from its record alone; a record without `invocation` reads as one run with none. */
|
|
413
|
+
export declare function runScope(record: RunRecord): RunScope;
|
|
1090
414
|
/**
|
|
1091
415
|
* The runs a side a compare defaults to and the plan suggests. It doesn't gate the effect: a lift
|
|
1092
416
|
* always reads as its observed direction, and whether its interval excludes 0 decides whether it
|
|
@@ -1340,7 +664,7 @@ export type CellResult = {
|
|
|
1340
664
|
/** On a variant, the direction of its pass rate against its baseline's; null on a control or with nothing counted on a side. */
|
|
1341
665
|
effect: Effect | null;
|
|
1342
666
|
/** The stored 95% interval on the variant's difference from its baseline. Not for display. */
|
|
1343
|
-
interval: Interval | null;
|
|
667
|
+
interval: Interval$1 | null;
|
|
1344
668
|
/** On a variant, the compare's `minLift`, a fraction like the interval; null without one. */
|
|
1345
669
|
minLift: number | null;
|
|
1346
670
|
/** Medians over the counted attempts; null below `COMPARE_RUNS` of them. */
|
|
@@ -1430,7 +754,7 @@ export type JudgeResult = {
|
|
|
1430
754
|
/** One per counted attempt that judged it. */
|
|
1431
755
|
scores: number[];
|
|
1432
756
|
};
|
|
1433
|
-
|
|
757
|
+
type Interval$1 = {
|
|
1434
758
|
low: number;
|
|
1435
759
|
high: number;
|
|
1436
760
|
};
|
|
@@ -1450,7 +774,7 @@ export type PooledArm = {
|
|
|
1450
774
|
effect: Effect;
|
|
1451
775
|
/** Cases with counted attempts on both sides. */
|
|
1452
776
|
pairs: number;
|
|
1453
|
-
interval: Interval;
|
|
777
|
+
interval: Interval$1;
|
|
1454
778
|
/** The pooled lift's verdict: `insufficient` below `PAIRED_MIN` discordant pairs or cases. */
|
|
1455
779
|
verdict: FamilyVerdict;
|
|
1456
780
|
/** The compare's `minLift`, a fraction like the interval, which gates its low end; null without one. */
|
|
@@ -1493,5 +817,13 @@ export declare const isStudy: (cells: readonly Pick<CellResult, 'gated'>[]) => b
|
|
|
1493
817
|
export declare const isNotGatedResult: (e: Pick<EvalResult, 'outcome' | 'cells' | 'cases' | 'pooled' | 'rates'>) => boolean;
|
|
1494
818
|
/** The facts of a run, from its record. Pure: the same record always gives the same result. */
|
|
1495
819
|
export declare function suiteResult(folded: RunRecord): SuiteResult;
|
|
1496
|
-
|
|
1497
|
-
|
|
820
|
+
/**
|
|
821
|
+
* Classifies an unhandled mid-turn error as `config` or `transport`.
|
|
822
|
+
*
|
|
823
|
+
* `config`: the harness refused the request before any model output (HTTP 400,
|
|
824
|
+
* 401, 403, 404, or 422, or an adapter refusal of a model or option).
|
|
825
|
+
* Everything else is `transport` (5xx, network, stream reset, `[unavailable]`).
|
|
826
|
+
*/
|
|
827
|
+
type TurnErrorKind = 'transport' | 'config';
|
|
828
|
+
export declare function turnErrorKind(error: string | null | undefined): TurnErrorKind;
|
|
829
|
+
export { ANALYSIS_PROMPT_VERSION, ActionKind, AnalysisAction, AnalysisAnomaly, AnalysisBody, AnalysisClaim, AnalysisClaimRole, AnalysisDoc, AnalysisEntry, AnalysisIndex, AnalysisNote, AnalysisScope, AnalysisValidation, AnomalyKind, AnswerRate, AnswerRow, type Arm, type ArmRef, type ArmRow, type ArtifactRef, type AssertionRow, AttemptAnswerItem, AttemptAnswerRow, type AttemptFacts, type AttemptIdMap, type AttemptIndexRow, type AttemptRecord, type AttemptRef, type BehaviorRow, type BlockMarkdown, type BreakdownRow, COMPARE_ORDER, COMPARE_STATUS_LABEL, type CaptureConsoleMessage, type CaptureFailedRequest, type CaptureGallery, type CapturePage, type CapturePageFile, type CaptureShot, CellEffort, type CellRecord, CellRow, type ChangeRow, type CheckCell, type ChoiceBreakdown, CollapsedJudge, CompareCell, CompareFamilyLift, CompareFamilyRate, CompareRow, CompareStatus, CompareViewRow, Confidence, type ContrastRow, type Count, DIGEST_VERSION, DIRECTIONAL_MIN, DISCOVERY_SOURCE_LABEL, type Delta, type DiscoveryRate, EFFORT_ALPHA, EFFORT_MIN_RUNS, EffortContrast, EffortMetric, type EffortPivotRow, EffortSample, EffortVerdict, type EvalDetail, type EvalEntry, type EvalHeadline, EvalHistoryPoint, type EvalSection, type EvalShape, type EvalSummary, type ExecutiveRollup, type FailureCluster, type FailureClusters, type Fairness, type FamilyLift, type FamilyRate, type FamilyRecord, type Figure, type FileRef, type HeadlinePrimary, type HistoryCell, type HistoryEntry, type IntegrityIssue, type IntegrityKind, type Interval, JudgeKeyRate, type JudgeRow, type JudgmentBreakdown, type JudgmentRecord, Level, MEASURE_LABEL, type MeasureCells, type MeasureKey, type MeasureVsBaseline, type Outcome, PASS_RATE_OUTCOME_LABEL, type PairwiseRow, type PassRateOutcome, type ProvenanceState, type QualitativeAnswers, type Quote, REMARKS_FILE, REMARK_KINDS, REPORT_POLICY, REPORT_VERSION, type Rate, type RegressionChanges, Remark, type ReportClaim, type ReportLinks, type ReportPolicy, type RosterRow, type RunDebug, type RunHeader, type RunRecord, RunRemarks, type RunReport, type RunReportIndex, type RunReportOptions, RunRow, type RunScope, type RunTally, type SandboxWarning, type SplitRunReport, type Spread, StackChange, StackSource, type StackSummary, TOKENS_UNREPORTED, Tally, type TaskResults, type TranscriptEvent, TranscriptItem, TranscriptLine, TreeNode, type TurnErrorKind, VERDICT_LABEL, VERDICT_TEXT, type VariantComparison, activityLabel, addedRow, answerRows, attemptAnchor, attemptAnchorId, attemptAnswerRows, attemptFacts, attemptHref, attemptIds, attemptRef, bothUnderATenth, buildRunReport, captureHref, cellContrast, cellEffort, cellJudgeRates, cellRows, cellSamples, cellUsage, cellsChanged, checkLabels, checkTries, checksByLabel, collapsedJudgeLabel, collapsedJudges, commandLine, compare, compareStatusLabel, compareView, contrastEffort, contrastMeasureLabel, countFigure, countOf, defaultAttempt, diffLinesText, directional, discoverySourceLabel, durationFigure, effortChangeLabel, effortLabel, effortOf, effortSample, effortSummary, effortVerdict, evalAnchorHref, evalHistory, evalMarkOutcome, evalTree, failedJudgeLabels, figure, filesChanged, findAttempt, findEval, formatCount, formatDuration, formatPercent, formatRatio, formatSeconds, formatUsage, gfmAnchor, harnessLabel, hasUngatedCells, idOf, isMatrix, isNotGatedPass, isSuperseded, isUngatedMiss, joinRunReport, judgeLabel, judgeRatesLabel, judgeScoreText, labeledJudgments, languageLabel, measurePhrase, medianWithDelta, newRows, notRunYet, parseTranscript, parseTranscriptLine, passRateOutcomeFromMatchers, pctDelta, pinnedAnalysis, provenanceState, ptsDelta, ptsInterval, questionHref, rateOf, reasoningLabel, recordHref, runLead, runOutcome, runRows, runtimeLabel, selectedEvals, shortSha, sortAnalyses, sourceChanges, splitEvalId, splitLabel, splitOf, splitRunReport, spreadOf, stackChanges, stateRows, summaryJudgeRates, tokenValues, tokensFigure, tokensText, transcriptByLine, transcriptItems, triesNote, uniqueJudgeLabels, usdFigure, verdictLabel, warnings, whyChanged, withBase, worktreePatchLink };
|