reeval 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.md +17 -17
- package/THIRD_PARTY_NOTICES.md +0 -2068
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +65 -50
- package/dist/{adapter-Dlq3-4OS.js → adapter-vLxVtLdk.js} +6 -6
- package/dist/{adapter-Dlq3-4OS.js.map → adapter-vLxVtLdk.js.map} +1 -1
- package/dist/{agent-OsAfayRf.js → agent-DN36i629.js} +9 -9
- package/dist/{agent-OsAfayRf.js.map → agent-DN36i629.js.map} +1 -1
- package/dist/{agents-BztvTkfM.js → agents-Bp40Y7Lk.js} +47 -59
- package/dist/agents-Bp40Y7Lk.js.map +1 -0
- package/dist/{analysis-BnG_1x_O.js → analysis-8vvMnUJB.js} +4 -5
- package/dist/analysis-8vvMnUJB.js.map +1 -0
- package/dist/api-ZeGnedB3.js +141 -0
- package/dist/api-ZeGnedB3.js.map +1 -0
- package/dist/assemble-ChNVeWNr.js +1 -0
- package/dist/{assemble-Doa1m-HW.js → assemble-DsllweEG.js} +132 -68
- package/dist/assemble-DsllweEG.js.map +1 -0
- package/dist/{attach-BH2HdwH_.js → attach-DFf2hFl7.js} +18 -10
- package/dist/attach-DFf2hFl7.js.map +1 -0
- package/dist/{attempt-_dUsblVx.js → attempt-CfwXgF2q.js} +3 -3
- package/dist/{attempt-_dUsblVx.js.map → attempt-CfwXgF2q.js.map} +1 -1
- package/dist/{attempt-host-8mQV8Uee.js → attempt-host-XyyFR6jc.js} +37 -14
- package/dist/attempt-host-XyyFR6jc.js.map +1 -0
- package/dist/boat-BCNNEJni.js +298 -0
- package/dist/boat-BCNNEJni.js.map +1 -0
- package/dist/{boat-DKjqo_ie.js → boat-CNUnhvWH.js} +117 -197
- package/dist/boat-CNUnhvWH.js.map +1 -0
- package/dist/boat-DRTQHXlj.js +1 -0
- package/dist/{boat-images-CRN_U_eG.js → boat-images-B0MsRBFx.js} +18 -22
- package/dist/boat-images-B0MsRBFx.js.map +1 -0
- package/dist/{bundle-DU2aQVeq.js → bundle-B1v8DEEX.js} +26 -15
- package/dist/bundle-B1v8DEEX.js.map +1 -0
- package/dist/{capture-By1byHaX.js → capture-BAtaw-L-.js} +20 -19
- package/dist/{capture-By1byHaX.js.map → capture-BAtaw-L-.js.map} +1 -1
- package/dist/capture-DOHR_KXl.js +1 -0
- package/dist/cases-Bsp-G-ZL.js +1 -0
- package/dist/{cases-CM-MQRQp.js → cases-C6vnV-gn.js} +61 -43
- package/dist/cases-C6vnV-gn.js.map +1 -0
- package/dist/classify-BIJQ2Xqo.js +163 -0
- package/dist/classify-BIJQ2Xqo.js.map +1 -0
- package/dist/{cli-DMHOwdVr.js → cli-BFfZy9Ax.js} +385 -178
- package/dist/cli-BFfZy9Ax.js.map +1 -0
- package/dist/{cli-spec-D0wuUaj9.js → cli-spec-Bj2z-4Pa.js} +242 -39
- package/dist/cli-spec-Bj2z-4Pa.js.map +1 -0
- package/dist/cli.js +6 -2
- package/dist/cli.js.map +1 -1
- package/dist/client-Dj65gXG3.js +105 -0
- package/dist/client-Dj65gXG3.js.map +1 -0
- package/dist/cloud-DgMSxpCk.js +154 -0
- package/dist/cloud-DgMSxpCk.js.map +1 -0
- package/dist/{hosted-5Zz2peTR.js → cloud-TuNUlZ0L.js} +31 -31
- package/dist/cloud-TuNUlZ0L.js.map +1 -0
- package/dist/cloud-files-BjoNhDJs.js +301 -0
- package/dist/cloud-files-BjoNhDJs.js.map +1 -0
- package/dist/cloud-project-B1lN8D9m.js +1 -0
- package/dist/cloud-project-Dh17iadw.js +110 -0
- package/dist/cloud-project-Dh17iadw.js.map +1 -0
- package/dist/cloud-push-CdYqsM34.js +817 -0
- package/dist/cloud-push-CdYqsM34.js.map +1 -0
- package/dist/cloud-push-GXvEYHoa.js +1 -0
- package/dist/cloud-run-BfFCs6Fm.js +1 -0
- package/dist/cloud-run-CZOpabN1.js +517 -0
- package/dist/cloud-run-CZOpabN1.js.map +1 -0
- package/dist/cloud-secrets-BskXM_2n.js +1 -0
- package/dist/cloud-secrets-DrrgGOF_.js +345 -0
- package/dist/cloud-secrets-DrrgGOF_.js.map +1 -0
- package/dist/{command-CXxckfsq.js → command-Bic6f84Y.js} +28 -130
- package/dist/command-Bic6f84Y.js.map +1 -0
- package/dist/{command-CAmnWLFL.js → command-C7m3ZoXS.js} +93 -27
- package/dist/command-C7m3ZoXS.js.map +1 -0
- package/dist/command-DHrT6YVl.js +183 -0
- package/dist/command-DHrT6YVl.js.map +1 -0
- package/dist/command-DMwSLfJT.js +141 -0
- package/dist/command-DMwSLfJT.js.map +1 -0
- package/dist/{command-ao0NZ8h0.js → command-DP8sQ-dk.js} +29 -23
- package/dist/command-DP8sQ-dk.js.map +1 -0
- package/dist/{commands-ttChCsAy.js → commands-BPLexk6R.js} +62 -41
- package/dist/commands-BPLexk6R.js.map +1 -0
- package/dist/commands-D8D7ZJ0I.js +1 -0
- package/dist/{config-myJT1Qjc.js → config-DQqcGmz7.js} +67 -47
- package/dist/config-DQqcGmz7.js.map +1 -0
- package/dist/core-BXYzOEyQ.js +2058 -0
- package/dist/core-BXYzOEyQ.js.map +1 -0
- package/dist/{slug-IHh3f770.js → cost-DQ0ayy6h.js} +37 -2
- package/dist/cost-DQ0ayy6h.js.map +1 -0
- package/dist/{cost-C21gGrju.js → cost-Dp-xFGJ1.js} +2 -2
- package/dist/{cost-C21gGrju.js.map → cost-Dp-xFGJ1.js.map} +1 -1
- package/dist/{cost-CdRX1s2p.js → cost-a_rUSdsf.js} +1 -1
- package/dist/{cursor-catalog-CUQrabJ_.js → cursor-catalog-DQCmwsX3.js} +4 -4
- package/dist/{cursor-catalog-CUQrabJ_.js.map → cursor-catalog-DQCmwsX3.js.map} +1 -1
- package/dist/{daytona-Dfhwtr9P.js → daytona-DOeJzXl_.js} +27 -19
- package/dist/daytona-DOeJzXl_.js.map +1 -0
- package/dist/{daytona-image-CnG6G00x.js → daytona-image-B9JpVeP5.js} +5 -4
- package/dist/daytona-image-B9JpVeP5.js.map +1 -0
- package/dist/daytona-image-BsCbvDGn.js +1 -0
- package/dist/db-BqFlDMmS.js +1 -0
- package/dist/{db-5H95sFii.js → db-DU3fgU0h.js} +32 -120
- package/dist/db-DU3fgU0h.js.map +1 -0
- package/dist/debug-CGg7KHKq.js +1 -0
- package/dist/{debug-CIXzB4wK.js → debug-T29p6HOX.js} +2 -10
- package/dist/debug-T29p6HOX.js.map +1 -0
- package/dist/{default-effort-DYpCkJRk.js → default-effort-BFl6WLBi.js} +3 -3
- package/dist/{default-effort-DYpCkJRk.js.map → default-effort-BFl6WLBi.js.map} +1 -1
- package/dist/{default-effort-CtNY9CGW.js → default-effort-C6Gcd-Fy.js} +1 -1
- package/dist/{dist-DeqeCUZe.js → dist-DCG7vZGQ.js} +3 -3
- package/dist/{dist-DeqeCUZe.js.map → dist-DCG7vZGQ.js.map} +1 -1
- package/dist/{docker-kpudeSxV.js → docker-B5YBmpPz.js} +1 -1
- package/dist/{docker-CR0JZ1FE.js → docker-hIo_H0W0.js} +4 -4
- package/dist/{docker-CR0JZ1FE.js.map → docker-hIo_H0W0.js.map} +1 -1
- package/dist/{doctor-aE-RMbB_.js → doctor-DGWbvYtu.js} +29 -29
- package/dist/{doctor-aE-RMbB_.js.map → doctor-DGWbvYtu.js.map} +1 -1
- package/dist/duration-Bz2H7pXb.js +34 -0
- package/dist/duration-Bz2H7pXb.js.map +1 -0
- package/dist/{environment-C8pUCD_A.js → environment-WC8PrLoe.js} +14 -11
- package/dist/environment-WC8PrLoe.js.map +1 -0
- package/dist/errors-BjzEWESY.js +93 -0
- package/dist/errors-BjzEWESY.js.map +1 -0
- package/dist/{errors-DPUoUTdK.js → errors-Cda7UzRA.js} +2 -2
- package/dist/{errors-DPUoUTdK.js.map → errors-Cda7UzRA.js.map} +1 -1
- package/dist/{eval-source-CaLIGH3U.js → eval-source-DL1HOi8U.js} +6 -6
- package/dist/{eval-source-CaLIGH3U.js.map → eval-source-DL1HOi8U.js.map} +1 -1
- package/dist/{evaluate-4kBVtwZZ.js → evaluate-DHnD8yzq.js} +280 -59
- package/dist/evaluate-DHnD8yzq.js.map +1 -0
- package/dist/{evaluate-DHaNtgVf.d.ts → evaluate-Nnh0iPuo.d.ts} +2 -3
- package/dist/{events-CXxFvVMK.js → events-oe84EOLK.js} +153 -34
- package/dist/{events-CXxFvVMK.js.map → events-oe84EOLK.js.map} +1 -1
- package/dist/{fake-bootstrap-CzxrCqO3.js → fake-bootstrap-Ct1qEs-Z.js} +2 -2
- package/dist/{fake-bootstrap-CzxrCqO3.js.map → fake-bootstrap-Ct1qEs-Z.js.map} +1 -1
- package/dist/files-2fO6iaJG.js +1 -0
- package/dist/{files-DpwJW3w6.js → files-4Qt4KUyR.js} +3 -3
- package/dist/{files-DpwJW3w6.js.map → files-4Qt4KUyR.js.map} +1 -1
- package/dist/{files-DgHeTdCZ.js → files-4XWe_gow.js} +27 -41
- package/dist/files-4XWe_gow.js.map +1 -0
- package/dist/files-63CNzRBF.js +1 -0
- package/dist/files-BDkCNIEh.js +1 -0
- package/dist/{files-CDfG8swM.js → files-Bv6ipFjc.js} +27 -18
- package/dist/files-Bv6ipFjc.js.map +1 -0
- package/dist/{files-Cs2dtb7-.js → files-ClUbQPj9.js} +2 -2
- package/dist/{files-Cs2dtb7-.js.map → files-ClUbQPj9.js.map} +1 -1
- package/dist/files-CzgXLYGq.js +1609 -0
- package/dist/files-CzgXLYGq.js.map +1 -0
- package/dist/files-DCcRkJN4.js +1 -0
- package/dist/{files-BkvxIqfI.js → files-DW8y3WCx.js} +5 -5
- package/dist/{files-BkvxIqfI.js.map → files-DW8y3WCx.js.map} +1 -1
- package/dist/gateway.d.ts +2 -3
- package/dist/{git-BnnNEXqh.js → git-CiCya2yZ.js} +3 -3
- package/dist/{git-BnnNEXqh.js.map → git-CiCya2yZ.js.map} +1 -1
- package/dist/{harness-FcTyyPco.js → harness-Dcd3ffkY.js} +3 -2
- package/dist/harness-Dcd3ffkY.js.map +1 -0
- package/dist/{harness-nwnyghJZ.js → harness-DvqNGwl6.js} +3 -3
- package/dist/{harness-nwnyghJZ.js.map → harness-DvqNGwl6.js.map} +1 -1
- package/dist/{harness-install-BfMBYDfP.js → harness-install-BSet5fE5.js} +4 -3
- package/dist/harness-install-BSet5fE5.js.map +1 -0
- package/dist/harness.d.ts +2 -3
- package/dist/harness.js +1 -1
- package/dist/http-DtlSzw4A.js +96 -0
- package/dist/http-DtlSzw4A.js.map +1 -0
- package/dist/{ids-a6cr243_.js → ids-CtxRn13a.js} +3 -2
- package/dist/ids-CtxRn13a.js.map +1 -0
- package/dist/{images-DtXwj2wL.js → images-BkJn7NHU.js} +13 -13
- package/dist/{images-DtXwj2wL.js.map → images-BkJn7NHU.js.map} +1 -1
- package/dist/index.d.ts +9 -5
- package/dist/index.js +4 -5
- package/dist/init-BOYNr3Ql.js +1 -0
- package/dist/{init-Ci_IAp8p.js → init-CAJvEKEt.js} +6 -6
- package/dist/{init-Ci_IAp8p.js.map → init-CAJvEKEt.js.map} +1 -1
- package/dist/{ink-list-Dnh6XD3C.js → ink-list-BJIKlKpN.js} +3 -3
- package/dist/{ink-list-Dnh6XD3C.js.map → ink-list-BJIKlKpN.js.map} +1 -1
- package/dist/{inputs-z4JqnQ5g.js → inputs-CjhDEzr4.js} +6 -6
- package/dist/{inputs-z4JqnQ5g.js.map → inputs-CjhDEzr4.js.map} +1 -1
- package/dist/inputs-load-C8HEtAej.js +1 -0
- package/dist/{inputs-load-DPUP_IFL.js → inputs-load-DQRC3dUA.js} +3 -3
- package/dist/{inputs-load-DPUP_IFL.js.map → inputs-load-DQRC3dUA.js.map} +1 -1
- package/dist/internal/vm-api/boat.d.ts +46 -0
- package/dist/internal/vm-api/boat.js +1 -0
- package/dist/internal/vm-api/daytona.d.ts +26 -0
- package/dist/internal/vm-api/daytona.js +220 -0
- package/dist/internal/vm-api/daytona.js.map +1 -0
- package/dist/internal/vm-api.d.ts +86 -0
- package/dist/internal/vm-api.js +3 -0
- package/dist/internal/vm.d.ts +7 -8
- package/dist/internal/vm.js +6 -6
- package/dist/internal/vm.js.map +1 -1
- package/dist/{invocations-C1gY3E0J.js → invocations-DUFiIb_S.js} +6 -6
- package/dist/{invocations-C1gY3E0J.js.map → invocations-DUFiIb_S.js.map} +1 -1
- package/dist/issue-BKMnnit3.js +120 -0
- package/dist/issue-BKMnnit3.js.map +1 -0
- package/dist/issue-esBvB5Df.js +1 -0
- package/dist/{jiti-ILwyxt93.js → jiti-aNfS_lSj.js} +2 -2
- package/dist/{jiti-ILwyxt93.js.map → jiti-aNfS_lSj.js.map} +1 -1
- package/dist/{judge-DwzqucLS.js → judge-_fsdF4YB.js} +7 -4
- package/dist/judge-_fsdF4YB.js.map +1 -0
- package/dist/judge.d.ts +5 -6
- package/dist/judge.js +1 -1
- package/dist/{judgments-Cj8bFaDG.js → judgments-CVCUWT-g.js} +8 -5
- package/dist/{judgments-Cj8bFaDG.js.map → judgments-CVCUWT-g.js.map} +1 -1
- package/dist/{kinds-DEKXOWVJ.js → kinds-5utDFJ15.js} +37 -12
- package/dist/kinds-5utDFJ15.js.map +1 -0
- package/dist/labels-Dtyf5-pd.js +251 -0
- package/dist/labels-Dtyf5-pd.js.map +1 -0
- package/dist/{layout-nCnU9Vu2.js → layout-Co4tPHWI.js} +3 -3
- package/dist/{layout-nCnU9Vu2.js.map → layout-Co4tPHWI.js.map} +1 -1
- package/dist/link-Dat_O7Gn.js +128 -0
- package/dist/link-Dat_O7Gn.js.map +1 -0
- package/dist/{list-MK4L8tfW.js → list-CjcNH-5e.js} +13 -8
- package/dist/list-CjcNH-5e.js.map +1 -0
- package/dist/live-BElpkw5b.js +56 -0
- package/dist/live-BElpkw5b.js.map +1 -0
- package/dist/{live-Dlq-S2yy.js → live-BRF0ogR6.js} +3 -5
- package/dist/live-BRF0ogR6.js.map +1 -0
- package/dist/{machine-service-Byl0BlcT.js → machine-service-Cymfk7E-.js} +1 -1
- package/dist/{machine-service-CZiXiVsS.js → machine-service-DYdTM_rW.js} +146 -13
- package/dist/machine-service-DYdTM_rW.js.map +1 -0
- package/dist/{machines-Ch5bhgvk.js → machines-B6rS9-Xu.js} +38 -38
- package/dist/{machines-Ch5bhgvk.js.map → machines-B6rS9-Xu.js.map} +1 -1
- package/dist/markdown-COts-jxf.js +653 -0
- package/dist/markdown-COts-jxf.js.map +1 -0
- package/dist/{model-lLaUxxno.js → model-BdLvfSQO.js} +1 -1
- package/dist/{model-DRa7yv6K.js → model-CIMBjBS7.js} +5 -5
- package/dist/{model-DRa7yv6K.js.map → model-CIMBjBS7.js.map} +1 -1
- package/dist/model-CrsA2Evf.js +1 -0
- package/dist/{model-CRh8aiYS.js → model-oBFmfzES.js} +4 -4
- package/dist/{model-CRh8aiYS.js.map → model-oBFmfzES.js.map} +1 -1
- package/dist/{model-LoLnq9y1.js → model-xemZQ7DK.js} +251 -65
- package/dist/model-xemZQ7DK.js.map +1 -0
- package/dist/{node-BQjSQw0d.js → node-DyRPGAT5.js} +19 -8
- package/dist/{node-BQjSQw0d.js.map → node-DyRPGAT5.js.map} +1 -1
- package/dist/options-CeIRZslO.js +1 -0
- package/dist/otlp-ehLWBhVL.js +123 -0
- package/dist/otlp-ehLWBhVL.js.map +1 -0
- package/dist/outcome-1LBk2e3w.js +38 -0
- package/dist/outcome-1LBk2e3w.js.map +1 -0
- package/dist/{patch-D8X6VB-e.js → patch-BgSw3TXR.js} +3 -3
- package/dist/{patch-D8X6VB-e.js.map → patch-BgSw3TXR.js.map} +1 -1
- package/dist/{patch-C7yBztZj.js → patch-uwo6t4Zy.js} +26 -2
- package/dist/patch-uwo6t4Zy.js.map +1 -0
- package/dist/{pi-providers-HygmL1y-.js → pi-providers-BjcQv2A1.js} +23 -9
- package/dist/{pi-providers-HygmL1y-.js.map → pi-providers-BjcQv2A1.js.map} +1 -1
- package/dist/{plain-Ctn8oUdp.js → plain-DXUNwqpO.js} +2 -2
- package/dist/{plain-Ctn8oUdp.js.map → plain-DXUNwqpO.js.map} +1 -1
- package/dist/{plan-DgIVIrXJ.js → plan-x-_vZMQH.js} +11 -11
- package/dist/{plan-DgIVIrXJ.js.map → plan-x-_vZMQH.js.map} +1 -1
- package/dist/{plugins-DvEbow6a.js → plugins-0DuNw_c5.js} +7 -7
- package/dist/{plugins-DvEbow6a.js.map → plugins-0DuNw_c5.js.map} +1 -1
- package/dist/{preflight-dJqs5x3r.js → preflight-0Wt3wufx.js} +1 -1
- package/dist/preflight-CKrvQ-of.js +1 -0
- package/dist/{preflight-BKGbxN67.js → preflight-CUc-Y1d3.js} +9 -9
- package/dist/{preflight-BKGbxN67.js.map → preflight-CUc-Y1d3.js.map} +1 -1
- package/dist/{preflight-Qm4wE094.js → preflight-Cce8YEPE.js} +4 -4
- package/dist/{preflight-Qm4wE094.js.map → preflight-Cce8YEPE.js.map} +1 -1
- package/dist/{prepare-CRPPfRJf.js → prepare-Bj8KzeBU.js} +2 -2
- package/dist/{prepare-BAjRSoku.js → prepare-FXc1YcxO.js} +8 -8
- package/dist/{prepare-BAjRSoku.js.map → prepare-FXc1YcxO.js.map} +1 -1
- package/dist/prices-CfnkJOth.js +1 -0
- package/dist/{prices-DCokI_Ka.js → prices-CoLymMRd.js} +3 -3
- package/dist/{prices-DCokI_Ka.js.map → prices-CoLymMRd.js.map} +1 -1
- package/dist/process-B86Nvm4J.js +118 -0
- package/dist/process-B86Nvm4J.js.map +1 -0
- package/dist/provenance-Cylt_YxT.d.ts +222 -0
- package/dist/{provenance-CSyRCc9B.js → provenance-DDZGWE4O.js} +6 -6
- package/dist/{provenance-CSyRCc9B.js.map → provenance-DDZGWE4O.js.map} +1 -1
- package/dist/provider-B24hZX95.js +1 -0
- package/dist/{provider-ieWOn4YS.js → provider-Cn7lmECy.js} +2 -2
- package/dist/{provider-ieWOn4YS.js.map → provider-Cn7lmECy.js.map} +1 -1
- package/dist/{provider-stop-CxzWW7Ft.js → provider-stop-DjBMfZW5.js} +2 -2
- package/dist/{provider-stop-CxzWW7Ft.js.map → provider-stop-DjBMfZW5.js.map} +1 -1
- package/dist/{providers-DxZ26YoG.js → providers-BPujoqQM.js} +1 -1
- package/dist/{providers-DkaEJvF5.js → providers-Cqb25Aq1.js} +35 -35
- package/dist/{providers-DkaEJvF5.js.map → providers-Cqb25Aq1.js.map} +1 -1
- package/dist/{query-BL66BOqL.js → query-C_lwW-ok.js} +2 -2
- package/dist/{query-BL66BOqL.js.map → query-C_lwW-ok.js.map} +1 -1
- package/dist/query-DzS1Z4bw.js +1 -0
- package/dist/{questions-boVPzGoC.js → questions-DRPM7pky.js} +19 -13
- package/dist/questions-DRPM7pky.js.map +1 -0
- package/dist/{recipe-qyIpP96r.js → recipe-CV_EifJk.js} +4 -4
- package/dist/{recipe-qyIpP96r.js.map → recipe-CV_EifJk.js.map} +1 -1
- package/dist/{record-D1_D1KFY.js → record-Cj3yWR0E.js} +50 -13
- package/dist/record-Cj3yWR0E.js.map +1 -0
- package/dist/{redact-t-L3VGy8.js → redact-DMueYS-v.js} +28 -6
- package/dist/redact-DMueYS-v.js.map +1 -0
- package/dist/{remarks-DqriPKxm.js → remarks-BPKGorBW.js} +4 -2
- package/dist/remarks-BPKGorBW.js.map +1 -0
- package/dist/{remarks-DEvk5f8v.js → remarks-Bded6YeV.js} +22 -12
- package/dist/{remarks-DEvk5f8v.js.map → remarks-Bded6YeV.js.map} +1 -1
- package/dist/{remote-CQoBRpOv.js → remote-DMuAUedR.js} +9 -7
- package/dist/{remote-CQoBRpOv.js.map → remote-DMuAUedR.js.map} +1 -1
- package/dist/renderer/ascii.d.ts +6 -0
- package/dist/renderer/ascii.js +7 -0
- package/dist/renderer/ascii.js.map +1 -0
- package/dist/renderer/markdown.d.ts +31 -0
- package/dist/renderer/markdown.js +1 -0
- package/dist/report-BSi-R8hC.js +941 -0
- package/dist/report-BSi-R8hC.js.map +1 -0
- package/dist/report-inputs-JhiwKAZH.js +113 -0
- package/dist/report-inputs-JhiwKAZH.js.map +1 -0
- package/dist/{resolve-Bfo8sjd-.js → resolve-BEltmYhD.js} +23 -7
- package/dist/resolve-BEltmYhD.js.map +1 -0
- package/dist/resolve-s6JzFZu0.js +1 -0
- package/dist/{resolve-D2RqmybX.js → resolve-usf6ejs_.js} +64 -19
- package/dist/resolve-usf6ejs_.js.map +1 -0
- package/dist/{results-BpxzyoMd.js → results-rYLTGa1F.js} +11 -7
- package/dist/{results-BpxzyoMd.js.map → results-rYLTGa1F.js.map} +1 -1
- package/dist/{resume-2ekm_9OB.js → resume-aGIZhM58.js} +7 -7
- package/dist/{resume-2ekm_9OB.js.map → resume-aGIZhM58.js.map} +1 -1
- package/dist/{retry-CAhpW1hH.js → retry-BNFAHTES.js} +5 -5
- package/dist/{retry-CAhpW1hH.js.map → retry-BNFAHTES.js.map} +1 -1
- package/dist/{root-BzeoyfGd.js → root-Co0jhOT5.js} +2 -2
- package/dist/{root-BzeoyfGd.js.map → root-Co0jhOT5.js.map} +1 -1
- package/dist/run-DSKRllpd.js +127 -0
- package/dist/run-DSKRllpd.js.map +1 -0
- package/dist/{client-B41UK150.js → runtime-B2zv9GPL.js} +525 -105
- package/dist/runtime-B2zv9GPL.js.map +1 -0
- package/dist/sandbox/boat.d.ts +4 -5
- package/dist/sandbox/boat.js +2 -2
- package/dist/sandbox/daytona.d.ts +4 -5
- package/dist/sandbox/daytona.js +1 -1
- package/dist/{sandbox-t-Q-8lA_.js → sandbox-DbTHBA7g.js} +3 -3
- package/dist/{sandbox-t-Q-8lA_.js.map → sandbox-DbTHBA7g.js.map} +1 -1
- package/dist/sandbox.d.ts +4 -5
- package/dist/sandbox.js +1 -1
- package/dist/secrets-file-BEScYC3S.js +100 -0
- package/dist/secrets-file-BEScYC3S.js.map +1 -0
- package/dist/secrets-file-take-BXfYMTM-.js +33 -0
- package/dist/secrets-file-take-BXfYMTM-.js.map +1 -0
- package/dist/server-BIqAFNjF.js +1 -0
- package/dist/server-Dc-V_-C1.js +204 -0
- package/dist/server-Dc-V_-C1.js.map +1 -0
- package/dist/{services-BG8xuU82.js → services-CHh83UDC.js} +4 -2
- package/dist/services-CHh83UDC.js.map +1 -0
- package/dist/shared-B3iLKAiu.d.ts +1 -0
- package/dist/{shared-DtZwIAfy.js → shared-bXaqVxk-.js} +5 -34
- package/dist/shared-bXaqVxk-.js.map +1 -0
- package/dist/skill-dirs-B7o83G5I.js +63 -0
- package/dist/skill-dirs-B7o83G5I.js.map +1 -0
- package/dist/{source-CrXM19HD.js → source-BodZTD70.js} +33 -18
- package/dist/source-BodZTD70.js.map +1 -0
- package/dist/source-BtdFbZLW.js +1 -0
- package/dist/spans-C3FlFC22.js +1 -0
- package/dist/spans-Dq3fo6HM.js +6 -0
- package/dist/spans-Dq3fo6HM.js.map +1 -0
- package/dist/spans-WvAmJF2z.js +72 -0
- package/dist/spans-WvAmJF2z.js.map +1 -0
- package/dist/{src-C6MsGyD6.js → src-AAdLbAn2.js} +1 -1
- package/dist/{src-VG2IRoT4.js → src-BbaLTc-p.js} +2 -2
- package/dist/src-BjiNaGf5.js +2 -0
- package/dist/{src-Cf4vRyea.js → src-BkkdN2w0.js} +2 -2
- package/dist/{src-C5rrQlt8.js → src-BzeiF5Lh.js} +197 -138
- package/dist/src-BzeiF5Lh.js.map +1 -0
- package/dist/{src-Ds1pmXTg.js → src-CfPSD_GS.js} +2 -2
- package/dist/{src-C-ChdnG6.js → src-ClMhIgi3.js} +2 -2
- package/dist/{src-C-ChdnG6.js.map → src-ClMhIgi3.js.map} +1 -1
- package/dist/{src-io_SKoZ5.js → src-D5i8gCa3.js} +2 -2
- package/dist/{src-Bb3k9Kn_.js → src-DDIV_RfE.js} +65 -10
- package/dist/{src-Bb3k9Kn_.js.map → src-DDIV_RfE.js.map} +1 -1
- package/dist/src-DXZXofCJ.js +1 -0
- package/dist/src-Db01fVzc.js +2 -0
- package/dist/{src-DwcTn4JY.js → src-DhZuOcgq.js} +5 -5
- package/dist/{src-DwcTn4JY.js.map → src-DhZuOcgq.js.map} +1 -1
- package/dist/{src-DwcCKsV0.js → src-DiBd7r1i.js} +3 -3
- package/dist/{src-DNWiU__5.js → src-Dm2YIeXn.js} +8 -8
- package/dist/{src-DNWiU__5.js.map → src-Dm2YIeXn.js.map} +1 -1
- package/dist/{src-c3ZQu7Uu.js → src-NYxoDS3b.js} +5 -5
- package/dist/{src-c3ZQu7Uu.js.map → src-NYxoDS3b.js.map} +1 -1
- package/dist/{src-C3OhA5pw.js → src-Y4AiZxLt.js} +5 -5
- package/dist/{src-C3OhA5pw.js.map → src-Y4AiZxLt.js.map} +1 -1
- package/dist/{src-Bf5RZlPi.js → src-YpmRtJmP.js} +2 -2
- package/dist/{src-Bf5RZlPi.js.map → src-YpmRtJmP.js.map} +1 -1
- package/dist/{ssh-pool-Dvfx5GuV.js → ssh-pool-B2c2vxzx.js} +20 -16
- package/dist/ssh-pool-B2c2vxzx.js.map +1 -0
- package/dist/ssh-pool-mcCYe6RO.js +1 -0
- package/dist/{start-COB07NpH.js → start-DyXCyHGX.js} +64 -12
- package/dist/start-DyXCyHGX.js.map +1 -0
- package/dist/{state-B319V1p8.js → state-CiR8Ema7.js} +21 -38
- package/dist/state-CiR8Ema7.js.map +1 -0
- package/dist/state-DBuFimCA.js +1 -0
- package/dist/store/local.d.ts +2 -3
- package/dist/store/s3.d.ts +2 -3
- package/dist/studio-share-C05twqpy.js +209 -0
- package/dist/studio-share-C05twqpy.js.map +1 -0
- package/dist/suite-fO-u8jlS.js +2 -0
- package/dist/{suite-D1tgXZxM.js → suite-oH7uiCRq.js} +108 -107
- package/dist/suite-oH7uiCRq.js.map +1 -0
- package/dist/summary-CTMWfnOF.js +75 -0
- package/dist/summary-CTMWfnOF.js.map +1 -0
- package/dist/{sync-mOG74RXH.js → sync-r8EvW6Hr.js} +6 -6
- package/dist/{sync-mOG74RXH.js.map → sync-r8EvW6Hr.js.map} +1 -1
- package/dist/{tart-DMppfbEm.js → tart-BUh58nxA.js} +37 -10
- package/dist/tart-BUh58nxA.js.map +1 -0
- package/dist/{tart-DmVuVsmX.js → tart-DNrumWZu.js} +1 -1
- package/dist/{tart-CzViW3WZ.js → tart-Duqp33Zh.js} +2 -2
- package/dist/{tart-CzViW3WZ.js.map → tart-Duqp33Zh.js.map} +1 -1
- package/dist/{tart-bake-Cva_qdir.js → tart-bake-ChrVpjuF.js} +8 -7
- package/dist/tart-bake-ChrVpjuF.js.map +1 -0
- package/dist/{tart-bake-DVIoXWr_.js → tart-bake-LwdUVOzS.js} +1 -1
- package/dist/{tart-cli-LzDud2Z6.js → tart-cli-Dv3dNC13.js} +3 -2
- package/dist/tart-cli-Dv3dNC13.js.map +1 -0
- package/dist/{tart-cli-D88c5YYJ.js → tart-cli-S388NU-r.js} +1 -1
- package/dist/{tart-hosts-CTRY5kQ2.js → tart-hosts-B-dQEGpc.js} +7 -6
- package/dist/tart-hosts-B-dQEGpc.js.map +1 -0
- package/dist/{tart-hosts-8N6Tt7Yb.js → tart-hosts-_nHGB4mC.js} +1 -1
- package/dist/{tart-images-JfHirU8W.js → tart-images-DAEYoC_M.js} +3 -3
- package/dist/{tart-images-JfHirU8W.js.map → tart-images-DAEYoC_M.js.map} +1 -1
- package/dist/testing.d.ts +13 -5
- package/dist/testing.js +4 -4
- package/dist/{tool-class-CncfCw03.js → tool-class-CB-m4PiF.js} +2 -2
- package/dist/{tool-class-CncfCw03.js.map → tool-class-CB-m4PiF.js.map} +1 -1
- package/dist/{trace-BpXByD7W.js → trace-CJQLlAbB.js} +2 -2
- package/dist/{trace-BpXByD7W.js.map → trace-CJQLlAbB.js.map} +1 -1
- package/dist/{traits-CCD8XsmI.js → traits-DPIffsN8.js} +2 -2
- package/dist/{traits-CCD8XsmI.js.map → traits-DPIffsN8.js.map} +1 -1
- package/dist/{transcript-BXWZiMuC.js → transcript-CW02A4s9.js} +3 -3
- package/dist/{transcript-BXWZiMuC.js.map → transcript-CW02A4s9.js.map} +1 -1
- package/dist/{resources-DqdwihhT.js → turn-error-Dmv_Muj7.js} +15 -2
- package/dist/turn-error-Dmv_Muj7.js.map +1 -0
- package/dist/{types-BDo8ttQo.d.ts → types-BZaItiui.d.ts} +209 -28
- package/dist/types-Buut5yGe.js +29 -0
- package/dist/types-Buut5yGe.js.map +1 -0
- package/dist/types-C48zuZbT.d.ts +156 -0
- package/dist/types-CYz4dqAd.d.ts +1366 -0
- package/dist/{ui-DUiVPbQ2.js → ui-CJvbt2Cc.js} +15 -14
- package/dist/ui-CJvbt2Cc.js.map +1 -0
- package/dist/vertex-D4H9bKHD.d.ts +2 -0
- package/dist/{vertex-BEWxHoxH.js → vertex-DQQGhUn7.js} +2 -2
- package/dist/{vertex-BEWxHoxH.js.map → vertex-DQQGhUn7.js.map} +1 -1
- package/dist/vertex-DtWe0Xf6.js +1 -0
- package/dist/view/index.d.ts +142 -810
- package/dist/view/index.js +17 -12
- package/dist/view/report/index.d.ts +217 -0
- package/dist/view/report/index.js +4 -0
- package/dist/{views-CiZGh3ws.js → views-CtnOlABJ.js} +6 -6
- package/dist/{views-CiZGh3ws.js.map → views-CtnOlABJ.js.map} +1 -1
- package/dist/views-crppDW_B.js +2 -0
- package/dist/{vm-container-BVn7pXfD.js → vm-container-DwroMAXW.js} +13 -10
- package/dist/vm-container-DwroMAXW.js.map +1 -0
- package/dist/{vm-images-CNk-Zl2a.js → vm-images-y0BcVyv3.js} +19 -21
- package/dist/vm-images-y0BcVyv3.js.map +1 -0
- package/dist/wait-CixM6W0G.js +65 -0
- package/dist/wait-CixM6W0G.js.map +1 -0
- package/dist/{worker-entry-DmXQ2Znu.js → worker-entry-8KHmPzr4.js} +7 -7
- package/dist/{worker-entry-DmXQ2Znu.js.map → worker-entry-8KHmPzr4.js.map} +1 -1
- package/dist/workspace-JlgoGyR0.js +1 -0
- package/dist/{workspace-wwvwEHzP.js → workspace-xCQ6VDR-.js} +9 -8
- package/dist/workspace-xCQ6VDR-.js.map +1 -0
- package/package.json +33 -9
- package/skills/reeval/SKILL.md +2 -2
- package/skills/reeval/references/cli.md +1 -1
- package/skills/reeval/references/config.md +1 -1
- package/skills/reeval/references/environments.md +1 -1
- package/skills/setup/SKILL.md +1 -1
- package/dist/agents-BztvTkfM.js.map +0 -1
- package/dist/analysis-BnG_1x_O.js.map +0 -1
- package/dist/app-files-6NHzghym.js +0 -10
- package/dist/app-files-6NHzghym.js.map +0 -1
- package/dist/assemble-Doa1m-HW.js.map +0 -1
- package/dist/assemble-XrZskSNj.js +0 -1
- package/dist/attach-BH2HdwH_.js.map +0 -1
- package/dist/attempt-host-8mQV8Uee.js.map +0 -1
- package/dist/boat-DKRH0Gv7.js +0 -1
- package/dist/boat-DKjqo_ie.js.map +0 -1
- package/dist/boat-images-CRN_U_eG.js.map +0 -1
- package/dist/bundle-DU2aQVeq.js.map +0 -1
- package/dist/capture-BzjNv0qP.js +0 -1
- package/dist/cases-CM-MQRQp.js.map +0 -1
- package/dist/cases-FvhbM8M5.js +0 -1
- package/dist/cli-DMHOwdVr.js.map +0 -1
- package/dist/cli-spec-D0wuUaj9.js.map +0 -1
- package/dist/client-B41UK150.js.map +0 -1
- package/dist/command-8GIHYGZf.js +0 -298
- package/dist/command-8GIHYGZf.js.map +0 -1
- package/dist/command-CAmnWLFL.js.map +0 -1
- package/dist/command-CXxckfsq.js.map +0 -1
- package/dist/command-ao0NZ8h0.js.map +0 -1
- package/dist/commands-B3lnIPj5.js +0 -1
- package/dist/commands-ttChCsAy.js.map +0 -1
- package/dist/config-myJT1Qjc.js.map +0 -1
- package/dist/context-DnzR9TNK.js +0 -120
- package/dist/context-DnzR9TNK.js.map +0 -1
- package/dist/daytona-Dfhwtr9P.js.map +0 -1
- package/dist/daytona-image-CnG6G00x.js.map +0 -1
- package/dist/daytona-image-yJK3c4EV.js +0 -1
- package/dist/db-5H95sFii.js.map +0 -1
- package/dist/db-Bbg1n_2M.js +0 -2
- package/dist/debug-CF4p4Lp_.js +0 -1
- package/dist/debug-CIXzB4wK.js.map +0 -1
- package/dist/environment-C8pUCD_A.js.map +0 -1
- package/dist/evaluate-4kBVtwZZ.js.map +0 -1
- package/dist/files-BFnkOAhn.js +0 -1
- package/dist/files-BgQxoDlN.js +0 -1
- package/dist/files-CDfG8swM.js.map +0 -1
- package/dist/files-CSe4VaMj.js +0 -1
- package/dist/files-CX_mmvCc.js +0 -775
- package/dist/files-CX_mmvCc.js.map +0 -1
- package/dist/files-D6M7J3iX.js +0 -1
- package/dist/files-DgHeTdCZ.js.map +0 -1
- package/dist/gate-CgYdaprG.js +0 -49
- package/dist/gate-CgYdaprG.js.map +0 -1
- package/dist/harness-FcTyyPco.js.map +0 -1
- package/dist/harness-install-BfMBYDfP.js.map +0 -1
- package/dist/hosted-5Zz2peTR.js.map +0 -1
- package/dist/hosted-Co-waHe0.js +0 -131
- package/dist/hosted-Co-waHe0.js.map +0 -1
- package/dist/hosted-run-202QicZ-.js +0 -1
- package/dist/hosted-run-BfAPIMyT.js +0 -230
- package/dist/hosted-run-BfAPIMyT.js.map +0 -1
- package/dist/ids-a6cr243_.js.map +0 -1
- package/dist/init-BiL6B3S4.js +0 -1
- package/dist/inputs-load-M-BGe7gd.js +0 -1
- package/dist/judge-DwzqucLS.js.map +0 -1
- package/dist/kinds-DEKXOWVJ.js.map +0 -1
- package/dist/list-MK4L8tfW.js.map +0 -1
- package/dist/live-BkPNXVmN.js +0 -120
- package/dist/live-BkPNXVmN.js.map +0 -1
- package/dist/live-Dlq-S2yy.js.map +0 -1
- package/dist/live-gZtkBwuo.js +0 -1
- package/dist/machine-service-CZiXiVsS.js.map +0 -1
- package/dist/model-C6aoJLg_.js +0 -1
- package/dist/model-LoLnq9y1.js.map +0 -1
- package/dist/options-I9I3FwMl.js +0 -1
- package/dist/pairs-BGb134Vu.js +0 -230
- package/dist/pairs-BGb134Vu.js.map +0 -1
- package/dist/patch-C7yBztZj.js.map +0 -1
- package/dist/paths-Dwmi7L4X.js +0 -75
- package/dist/paths-Dwmi7L4X.js.map +0 -1
- package/dist/preflight-fyV5nnea.js +0 -1
- package/dist/prices-C0gKS_8G.js +0 -1
- package/dist/provider-DHT20Gl8.js +0 -1
- package/dist/query-C9OKU1Vz.js +0 -1
- package/dist/questions-boVPzGoC.js.map +0 -1
- package/dist/record-D1_D1KFY.js.map +0 -1
- package/dist/redact-t-L3VGy8.js.map +0 -1
- package/dist/remarks-DqriPKxm.js.map +0 -1
- package/dist/resolve-BdpO8BN1.js +0 -1
- package/dist/resolve-Bfo8sjd-.js.map +0 -1
- package/dist/resolve-D2RqmybX.js.map +0 -1
- package/dist/resources-DqdwihhT.js.map +0 -1
- package/dist/runtime-C5fKSFCz.js +0 -457
- package/dist/runtime-C5fKSFCz.js.map +0 -1
- package/dist/server-BcLg9WzE.js +0 -284
- package/dist/server-BcLg9WzE.js.map +0 -1
- package/dist/server-Buav2-CG.js +0 -1
- package/dist/services-BG8xuU82.js.map +0 -1
- package/dist/shared-CHvO7eLi.d.ts +0 -1
- package/dist/shared-DtZwIAfy.js.map +0 -1
- package/dist/slug-IHh3f770.js.map +0 -1
- package/dist/source-CrXM19HD.js.map +0 -1
- package/dist/source-dvxkqYvk.js +0 -1
- package/dist/src-5dCrqbFd.js +0 -2
- package/dist/src-C5rrQlt8.js.map +0 -1
- package/dist/src-DEbNp-rx.js +0 -1
- package/dist/src-DrLbG1HM.js +0 -2
- package/dist/ssh-pool-BorCizYw.js +0 -1
- package/dist/ssh-pool-Dvfx5GuV.js.map +0 -1
- package/dist/start-COB07NpH.js.map +0 -1
- package/dist/state-B319V1p8.js.map +0 -1
- package/dist/state-C5oBvmoZ.js +0 -1
- package/dist/suite-BwJGOhCZ.js +0 -2
- package/dist/suite-D1tgXZxM.js.map +0 -1
- package/dist/summary-ClLns_wv.js +0 -169
- package/dist/summary-ClLns_wv.js.map +0 -1
- package/dist/tart-DMppfbEm.js.map +0 -1
- package/dist/tart-bake-Cva_qdir.js.map +0 -1
- package/dist/tart-cli-LzDud2Z6.js.map +0 -1
- package/dist/tart-hosts-CTRY5kQ2.js.map +0 -1
- package/dist/types-CbcnJxCy.js +0 -15
- package/dist/types-CbcnJxCy.js.map +0 -1
- package/dist/ui-DUiVPbQ2.js.map +0 -1
- package/dist/vertex-C7PI-WPc.d.ts +0 -3
- package/dist/vertex-DXfcXYKn.js +0 -1
- package/dist/views-B0qZR5Ef.js +0 -2
- package/dist/vm-container-BVn7pXfD.js.map +0 -1
- package/dist/vm-images-CNk-Zl2a.js.map +0 -1
- package/dist/workspace-CRgFdmgW.js +0 -1
- package/dist/workspace-wwvwEHzP.js.map +0 -1
|
@@ -0,0 +1,941 @@
|
|
|
1
|
+
import { n as dataProblem } from "./shared-bXaqVxk-.js";
|
|
2
|
+
import { a as readableAnalysis, i as fillDefaults, n as NewerSchemaError, t as MIGRATIONS } from "./record-Cj3yWR0E.js";
|
|
3
|
+
import "./core-BXYzOEyQ.js";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
// src/view/analysis-schema.ts
|
|
6
|
+
const confidenceSchema = z.enum([
|
|
7
|
+
"high",
|
|
8
|
+
"medium",
|
|
9
|
+
"low"
|
|
10
|
+
]);
|
|
11
|
+
const levelSchema = z.enum([
|
|
12
|
+
"high",
|
|
13
|
+
"medium",
|
|
14
|
+
"low"
|
|
15
|
+
]);
|
|
16
|
+
const anomalyKindSchema = z.enum([
|
|
17
|
+
"reeval-bug",
|
|
18
|
+
"eval-design",
|
|
19
|
+
"judge",
|
|
20
|
+
"noise",
|
|
21
|
+
"other"
|
|
22
|
+
]);
|
|
23
|
+
const actionKindSchema = z.enum([
|
|
24
|
+
"skill",
|
|
25
|
+
"docs",
|
|
26
|
+
"eval",
|
|
27
|
+
"judge",
|
|
28
|
+
"reeval",
|
|
29
|
+
"harness"
|
|
30
|
+
]);
|
|
31
|
+
const roleSchema = z.enum([
|
|
32
|
+
"finding",
|
|
33
|
+
"caveat",
|
|
34
|
+
"action"
|
|
35
|
+
]);
|
|
36
|
+
const attemptRefsSchema = z.array(z.object({
|
|
37
|
+
attemptId: z.string(),
|
|
38
|
+
path: z.string()
|
|
39
|
+
}));
|
|
40
|
+
const analysisClaimSchema = z.object({
|
|
41
|
+
text: z.string(),
|
|
42
|
+
cites: z.array(z.string()),
|
|
43
|
+
confidence: confidenceSchema,
|
|
44
|
+
quotes: z.array(z.string()).optional(),
|
|
45
|
+
unverifiedNumbers: z.array(z.string()).optional(),
|
|
46
|
+
evalId: z.string().optional(),
|
|
47
|
+
role: roleSchema.optional(),
|
|
48
|
+
attempts: attemptRefsSchema.optional()
|
|
49
|
+
});
|
|
50
|
+
const analysisAnomalySchema = z.object({
|
|
51
|
+
text: z.string(),
|
|
52
|
+
kind: anomalyKindSchema,
|
|
53
|
+
cites: z.array(z.string()),
|
|
54
|
+
unverifiedNumbers: z.array(z.string()).optional(),
|
|
55
|
+
evalId: z.string().optional(),
|
|
56
|
+
role: roleSchema.optional(),
|
|
57
|
+
attempts: attemptRefsSchema.optional()
|
|
58
|
+
});
|
|
59
|
+
const analysisActionSchema = z.object({
|
|
60
|
+
title: z.string(),
|
|
61
|
+
target: z.string(),
|
|
62
|
+
kind: actionKindSchema,
|
|
63
|
+
change: z.string(),
|
|
64
|
+
expectedEffect: z.string(),
|
|
65
|
+
verify: z.string().nullable(),
|
|
66
|
+
verifyProblem: z.string().optional(),
|
|
67
|
+
impact: levelSchema,
|
|
68
|
+
effort: levelSchema,
|
|
69
|
+
cites: z.array(z.string()),
|
|
70
|
+
evalId: z.string().optional(),
|
|
71
|
+
role: roleSchema.optional(),
|
|
72
|
+
attempts: attemptRefsSchema.optional()
|
|
73
|
+
});
|
|
74
|
+
const analysisBodySchema = z.object({
|
|
75
|
+
tldr: z.string(),
|
|
76
|
+
whatHappened: z.array(analysisClaimSchema),
|
|
77
|
+
results: z.array(analysisClaimSchema),
|
|
78
|
+
why: z.array(analysisClaimSchema),
|
|
79
|
+
anomalies: z.array(analysisAnomalySchema),
|
|
80
|
+
actions: z.array(analysisActionSchema),
|
|
81
|
+
confidence: z.object({
|
|
82
|
+
overall: confidenceSchema,
|
|
83
|
+
why: z.string()
|
|
84
|
+
})
|
|
85
|
+
});
|
|
86
|
+
const analysisValidationSchema = z.object({
|
|
87
|
+
citations: z.number(),
|
|
88
|
+
invalid: z.array(z.string()),
|
|
89
|
+
dropped: z.number(),
|
|
90
|
+
commands: z.number(),
|
|
91
|
+
invalidCommands: z.array(z.string()),
|
|
92
|
+
unverifiedNumbers: z.number()
|
|
93
|
+
});
|
|
94
|
+
const analysisScopeSchema = z.object({
|
|
95
|
+
kind: z.literal("run"),
|
|
96
|
+
head: z.string(),
|
|
97
|
+
base: z.string().nullable(),
|
|
98
|
+
evals: z.string().nullable()
|
|
99
|
+
});
|
|
100
|
+
/** A document's keys it doesn't list pass, unchecked: a newer reeval may have added them. */
|
|
101
|
+
const analysisDocSchema = z.object({
|
|
102
|
+
kind: z.literal("reeval-analysis"),
|
|
103
|
+
schemaVersion: z.literal(1),
|
|
104
|
+
scope: analysisScopeSchema,
|
|
105
|
+
anchor: z.string(),
|
|
106
|
+
inputs: z.object({
|
|
107
|
+
records: z.array(z.object({
|
|
108
|
+
runId: z.string(),
|
|
109
|
+
sha256: z.string()
|
|
110
|
+
})),
|
|
111
|
+
digestVersion: z.number(),
|
|
112
|
+
digestHash: z.string()
|
|
113
|
+
}),
|
|
114
|
+
promptVersion: z.string(),
|
|
115
|
+
model: z.string(),
|
|
116
|
+
resolvedModel: z.string().nullable(),
|
|
117
|
+
route: z.enum([
|
|
118
|
+
"direct",
|
|
119
|
+
"gateway",
|
|
120
|
+
"openrouter",
|
|
121
|
+
"vertex",
|
|
122
|
+
"fake"
|
|
123
|
+
]),
|
|
124
|
+
usage: z.object({
|
|
125
|
+
inputTokens: z.number().nullable(),
|
|
126
|
+
outputTokens: z.number().nullable()
|
|
127
|
+
}),
|
|
128
|
+
latencyMs: z.number(),
|
|
129
|
+
createdAt: z.string(),
|
|
130
|
+
author: z.string(),
|
|
131
|
+
validation: analysisValidationSchema,
|
|
132
|
+
analysis: analysisBodySchema,
|
|
133
|
+
attempts: z.record(z.string(), z.object({
|
|
134
|
+
runId: z.string(),
|
|
135
|
+
evalSlug: z.string(),
|
|
136
|
+
attempt: z.string()
|
|
137
|
+
})),
|
|
138
|
+
cited: z.record(z.string(), z.string()),
|
|
139
|
+
error: z.string().optional()
|
|
140
|
+
});
|
|
141
|
+
/**
|
|
142
|
+
* `value` as an analysis every reader can show: `readableAnalysis`'s migration and read defaults,
|
|
143
|
+
* then checked against `analysisDocSchema`. One that isn't an analysis, has no `schemaVersion`, or
|
|
144
|
+
* has a missing or mistyped field throws one sentence naming it; one from a newer reeval throws a
|
|
145
|
+
* `NewerSchemaError`.
|
|
146
|
+
*/
|
|
147
|
+
function parsedAnalysis(value, what = "This analysis") {
|
|
148
|
+
const v = value;
|
|
149
|
+
if (typeof v !== "object" || v === null || Array.isArray(v) || v.kind !== "reeval-analysis") throw new Error(`${what} isn't a reeval analysis: its kind isn't "reeval-analysis"`);
|
|
150
|
+
if (typeof v.schemaVersion !== "number") throw new Error(`${what} has no schemaVersion, which every reeval analysis has`);
|
|
151
|
+
const doc = readableAnalysis(value, what);
|
|
152
|
+
const problem = dataProblem(analysisDocSchema, doc, what, () => []);
|
|
153
|
+
if (problem) throw new Error(problem);
|
|
154
|
+
return doc;
|
|
155
|
+
}
|
|
156
|
+
// src/view/report/schema.ts
|
|
157
|
+
const figureSchema = z.strictObject({
|
|
158
|
+
value: z.number().nullable(),
|
|
159
|
+
text: z.string()
|
|
160
|
+
});
|
|
161
|
+
const deltaSchema = z.strictObject({
|
|
162
|
+
value: z.number().nullable(),
|
|
163
|
+
unit: z.enum(["pct", "pts"]),
|
|
164
|
+
text: z.string()
|
|
165
|
+
});
|
|
166
|
+
const intervalSchema = z.strictObject({
|
|
167
|
+
lo: z.number(),
|
|
168
|
+
hi: z.number(),
|
|
169
|
+
text: z.string()
|
|
170
|
+
});
|
|
171
|
+
const spreadSchema = z.strictObject({
|
|
172
|
+
n: z.number().int().nonnegative(),
|
|
173
|
+
median: figureSchema,
|
|
174
|
+
q1: figureSchema,
|
|
175
|
+
q3: figureSchema
|
|
176
|
+
});
|
|
177
|
+
const countSchema = z.strictObject({
|
|
178
|
+
k: z.number().int().nonnegative(),
|
|
179
|
+
n: z.number().int().nonnegative(),
|
|
180
|
+
text: z.string()
|
|
181
|
+
});
|
|
182
|
+
const rateSchema = z.strictObject({
|
|
183
|
+
k: z.number().int().nonnegative(),
|
|
184
|
+
n: z.number().int().nonnegative(),
|
|
185
|
+
rate: z.number().nullable(),
|
|
186
|
+
text: z.string(),
|
|
187
|
+
interval: intervalSchema.nullable()
|
|
188
|
+
});
|
|
189
|
+
const attemptRefSchema = z.strictObject({
|
|
190
|
+
id: z.string(),
|
|
191
|
+
key: z.string(),
|
|
192
|
+
href: z.string()
|
|
193
|
+
});
|
|
194
|
+
const armRefSchema = z.strictObject({
|
|
195
|
+
id: z.string(),
|
|
196
|
+
label: z.string()
|
|
197
|
+
});
|
|
198
|
+
const measureKeySchema = z.enum([
|
|
199
|
+
"tokens",
|
|
200
|
+
"toolCalls",
|
|
201
|
+
"turns",
|
|
202
|
+
"costUsd",
|
|
203
|
+
"agentMs",
|
|
204
|
+
"filesChanged",
|
|
205
|
+
"linesAdded",
|
|
206
|
+
"linesRemoved"
|
|
207
|
+
]);
|
|
208
|
+
const measureUnitSchema = z.enum([
|
|
209
|
+
"tokens",
|
|
210
|
+
"ms",
|
|
211
|
+
"usd",
|
|
212
|
+
"count",
|
|
213
|
+
"lines",
|
|
214
|
+
"files"
|
|
215
|
+
]);
|
|
216
|
+
const measureCellsSchema = z.strictObject({
|
|
217
|
+
tokens: spreadSchema.nullable(),
|
|
218
|
+
toolCalls: spreadSchema.nullable(),
|
|
219
|
+
turns: spreadSchema.nullable(),
|
|
220
|
+
costUsd: spreadSchema.nullable(),
|
|
221
|
+
agentMs: spreadSchema.nullable(),
|
|
222
|
+
filesChanged: spreadSchema.nullable(),
|
|
223
|
+
linesAdded: spreadSchema.nullable(),
|
|
224
|
+
linesRemoved: spreadSchema.nullable()
|
|
225
|
+
});
|
|
226
|
+
const quoteSchema = z.strictObject({
|
|
227
|
+
text: z.string(),
|
|
228
|
+
attempt: attemptRefSchema,
|
|
229
|
+
harness: z.string(),
|
|
230
|
+
arm: z.string().nullable(),
|
|
231
|
+
case: z.string().nullable(),
|
|
232
|
+
input: z.string().nullable()
|
|
233
|
+
});
|
|
234
|
+
const otherVariantSchema = z.strictObject({
|
|
235
|
+
variant: armRefSchema,
|
|
236
|
+
points: deltaSchema,
|
|
237
|
+
interval: intervalSchema,
|
|
238
|
+
rate: rateSchema
|
|
239
|
+
});
|
|
240
|
+
const tiedLeadSchema = z.strictObject({
|
|
241
|
+
measure: measureKeySchema,
|
|
242
|
+
baseline: figureSchema,
|
|
243
|
+
variant: figureSchema,
|
|
244
|
+
delta: deltaSchema
|
|
245
|
+
});
|
|
246
|
+
const compareStatusSchema = z.enum([
|
|
247
|
+
"regressed",
|
|
248
|
+
"fixed",
|
|
249
|
+
"same",
|
|
250
|
+
"gate",
|
|
251
|
+
"drift",
|
|
252
|
+
"added",
|
|
253
|
+
"removed",
|
|
254
|
+
"infra"
|
|
255
|
+
]);
|
|
256
|
+
const changeRowSchema = z.strictObject({
|
|
257
|
+
signal: z.string(),
|
|
258
|
+
kind: z.enum([
|
|
259
|
+
"outcome",
|
|
260
|
+
"assertion",
|
|
261
|
+
"judgment",
|
|
262
|
+
"choice",
|
|
263
|
+
"measure",
|
|
264
|
+
"observation"
|
|
265
|
+
]),
|
|
266
|
+
harness: z.string().nullable(),
|
|
267
|
+
prior: z.string(),
|
|
268
|
+
current: z.string(),
|
|
269
|
+
delta: deltaSchema.nullable(),
|
|
270
|
+
status: compareStatusSchema,
|
|
271
|
+
gate: z.boolean()
|
|
272
|
+
});
|
|
273
|
+
const headlinePrimarySchema = z.discriminatedUnion("kind", [
|
|
274
|
+
z.strictObject({
|
|
275
|
+
kind: z.literal("pass-rate"),
|
|
276
|
+
rate: rateSchema,
|
|
277
|
+
assertions: z.number().int().positive(),
|
|
278
|
+
harnesses: z.number().int().positive(),
|
|
279
|
+
outcome: z.enum([
|
|
280
|
+
"passed",
|
|
281
|
+
"found",
|
|
282
|
+
"used"
|
|
283
|
+
])
|
|
284
|
+
}),
|
|
285
|
+
z.strictObject({
|
|
286
|
+
kind: z.literal("lift"),
|
|
287
|
+
baseline: armRefSchema,
|
|
288
|
+
variant: armRefSchema,
|
|
289
|
+
points: deltaSchema,
|
|
290
|
+
interval: intervalSchema,
|
|
291
|
+
baselineRate: rateSchema,
|
|
292
|
+
variantRate: rateSchema,
|
|
293
|
+
others: z.array(otherVariantSchema)
|
|
294
|
+
}),
|
|
295
|
+
z.strictObject({
|
|
296
|
+
kind: z.literal("tied-ceiling"),
|
|
297
|
+
baseline: armRefSchema,
|
|
298
|
+
variant: armRefSchema,
|
|
299
|
+
rate: rateSchema,
|
|
300
|
+
assertions: z.number().int().positive(),
|
|
301
|
+
lead: tiedLeadSchema
|
|
302
|
+
}),
|
|
303
|
+
z.strictObject({
|
|
304
|
+
kind: z.literal("choice"),
|
|
305
|
+
question: z.string(),
|
|
306
|
+
options: z.array(z.strictObject({
|
|
307
|
+
label: z.string(),
|
|
308
|
+
count: z.number().int().nonnegative()
|
|
309
|
+
})).min(2),
|
|
310
|
+
attempts: z.number().int().positive(),
|
|
311
|
+
noPick: z.number().int().nonnegative()
|
|
312
|
+
}),
|
|
313
|
+
z.strictObject({
|
|
314
|
+
kind: z.literal("measure"),
|
|
315
|
+
label: z.string(),
|
|
316
|
+
unit: measureUnitSchema,
|
|
317
|
+
spread: spreadSchema
|
|
318
|
+
}),
|
|
319
|
+
z.strictObject({
|
|
320
|
+
kind: z.literal("judge"),
|
|
321
|
+
label: z.string(),
|
|
322
|
+
question: z.string(),
|
|
323
|
+
scale: z.tuple([z.number(), z.number()]),
|
|
324
|
+
threshold: z.number().nullable(),
|
|
325
|
+
spread: spreadSchema,
|
|
326
|
+
atOrAbove: countSchema.nullable()
|
|
327
|
+
}),
|
|
328
|
+
z.strictObject({
|
|
329
|
+
kind: z.literal("text"),
|
|
330
|
+
question: z.string(),
|
|
331
|
+
answered: countSchema,
|
|
332
|
+
lead: quoteSchema.nullable(),
|
|
333
|
+
leadRole: z.enum(["sole", "representative"]).nullable()
|
|
334
|
+
}),
|
|
335
|
+
z.strictObject({
|
|
336
|
+
kind: z.literal("regression"),
|
|
337
|
+
worse: z.number().int().nonnegative(),
|
|
338
|
+
fixed: z.number().int().nonnegative(),
|
|
339
|
+
lead: changeRowSchema.nullable()
|
|
340
|
+
})
|
|
341
|
+
]);
|
|
342
|
+
const fairnessSchema = z.enum([
|
|
343
|
+
"clear",
|
|
344
|
+
"within-noise",
|
|
345
|
+
"too-few-runs"
|
|
346
|
+
]);
|
|
347
|
+
const evalShapeSchema = z.enum([
|
|
348
|
+
"task",
|
|
349
|
+
"ab",
|
|
350
|
+
"discoverability",
|
|
351
|
+
"regression"
|
|
352
|
+
]);
|
|
353
|
+
const evalHeadlineSchema = z.strictObject({
|
|
354
|
+
task: z.string(),
|
|
355
|
+
kicker: z.string().nullable(),
|
|
356
|
+
fairness: fairnessSchema,
|
|
357
|
+
primary: headlinePrimarySchema,
|
|
358
|
+
foot: z.string()
|
|
359
|
+
});
|
|
360
|
+
const checkCellSchema = z.strictObject({
|
|
361
|
+
status: z.enum([
|
|
362
|
+
"pass",
|
|
363
|
+
"fail",
|
|
364
|
+
"mixed",
|
|
365
|
+
"na",
|
|
366
|
+
"error",
|
|
367
|
+
"not-run"
|
|
368
|
+
]),
|
|
369
|
+
passed: z.number().int().nonnegative(),
|
|
370
|
+
answered: z.number().int().nonnegative(),
|
|
371
|
+
notApplicable: z.number().int().nonnegative(),
|
|
372
|
+
preSatisfied: z.number().int().nonnegative(),
|
|
373
|
+
text: z.string(),
|
|
374
|
+
reason: z.string().nullable()
|
|
375
|
+
});
|
|
376
|
+
const assertionRowSchema = z.strictObject({
|
|
377
|
+
label: z.string(),
|
|
378
|
+
matcher: z.string(),
|
|
379
|
+
at: z.string().nullable(),
|
|
380
|
+
cells: z.record(z.string(), checkCellSchema),
|
|
381
|
+
total: checkCellSchema
|
|
382
|
+
});
|
|
383
|
+
const breakdownRowSchema = z.strictObject({
|
|
384
|
+
keys: z.array(z.string()),
|
|
385
|
+
rate: rateSchema.nullable(),
|
|
386
|
+
measures: measureCellsSchema
|
|
387
|
+
});
|
|
388
|
+
const taskResultsSchema = z.strictObject({
|
|
389
|
+
kind: z.literal("task"),
|
|
390
|
+
columns: z.array(z.string()),
|
|
391
|
+
assertions: z.array(assertionRowSchema),
|
|
392
|
+
axes: z.array(z.string()),
|
|
393
|
+
breakdown: z.array(breakdownRowSchema),
|
|
394
|
+
median: breakdownRowSchema.nullable()
|
|
395
|
+
});
|
|
396
|
+
const armSchema = z.strictObject({
|
|
397
|
+
id: z.string(),
|
|
398
|
+
label: z.string(),
|
|
399
|
+
environment: z.string(),
|
|
400
|
+
environmentDigest: z.string().nullable(),
|
|
401
|
+
role: z.enum(["baseline", "variant"]),
|
|
402
|
+
change: z.string().nullable(),
|
|
403
|
+
prompts: z.array(z.string())
|
|
404
|
+
});
|
|
405
|
+
const measureVsBaselineSchema = z.strictObject({
|
|
406
|
+
value: spreadSchema.nullable(),
|
|
407
|
+
delta: deltaSchema.nullable(),
|
|
408
|
+
lower: countSchema.nullable()
|
|
409
|
+
});
|
|
410
|
+
const measureVsBaselineRecordSchema = z.strictObject({
|
|
411
|
+
tokens: measureVsBaselineSchema,
|
|
412
|
+
toolCalls: measureVsBaselineSchema,
|
|
413
|
+
turns: measureVsBaselineSchema,
|
|
414
|
+
costUsd: measureVsBaselineSchema,
|
|
415
|
+
agentMs: measureVsBaselineSchema,
|
|
416
|
+
filesChanged: measureVsBaselineSchema,
|
|
417
|
+
linesAdded: measureVsBaselineSchema,
|
|
418
|
+
linesRemoved: measureVsBaselineSchema
|
|
419
|
+
});
|
|
420
|
+
const armRowSchema = z.strictObject({
|
|
421
|
+
arm: z.string(),
|
|
422
|
+
rate: rateSchema,
|
|
423
|
+
lift: deltaSchema.nullable(),
|
|
424
|
+
interval: intervalSchema.nullable(),
|
|
425
|
+
verdict: z.enum([
|
|
426
|
+
"baseline",
|
|
427
|
+
"helped",
|
|
428
|
+
"made-worse",
|
|
429
|
+
"within-noise",
|
|
430
|
+
"too-few-runs"
|
|
431
|
+
]),
|
|
432
|
+
verdictScope: z.literal("pass-rate"),
|
|
433
|
+
measures: measureVsBaselineRecordSchema
|
|
434
|
+
});
|
|
435
|
+
const effortPivotRowSchema = z.strictObject({
|
|
436
|
+
harness: z.string(),
|
|
437
|
+
arm: z.string(),
|
|
438
|
+
rate: rateSchema,
|
|
439
|
+
measures: measureVsBaselineRecordSchema
|
|
440
|
+
});
|
|
441
|
+
const behaviorRowSchema = z.strictObject({
|
|
442
|
+
label: z.string(),
|
|
443
|
+
matcher: z.string(),
|
|
444
|
+
cells: z.record(z.string(), countSchema.nullable())
|
|
445
|
+
});
|
|
446
|
+
const contrastRowSchema = z.strictObject({
|
|
447
|
+
from: z.string(),
|
|
448
|
+
to: z.string(),
|
|
449
|
+
measure: z.union([z.literal("pass-rate"), measureKeySchema]),
|
|
450
|
+
delta: deltaSchema,
|
|
451
|
+
interval: intervalSchema.nullable(),
|
|
452
|
+
verdict: z.enum([
|
|
453
|
+
"helped",
|
|
454
|
+
"made-worse",
|
|
455
|
+
"within-noise",
|
|
456
|
+
"too-few-runs",
|
|
457
|
+
"fewer",
|
|
458
|
+
"more"
|
|
459
|
+
]),
|
|
460
|
+
verdictScope: z.enum(["pass-rate", "measure"])
|
|
461
|
+
});
|
|
462
|
+
const variantComparisonSchema = z.strictObject({
|
|
463
|
+
kind: z.literal("comparison"),
|
|
464
|
+
arms: z.array(armSchema),
|
|
465
|
+
uniformLead: z.string().nullable(),
|
|
466
|
+
summary: z.array(armRowSchema),
|
|
467
|
+
assertionsByArm: z.array(assertionRowSchema),
|
|
468
|
+
behavior: z.array(behaviorRowSchema),
|
|
469
|
+
contrasts: z.array(contrastRowSchema),
|
|
470
|
+
effortByHarness: z.array(effortPivotRowSchema)
|
|
471
|
+
});
|
|
472
|
+
const regressionChangesSchema = z.strictObject({
|
|
473
|
+
kind: z.literal("regression"),
|
|
474
|
+
baseRunId: z.string(),
|
|
475
|
+
prior: z.strictObject({
|
|
476
|
+
runId: z.string(),
|
|
477
|
+
startedAt: z.string(),
|
|
478
|
+
gitSha: z.string().nullable(),
|
|
479
|
+
branch: z.string().nullable(),
|
|
480
|
+
author: z.string()
|
|
481
|
+
}),
|
|
482
|
+
whatChanged: z.array(z.string()),
|
|
483
|
+
rows: z.array(changeRowSchema),
|
|
484
|
+
unchanged: z.array(changeRowSchema)
|
|
485
|
+
});
|
|
486
|
+
const primaryBlockSchema = z.discriminatedUnion("kind", [
|
|
487
|
+
taskResultsSchema,
|
|
488
|
+
variantComparisonSchema,
|
|
489
|
+
regressionChangesSchema
|
|
490
|
+
]);
|
|
491
|
+
const choiceBreakdownSchema = z.strictObject({
|
|
492
|
+
question: z.string(),
|
|
493
|
+
options: z.array(z.strictObject({
|
|
494
|
+
label: z.string(),
|
|
495
|
+
count: z.number().int().nonnegative()
|
|
496
|
+
})),
|
|
497
|
+
noPick: z.number().int().nonnegative(),
|
|
498
|
+
rows: z.array(z.strictObject({
|
|
499
|
+
arm: z.string().nullable(),
|
|
500
|
+
prompt: z.string().nullable(),
|
|
501
|
+
harness: z.string(),
|
|
502
|
+
counts: z.record(z.string(), z.number().int().nonnegative()),
|
|
503
|
+
attempts: z.number().int().nonnegative(),
|
|
504
|
+
text: z.string()
|
|
505
|
+
})),
|
|
506
|
+
passByPick: z.array(z.strictObject({
|
|
507
|
+
option: z.string(),
|
|
508
|
+
rate: rateSchema
|
|
509
|
+
}))
|
|
510
|
+
});
|
|
511
|
+
const discoveryRateSchema = z.strictObject({
|
|
512
|
+
what: z.string(),
|
|
513
|
+
matcher: z.string(),
|
|
514
|
+
found: countSchema,
|
|
515
|
+
rows: z.array(z.strictObject({
|
|
516
|
+
keys: z.array(z.string()),
|
|
517
|
+
found: countSchema
|
|
518
|
+
})),
|
|
519
|
+
sources: z.array(z.strictObject({
|
|
520
|
+
source: z.enum([
|
|
521
|
+
"placed",
|
|
522
|
+
"skill-dir",
|
|
523
|
+
"elsewhere",
|
|
524
|
+
"mcp",
|
|
525
|
+
"web"
|
|
526
|
+
]),
|
|
527
|
+
count: z.number().int().nonnegative()
|
|
528
|
+
}))
|
|
529
|
+
});
|
|
530
|
+
const judgeRowSchema = z.strictObject({
|
|
531
|
+
label: z.string(),
|
|
532
|
+
question: z.string(),
|
|
533
|
+
assertion: z.string().nullable(),
|
|
534
|
+
threshold: z.number().nullable(),
|
|
535
|
+
keys: z.array(z.string()),
|
|
536
|
+
spread: spreadSchema,
|
|
537
|
+
atOrAbove: countSchema.nullable()
|
|
538
|
+
});
|
|
539
|
+
const pairwiseRowSchema = z.strictObject({
|
|
540
|
+
question: z.string(),
|
|
541
|
+
cut: z.enum(["within", "across"]),
|
|
542
|
+
a: z.string(),
|
|
543
|
+
b: z.string(),
|
|
544
|
+
aWins: z.number().int().nonnegative(),
|
|
545
|
+
bWins: z.number().int().nonnegative(),
|
|
546
|
+
ties: z.number().int().nonnegative(),
|
|
547
|
+
pairs: z.number().int().nonnegative(),
|
|
548
|
+
reasons: z.array(z.strictObject({
|
|
549
|
+
text: z.string(),
|
|
550
|
+
cites: z.tuple([attemptRefSchema, attemptRefSchema]),
|
|
551
|
+
input: z.string().nullable()
|
|
552
|
+
}))
|
|
553
|
+
});
|
|
554
|
+
const judgmentBreakdownSchema = z.strictObject({
|
|
555
|
+
axes: z.array(z.string()),
|
|
556
|
+
scores: z.array(judgeRowSchema),
|
|
557
|
+
pairwise: z.array(pairwiseRowSchema)
|
|
558
|
+
});
|
|
559
|
+
const qualitativeAnswersSchema = z.strictObject({
|
|
560
|
+
question: z.string(),
|
|
561
|
+
answered: countSchema,
|
|
562
|
+
themes: z.array(z.strictObject({
|
|
563
|
+
title: z.string(),
|
|
564
|
+
quotes: z.array(quoteSchema)
|
|
565
|
+
})).nullable(),
|
|
566
|
+
entries: z.array(quoteSchema)
|
|
567
|
+
});
|
|
568
|
+
const captureShotSchema = z.strictObject({
|
|
569
|
+
name: z.string(),
|
|
570
|
+
href: z.string().nullable(),
|
|
571
|
+
meta: z.string().nullable(),
|
|
572
|
+
attempt: attemptRefSchema,
|
|
573
|
+
status: z.number().nullable(),
|
|
574
|
+
pageErrors: z.number().nullable(),
|
|
575
|
+
consoleErrors: z.number().nullable()
|
|
576
|
+
});
|
|
577
|
+
const captureGallerySchema = z.strictObject({ rows: z.array(z.strictObject({
|
|
578
|
+
harness: z.string(),
|
|
579
|
+
arm: z.string().nullable(),
|
|
580
|
+
case: z.string().nullable(),
|
|
581
|
+
shots: z.array(captureShotSchema)
|
|
582
|
+
})) });
|
|
583
|
+
const failureClusterSchema = z.strictObject({
|
|
584
|
+
kind: z.enum([
|
|
585
|
+
"assertion",
|
|
586
|
+
"turn-error",
|
|
587
|
+
"infra-error",
|
|
588
|
+
"capture-error",
|
|
589
|
+
"error"
|
|
590
|
+
]),
|
|
591
|
+
label: z.string(),
|
|
592
|
+
at: z.string().nullable(),
|
|
593
|
+
count: countSchema,
|
|
594
|
+
attempts: z.array(attemptRefSchema)
|
|
595
|
+
});
|
|
596
|
+
const failureClustersSchema = z.strictObject({
|
|
597
|
+
clusters: z.array(failureClusterSchema),
|
|
598
|
+
remarks: z.array(z.strictObject({
|
|
599
|
+
kind: z.enum([
|
|
600
|
+
"too-few-tries",
|
|
601
|
+
"cant-tell-apart",
|
|
602
|
+
"check-never-fails",
|
|
603
|
+
"flaky-check",
|
|
604
|
+
"slow-mcp-start",
|
|
605
|
+
"proxy-only-egress",
|
|
606
|
+
"setup-process-secret",
|
|
607
|
+
"setup-tunnel-closed",
|
|
608
|
+
"snapshot-too-large",
|
|
609
|
+
"observe-throws"
|
|
610
|
+
]),
|
|
611
|
+
severity: z.enum(["concern", "suggestion"]),
|
|
612
|
+
text: z.string(),
|
|
613
|
+
fix: z.strictObject({
|
|
614
|
+
at: z.string(),
|
|
615
|
+
edit: z.string()
|
|
616
|
+
})
|
|
617
|
+
}))
|
|
618
|
+
});
|
|
619
|
+
const reportClaimSchema = z.strictObject({
|
|
620
|
+
role: z.enum([
|
|
621
|
+
"finding",
|
|
622
|
+
"caveat",
|
|
623
|
+
"action"
|
|
624
|
+
]),
|
|
625
|
+
text: z.string(),
|
|
626
|
+
confidence: z.enum([
|
|
627
|
+
"high",
|
|
628
|
+
"medium",
|
|
629
|
+
"low"
|
|
630
|
+
]),
|
|
631
|
+
cites: z.array(attemptRefSchema),
|
|
632
|
+
action: z.strictObject({
|
|
633
|
+
kind: z.enum([
|
|
634
|
+
"skill",
|
|
635
|
+
"docs",
|
|
636
|
+
"eval",
|
|
637
|
+
"judge",
|
|
638
|
+
"reeval",
|
|
639
|
+
"harness"
|
|
640
|
+
]),
|
|
641
|
+
target: z.string(),
|
|
642
|
+
change: z.string(),
|
|
643
|
+
verify: z.string().nullable()
|
|
644
|
+
}).nullable()
|
|
645
|
+
});
|
|
646
|
+
const evalSectionSchema = z.strictObject({
|
|
647
|
+
id: z.string(),
|
|
648
|
+
anchor: z.string(),
|
|
649
|
+
name: z.string(),
|
|
650
|
+
file: z.string(),
|
|
651
|
+
shapes: z.array(evalShapeSchema),
|
|
652
|
+
design: z.string(),
|
|
653
|
+
headline: evalHeadlineSchema,
|
|
654
|
+
insights: z.array(reportClaimSchema),
|
|
655
|
+
primary: primaryBlockSchema.nullable(),
|
|
656
|
+
choices: z.array(choiceBreakdownSchema),
|
|
657
|
+
discovery: z.array(discoveryRateSchema),
|
|
658
|
+
judgments: judgmentBreakdownSchema.nullable(),
|
|
659
|
+
qualitative: z.array(qualitativeAnswersSchema),
|
|
660
|
+
captures: captureGallerySchema.nullable(),
|
|
661
|
+
failures: failureClustersSchema.nullable()
|
|
662
|
+
});
|
|
663
|
+
const runTallySchema = z.strictObject({
|
|
664
|
+
counted: z.number().int().nonnegative(),
|
|
665
|
+
passed: z.number().int().nonnegative(),
|
|
666
|
+
failed: z.number().int().nonnegative(),
|
|
667
|
+
ungated: z.strictObject({
|
|
668
|
+
passed: z.number().int().nonnegative(),
|
|
669
|
+
failed: z.number().int().nonnegative()
|
|
670
|
+
}),
|
|
671
|
+
infraError: z.number().int().nonnegative(),
|
|
672
|
+
reclassified: z.number().int().nonnegative(),
|
|
673
|
+
skipped: z.number().int().nonnegative()
|
|
674
|
+
});
|
|
675
|
+
const runHeaderSchema = z.strictObject({
|
|
676
|
+
title: z.string(),
|
|
677
|
+
status: z.enum([
|
|
678
|
+
"complete",
|
|
679
|
+
"interrupted",
|
|
680
|
+
"budget"
|
|
681
|
+
]),
|
|
682
|
+
startedAt: z.string(),
|
|
683
|
+
author: z.string(),
|
|
684
|
+
reeval: z.string(),
|
|
685
|
+
profile: z.string().nullable(),
|
|
686
|
+
scope: z.strictObject({
|
|
687
|
+
profile: z.string().nullable(),
|
|
688
|
+
subset: z.boolean(),
|
|
689
|
+
configKey: z.string().regex(/^[0-9a-f]{64}$/)
|
|
690
|
+
}).optional(),
|
|
691
|
+
runsPerCell: z.number().int().positive().nullable(),
|
|
692
|
+
wall: figureSchema,
|
|
693
|
+
tally: runTallySchema,
|
|
694
|
+
tokens: z.strictObject({
|
|
695
|
+
total: figureSchema,
|
|
696
|
+
cacheRead: figureSchema
|
|
697
|
+
}).nullable(),
|
|
698
|
+
cost: z.strictObject({
|
|
699
|
+
agent: figureSchema.nullable(),
|
|
700
|
+
judge: figureSchema.nullable(),
|
|
701
|
+
total: figureSchema.nullable(),
|
|
702
|
+
basis: z.enum([
|
|
703
|
+
"reported",
|
|
704
|
+
"estimated",
|
|
705
|
+
"mixed"
|
|
706
|
+
])
|
|
707
|
+
}).nullable(),
|
|
708
|
+
git: z.strictObject({
|
|
709
|
+
sha: z.string(),
|
|
710
|
+
branch: z.string().nullable(),
|
|
711
|
+
dirty: z.boolean(),
|
|
712
|
+
repo: z.string().nullable()
|
|
713
|
+
}).nullable()
|
|
714
|
+
});
|
|
715
|
+
const integrityIssueSchema = z.strictObject({
|
|
716
|
+
kind: z.enum([
|
|
717
|
+
"run-incomplete",
|
|
718
|
+
"turn-error",
|
|
719
|
+
"pre-satisfied",
|
|
720
|
+
"fairness-floor",
|
|
721
|
+
"served-model-drift",
|
|
722
|
+
"unreported-usage",
|
|
723
|
+
"prompt-confounded",
|
|
724
|
+
"duplicate-environment",
|
|
725
|
+
"baseline-never-passes",
|
|
726
|
+
"dirty-worktree"
|
|
727
|
+
]),
|
|
728
|
+
severity: z.enum(["error", "warn"]),
|
|
729
|
+
text: z.string(),
|
|
730
|
+
evalId: z.string().nullable(),
|
|
731
|
+
cites: z.array(attemptRefSchema)
|
|
732
|
+
});
|
|
733
|
+
const executiveRollupSchema = z.strictObject({
|
|
734
|
+
lead: reportClaimSchema.nullable(),
|
|
735
|
+
rows: z.array(z.strictObject({
|
|
736
|
+
evalId: z.string(),
|
|
737
|
+
name: z.string(),
|
|
738
|
+
anchor: z.string(),
|
|
739
|
+
shapes: z.array(evalShapeSchema),
|
|
740
|
+
design: z.string(),
|
|
741
|
+
primary: z.string(),
|
|
742
|
+
medianTokens: figureSchema
|
|
743
|
+
})),
|
|
744
|
+
findings: z.array(reportClaimSchema)
|
|
745
|
+
});
|
|
746
|
+
const rosterRowSchema = z.strictObject({
|
|
747
|
+
harness: z.string(),
|
|
748
|
+
model: z.string().nullable(),
|
|
749
|
+
effort: z.string().nullable(),
|
|
750
|
+
servedModels: z.array(z.string()),
|
|
751
|
+
gateway: z.string().nullable(),
|
|
752
|
+
attempts: z.number().int().nonnegative()
|
|
753
|
+
});
|
|
754
|
+
const provenanceStateSchema = z.strictObject({
|
|
755
|
+
rows: z.array(z.strictObject({
|
|
756
|
+
field: z.string(),
|
|
757
|
+
value: z.string()
|
|
758
|
+
})),
|
|
759
|
+
context: z.strictObject({
|
|
760
|
+
count: z.number().int().nonnegative(),
|
|
761
|
+
files: z.array(z.strictObject({
|
|
762
|
+
path: z.string(),
|
|
763
|
+
sha256: z.string()
|
|
764
|
+
}))
|
|
765
|
+
}),
|
|
766
|
+
artifacts: z.enum([
|
|
767
|
+
"all",
|
|
768
|
+
"evidence",
|
|
769
|
+
"failures",
|
|
770
|
+
"none"
|
|
771
|
+
]),
|
|
772
|
+
record: z.string(),
|
|
773
|
+
warnings: z.array(z.string())
|
|
774
|
+
});
|
|
775
|
+
const attemptFactsSchema = z.strictObject({
|
|
776
|
+
toolCalls: z.number().int().nonnegative(),
|
|
777
|
+
byClass: z.strictObject({
|
|
778
|
+
context: z.number().int().nonnegative(),
|
|
779
|
+
dependency: z.number().int().nonnegative(),
|
|
780
|
+
web: z.number().int().nonnegative(),
|
|
781
|
+
verify: z.number().int().nonnegative(),
|
|
782
|
+
workspace: z.number().int().nonnegative()
|
|
783
|
+
}),
|
|
784
|
+
skills: z.array(z.strictObject({
|
|
785
|
+
skill: z.string(),
|
|
786
|
+
source: z.enum([
|
|
787
|
+
"placed",
|
|
788
|
+
"skill-dir",
|
|
789
|
+
"elsewhere"
|
|
790
|
+
]),
|
|
791
|
+
reads: z.number().int().positive(),
|
|
792
|
+
firstLine: z.number().int().positive()
|
|
793
|
+
})),
|
|
794
|
+
contextFiles: z.array(z.strictObject({
|
|
795
|
+
path: z.string(),
|
|
796
|
+
reads: z.number().int().positive(),
|
|
797
|
+
firstLine: z.number().int().positive()
|
|
798
|
+
}))
|
|
799
|
+
});
|
|
800
|
+
const attemptIndexRowSchema = z.strictObject({
|
|
801
|
+
id: z.string(),
|
|
802
|
+
key: z.string(),
|
|
803
|
+
evalId: z.string(),
|
|
804
|
+
harness: z.string(),
|
|
805
|
+
arm: z.string().nullable(),
|
|
806
|
+
case: z.string().nullable(),
|
|
807
|
+
outcome: z.enum([
|
|
808
|
+
"pass",
|
|
809
|
+
"fail",
|
|
810
|
+
"infra_error",
|
|
811
|
+
"canceled"
|
|
812
|
+
]),
|
|
813
|
+
reclassified: z.boolean(),
|
|
814
|
+
turnError: z.string().nullable(),
|
|
815
|
+
tokens: figureSchema,
|
|
816
|
+
agentMs: figureSchema,
|
|
817
|
+
toolCalls: figureSchema,
|
|
818
|
+
facts: attemptFactsSchema.optional(),
|
|
819
|
+
diff: z.strictObject({
|
|
820
|
+
files: z.number().int().nonnegative(),
|
|
821
|
+
added: z.number().int().nullable(),
|
|
822
|
+
removed: z.number().int().nullable(),
|
|
823
|
+
text: z.string()
|
|
824
|
+
}).nullable(),
|
|
825
|
+
links: z.strictObject({
|
|
826
|
+
transcript: z.string().nullable(),
|
|
827
|
+
diff: z.string().nullable(),
|
|
828
|
+
captures: z.array(z.strictObject({
|
|
829
|
+
name: z.string(),
|
|
830
|
+
href: z.string()
|
|
831
|
+
})),
|
|
832
|
+
questions: z.array(z.string())
|
|
833
|
+
}),
|
|
834
|
+
href: z.string()
|
|
835
|
+
});
|
|
836
|
+
const runReportSchema = z.strictObject({
|
|
837
|
+
reportVersion: z.literal(1),
|
|
838
|
+
runId: z.string(),
|
|
839
|
+
single: z.boolean(),
|
|
840
|
+
header: runHeaderSchema,
|
|
841
|
+
integrity: z.array(integrityIssueSchema),
|
|
842
|
+
rollup: executiveRollupSchema.nullable(),
|
|
843
|
+
roster: z.array(rosterRowSchema),
|
|
844
|
+
evals: z.array(evalSectionSchema).min(1, "must hold at least one eval"),
|
|
845
|
+
actions: z.array(reportClaimSchema),
|
|
846
|
+
provenance: provenanceStateSchema,
|
|
847
|
+
attempts: z.array(attemptIndexRowSchema)
|
|
848
|
+
});
|
|
849
|
+
const LEAVES = /* @__PURE__ */ new Set([
|
|
850
|
+
"string",
|
|
851
|
+
"number",
|
|
852
|
+
"boolean",
|
|
853
|
+
"literal",
|
|
854
|
+
"enum",
|
|
855
|
+
"null",
|
|
856
|
+
"unknown"
|
|
857
|
+
]);
|
|
858
|
+
const loosened = /* @__PURE__ */ new WeakMap();
|
|
859
|
+
/**
|
|
860
|
+
* `schema` with every object in it, at any depth, accepting keys it doesn't list (kept, not
|
|
861
|
+
* stripped). Each node is a clone of the strict one with only its children and `catchall`
|
|
862
|
+
* replaced, so checks such as `.min(1)` hold in both.
|
|
863
|
+
*/
|
|
864
|
+
function loose(schema) {
|
|
865
|
+
const known = loosened.get(schema);
|
|
866
|
+
if (known) return known;
|
|
867
|
+
const def = schema.def;
|
|
868
|
+
let out;
|
|
869
|
+
switch (def.type) {
|
|
870
|
+
case "object":
|
|
871
|
+
out = schema.clone({
|
|
872
|
+
...def,
|
|
873
|
+
shape: Object.fromEntries(Object.entries(def.shape).map(([k, v]) => [k, loose(v)])),
|
|
874
|
+
catchall: z.unknown()
|
|
875
|
+
});
|
|
876
|
+
break;
|
|
877
|
+
case "array":
|
|
878
|
+
out = schema.clone({
|
|
879
|
+
...def,
|
|
880
|
+
element: loose(def.element)
|
|
881
|
+
});
|
|
882
|
+
break;
|
|
883
|
+
case "nullable":
|
|
884
|
+
case "optional":
|
|
885
|
+
out = schema.clone({
|
|
886
|
+
...def,
|
|
887
|
+
innerType: loose(def.innerType)
|
|
888
|
+
});
|
|
889
|
+
break;
|
|
890
|
+
case "union":
|
|
891
|
+
out = schema.clone({
|
|
892
|
+
...def,
|
|
893
|
+
options: def.options.map(loose)
|
|
894
|
+
});
|
|
895
|
+
break;
|
|
896
|
+
case "tuple":
|
|
897
|
+
out = schema.clone({
|
|
898
|
+
...def,
|
|
899
|
+
items: def.items.map(loose),
|
|
900
|
+
rest: def.rest && loose(def.rest)
|
|
901
|
+
});
|
|
902
|
+
break;
|
|
903
|
+
case "record":
|
|
904
|
+
out = schema.clone({
|
|
905
|
+
...def,
|
|
906
|
+
valueType: loose(def.valueType)
|
|
907
|
+
});
|
|
908
|
+
break;
|
|
909
|
+
default:
|
|
910
|
+
if (!LEAVES.has(def.type)) throw new Error(`loose(): no rule for a Zod ${def.type} schema`);
|
|
911
|
+
out = schema;
|
|
912
|
+
}
|
|
913
|
+
loosened.set(schema, out);
|
|
914
|
+
return out;
|
|
915
|
+
}
|
|
916
|
+
/** `runReportSchema` with unknown fields allowed at every level, for reading a stored report. */
|
|
917
|
+
const looseRunReportSchema = loose(runReportSchema);
|
|
918
|
+
/**
|
|
919
|
+
* A stored `report.json` as a reader sees it: migrated to this reeval's `reportVersion`, each field
|
|
920
|
+
* added after it was written read as its default (`READ_DEFAULTS['run-report']`), and fields a
|
|
921
|
+
* newer reeval added, at any depth, passed through. A missing or mistyped field throws one sentence
|
|
922
|
+
* naming it (`This report's header.title is missing: …`), and a `reportVersion` above this reeval's
|
|
923
|
+
* a `NewerSchemaError`.
|
|
924
|
+
*/
|
|
925
|
+
function readableRunReport(value, what = "This report") {
|
|
926
|
+
let version = value?.reportVersion;
|
|
927
|
+
if (typeof version === "number" && version > 1) throw new NewerSchemaError(what, version, 1, "reportVersion");
|
|
928
|
+
let v = value;
|
|
929
|
+
for (; typeof version === "number" && version < 1; version++) {
|
|
930
|
+
const migrate = MIGRATIONS["run-report"]?.[version];
|
|
931
|
+
if (!migrate) break;
|
|
932
|
+
v = migrate(v);
|
|
933
|
+
}
|
|
934
|
+
const filled = fillDefaults("run-report", v);
|
|
935
|
+
const parsed = looseRunReportSchema.safeParse(filled);
|
|
936
|
+
if (parsed.success) return parsed.data;
|
|
937
|
+
throw new Error(dataProblem(looseRunReportSchema, filled, what, () => []));
|
|
938
|
+
}
|
|
939
|
+
export { headlinePrimarySchema as a, rateSchema as c, runReportSchema as d, spreadSchema as f, figureSchema as i, readableRunReport as l, parsedAnalysis as m, countSchema as n, intervalSchema as o, analysisDocSchema as p, deltaSchema as r, looseRunReportSchema as s, attemptRefSchema as t, reportClaimSchema as u };
|
|
940
|
+
|
|
941
|
+
//# sourceMappingURL=report-BSi-R8hC.js.map
|