reeval 0.14.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/THIRD_PARTY_NOTICES.md +69 -69
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-BloGVZ_a.js → adapter-CRKYPkRt.js} +37 -8
- package/dist/adapter-CRKYPkRt.js.map +1 -0
- package/dist/{agent-m-_t-IuW.js → agent-BgH4-kFc.js} +58 -16
- package/dist/agent-BgH4-kFc.js.map +1 -0
- package/dist/{app-files-C-jwhRDt.js → app-files-BMVBr079.js} +4 -4
- package/dist/app-files-BMVBr079.js.map +1 -0
- package/dist/{assemble-BDEpelpi.js → assemble-BVbI6NW-.js} +214 -128
- package/dist/assemble-BVbI6NW-.js.map +1 -0
- package/dist/assemble-SiHa4__F.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-CpbZ9arV.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-CpbZ9arV.js.map} +1 -1
- package/dist/{match-Bz0nSdzN.js → attempt-host-CIxjs2Be.js} +104 -6
- package/dist/attempt-host-CIxjs2Be.js.map +1 -0
- package/dist/{capture-CVRwRTnb.js → capture-iQ6kYJV0.js} +2360 -1523
- package/dist/capture-iQ6kYJV0.js.map +1 -0
- package/dist/capture-px7Ey95_.js +1 -0
- package/dist/{cli-C_4gaJ50.js → cli-1YQU-66F.js} +153 -180
- package/dist/cli-1YQU-66F.js.map +1 -0
- package/dist/{cli-spec-CpnbATtI.js → cli-spec-CPWT1mAG.js} +10 -21
- package/dist/cli-spec-CPWT1mAG.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-D5hNjpEx.js → client-DMmOkbRA.js} +14 -7
- package/dist/client-DMmOkbRA.js.map +1 -0
- package/dist/{command-BXLUfw4h.js → command-8_Cg5v6b.js} +35 -22
- package/dist/command-8_Cg5v6b.js.map +1 -0
- package/dist/{command-I1euLLio.js → command-BlYCAqu-.js} +7 -7
- package/dist/{command-I1euLLio.js.map → command-BlYCAqu-.js.map} +1 -1
- package/dist/{validate-BhwqrVsm.js → command-C-HeGlap.js} +388 -60
- package/dist/command-C-HeGlap.js.map +1 -0
- package/dist/{command-Bxdy62dW.js → command-Cr1zjRvl.js} +32 -31
- package/dist/command-Cr1zjRvl.js.map +1 -0
- package/dist/{commands-p1S4nw_H.js → commands-Bj9NbmsO.js} +1 -1
- package/dist/{commands-jzigbNFb.js → commands-BvTNZ2-b.js} +181 -73
- package/dist/commands-BvTNZ2-b.js.map +1 -0
- package/dist/{config-BrdHA9bJ.js → config-CneBZB84.js} +883 -433
- package/dist/config-CneBZB84.js.map +1 -0
- package/dist/{contrasts-DRxMQQTC.js → contrasts-lhFsekco.js} +3 -3
- package/dist/{contrasts-DRxMQQTC.js.map → contrasts-lhFsekco.js.map} +1 -1
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/{cost-D4elx933.js → cost-ChPa8-Fp.js} +3 -3
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-BnXyamuj.js +2 -0
- package/dist/db-Cnh3jIv_.js +1530 -0
- package/dist/db-Cnh3jIv_.js.map +1 -0
- package/dist/debug-CAx3qcaP.js +1 -0
- package/dist/{debug-D24NOQH1.js → debug-CgG3T071.js} +5 -2
- package/dist/debug-CgG3T071.js.map +1 -0
- package/dist/{docker-BnRlTpZA.js → docker-BihzJwbV.js} +11 -6
- package/dist/docker-BihzJwbV.js.map +1 -0
- package/dist/{docker-D01slDF3.js → docker-Cqq03H72.js} +26 -8
- package/dist/docker-Cqq03H72.js.map +1 -0
- package/dist/{docker-DivfZl2u.js → docker-DCAtFMo9.js} +1 -1
- package/dist/{doctor-DQ2p0euG.js → doctor-Dl4vMeqt.js} +275 -37
- package/dist/doctor-Dl4vMeqt.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-CO0P0soF.js → environment-Cf5HMjib.js} +116 -297
- package/dist/environment-Cf5HMjib.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-CHebohjc.js} +10 -4
- package/dist/{errors-r025B0Zb.js.map → errors-CHebohjc.js.map} +1 -1
- package/dist/eval-source-CAE4y2Sr.js +146 -0
- package/dist/eval-source-CAE4y2Sr.js.map +1 -0
- package/dist/{evaluate-BUmsN7yG.js → evaluate-CUtgyr43.js} +682 -150
- package/dist/evaluate-CUtgyr43.js.map +1 -0
- package/dist/evaluate-D35T51AC.d.ts +45 -0
- package/dist/files-BEnUJW1e.js +1 -0
- package/dist/{files-B-CJeeEh.js → files-BRViE7m-.js} +200 -87
- package/dist/files-BRViE7m-.js.map +1 -0
- package/dist/{files-kUCrswI6.js → files-BSqA2sZE.js} +20 -9
- package/dist/files-BSqA2sZE.js.map +1 -0
- package/dist/files-C3tQtD26.js +1 -0
- package/dist/files-CNiJ_fZf.js +37 -0
- package/dist/files-CNiJ_fZf.js.map +1 -0
- package/dist/{files-Ca0d6Uq3.js → files-D1y8Pg4Z.js} +35 -7
- package/dist/files-D1y8Pg4Z.js.map +1 -0
- package/dist/files-Dts7EY3H.js +1 -0
- package/dist/{pairs-CHfZNRT6.js → gate-BC6SdQZs.js} +58 -4
- package/dist/gate-BC6SdQZs.js.map +1 -0
- package/dist/gateway.d.ts +22 -9
- package/dist/gateway.js +30 -8
- package/dist/gateway.js.map +1 -1
- package/dist/{git-Bk8DSwHy.js → git-UNxT_3z0.js} +3 -3
- package/dist/{git-Bk8DSwHy.js.map → git-UNxT_3z0.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +12 -8
- package/dist/harness.js +1 -1
- package/dist/{images-DzLeFjLx.js → images-8A_LzVfw.js} +8 -8
- package/dist/{images-DzLeFjLx.js.map → images-8A_LzVfw.js.map} +1 -1
- package/dist/index.d.ts +99 -11
- package/dist/index.js +5 -5
- package/dist/init-okoALIah.js +1 -0
- package/dist/{init-SRalM5xF.js → init-sFBgODhu.js} +10 -11
- package/dist/init-sFBgODhu.js.map +1 -0
- package/dist/{invocations-DzyKSJBa.js → invocations-C9ejlOYl.js} +10 -6
- package/dist/invocations-C9ejlOYl.js.map +1 -0
- package/dist/{judge-yLvjC5A3.js → judge-Bg4v4e_K.js} +236 -46
- package/dist/judge-Bg4v4e_K.js.map +1 -0
- package/dist/judge.d.ts +5 -7
- package/dist/judge.js +4 -8
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-DeMvnDD5.js} +3 -3
- package/dist/{live-DYd2QEyR.js.map → live-DeMvnDD5.js.map} +1 -1
- package/dist/{matchers-D4eUqx4j.js → matchers-BKPuaXlj.js} +14 -13
- package/dist/matchers-BKPuaXlj.js.map +1 -0
- package/dist/{model-1ZLr9Dcv.js → model-2rBZrdZA.js} +29 -36
- package/dist/model-2rBZrdZA.js.map +1 -0
- package/dist/{model-B45lDUV1.js → model-3MJtpho8.js} +22 -13
- package/dist/model-3MJtpho8.js.map +1 -0
- package/dist/{model-CgnOpss4.js → model-Bz7-j71_.js} +245 -68
- package/dist/model-Bz7-j71_.js.map +1 -0
- package/dist/model-DkPwypN4.js +1 -0
- package/dist/model-bREXVXwT.js +1 -0
- package/dist/{node-BSq5qHnQ.js → node-fP-H2uxn.js} +72 -50
- package/dist/node-fP-H2uxn.js.map +1 -0
- package/dist/{object-CLf2-tNq.js → object-BkXGPKWY.js} +3 -2
- package/dist/object-BkXGPKWY.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js +34 -0
- package/dist/openrouter-models-Bo8PZ4Ew.js.map +1 -0
- package/dist/{patch-DWf-I_Q_.js → patch-6T9XAipm.js} +7 -9
- package/dist/patch-6T9XAipm.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{plan-BIwUzlB8.js → plan-Dw2BjafJ.js} +79 -35
- package/dist/plan-Dw2BjafJ.js.map +1 -0
- package/dist/{plugins-Dkp_H3pB.js → plugins-ngQEJBjJ.js} +274 -17
- package/dist/plugins-ngQEJBjJ.js.map +1 -0
- package/dist/{preflight-D9y-HvZQ.js → preflight-BYXkweNN.js} +1 -1
- package/dist/{preflight-DJCD_BDn.js → preflight-CDRpog9o.js} +3 -3
- package/dist/{preflight-DJCD_BDn.js.map → preflight-CDRpog9o.js.map} +1 -1
- package/dist/{prepare-BJG_pD54.js → prepare-B3btUlT6.js} +2 -2
- package/dist/{prepare-CIxCGG0M.js → prepare-PkARUGoF.js} +8 -8
- package/dist/{prepare-CIxCGG0M.js.map → prepare-PkARUGoF.js.map} +1 -1
- package/dist/{prices-BcLEXvdY.js → prices-QaS-kjSQ.js} +7 -4
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-CfZpFpAh.js → provenance-7Hzg88f4.js} +4 -4
- package/dist/{provenance-CfZpFpAh.js.map → provenance-7Hzg88f4.js.map} +1 -1
- package/dist/{providers-D1Zxp2he.js → providers-DDNMbv1T.js} +1 -1
- package/dist/{providers-DFk_6W4K.js → providers-PKoJtGK0.js} +5 -5
- package/dist/{providers-DFk_6W4K.js.map → providers-PKoJtGK0.js.map} +1 -1
- package/dist/query-Bt6vLJ3m.js +1 -0
- package/dist/{query-CemclveB.js → query-nBXrgEwl.js} +3 -3
- package/dist/{query-CemclveB.js.map → query-nBXrgEwl.js.map} +1 -1
- package/dist/{questions-bAudIc04.js → questions-CkAwePPR.js} +110 -36
- package/dist/questions-CkAwePPR.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-Bp6AwPgb.js +1 -0
- package/dist/{rules-VhftPOOT.js → remarks-Bt0zwiQB.js} +330 -66
- package/dist/remarks-Bt0zwiQB.js.map +1 -0
- package/dist/{header-BLPP6aCU.js → remarks-IFFe_oty.js} +35 -9
- package/dist/remarks-IFFe_oty.js.map +1 -0
- package/dist/{remarks-mujJv1Ey.js → remarks-eBcrIZRu.js} +10 -5
- package/dist/remarks-eBcrIZRu.js.map +1 -0
- package/dist/{remote-CKcEKja4.js → remote-DfJYiP-R.js} +39 -99
- package/dist/remote-DfJYiP-R.js.map +1 -0
- package/dist/resolve-CT4XZIfV.js +1 -0
- package/dist/{resolve-CBoKzQMP.js → resolve-D_cLa4tW.js} +46 -30
- package/dist/resolve-D_cLa4tW.js.map +1 -0
- package/dist/results-DrKJuIQF.js +63 -0
- package/dist/results-DrKJuIQF.js.map +1 -0
- package/dist/{resume-B9SOSnK9.js → resume-BmIM52MJ.js} +47 -17
- package/dist/resume-BmIM52MJ.js.map +1 -0
- package/dist/{runtime-BgQRyCLl.js → runtime-CZVpkr04.js} +139 -48
- package/dist/runtime-CZVpkr04.js.map +1 -0
- package/dist/{sandbox-BOYZ0hlo.js → sandbox-Cmi6IQML.js} +2 -2
- package/dist/{sandbox-BOYZ0hlo.js.map → sandbox-Cmi6IQML.js.map} +1 -1
- package/dist/sandbox.d.ts +1 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CKJOnIB0.js +1 -0
- package/dist/{server-TGW4WyAz.js → server-DS8BGhGU.js} +75 -33
- package/dist/server-DS8BGhGU.js.map +1 -0
- package/dist/{shared-BDUzStad.js → shared-B_lvGqWz.js} +19 -11
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/source-CgUqpumI.js +1 -0
- package/dist/source-DcXv4de-.js +59 -0
- package/dist/source-DcXv4de-.js.map +1 -0
- package/dist/{src-CHIwOy0o.js → src-1lpTocbd.js} +75 -30
- package/dist/{src-CHIwOy0o.js.map → src-1lpTocbd.js.map} +1 -1
- package/dist/{src-DjF_M8_f.js → src-8yTF251P.js} +30 -8
- package/dist/{src-DjF_M8_f.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-DjRI-IeE.js → src-A8HLtO0a.js} +135 -35
- package/dist/{src-DjRI-IeE.js.map → src-A8HLtO0a.js.map} +1 -1
- package/dist/{src-NSdQ9zI6.js → src-BO8AYqJ5.js} +17 -5
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/src-BTqJbvj6.js +2 -0
- package/dist/{src-DmUgeTlE.js → src-BYDOXw_Y.js} +23 -7
- package/dist/{src-DmUgeTlE.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BkiComGh.js +2 -0
- package/dist/src-BvgeZ4tS.js +2 -0
- package/dist/{src-8zoX_qrb.js → src-C0ySiej5.js} +572 -1067
- package/dist/src-C0ySiej5.js.map +1 -0
- package/dist/src-CgJrz5p0.js +3 -0
- package/dist/src-Cp-2Hdib.js +2 -0
- package/dist/{src-CyaDNG9r.js → src-DMo2eCOJ.js} +1 -1
- package/dist/{src-ChhtI6BP.js → src-DSZyg0Uq.js} +177 -28
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/src-Dk2Na0b-.js +2 -0
- package/dist/src-OESRtvOi.js +2 -0
- package/dist/{src-DKPVX2Us.js → src-gn7YlnRg.js} +34 -5
- package/dist/{src-DKPVX2Us.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-h5uDlk3m.js +1 -0
- package/dist/state-C97udoh3.js +1 -0
- package/dist/{state-B1t143wG.js → state-C_kdhp5v.js} +15 -15
- package/dist/{state-B1t143wG.js.map → state-C_kdhp5v.js.map} +1 -1
- package/dist/{stats-DSy5HfOv.js → stats-hfpuYsAM.js} +2 -2
- package/dist/{stats-DSy5HfOv.js.map → stats-hfpuYsAM.js.map} +1 -1
- package/dist/suite-ByT1lsdD.js +2 -0
- package/dist/{suite-WgT7m6Vs.js → suite-DieLXvCT.js} +61 -65
- package/dist/suite-DieLXvCT.js.map +1 -0
- package/dist/{summary-DIGEVr08.js → summary-CSzyBVsF.js} +113 -277
- package/dist/summary-CSzyBVsF.js.map +1 -0
- package/dist/tarball-CHcahyNK.js +258 -0
- package/dist/tarball-CHcahyNK.js.map +1 -0
- package/dist/testing.d.ts +426 -143
- package/dist/testing.js +12 -10
- package/dist/testing.js.map +1 -1
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{tool-target-D2mcYlSQ.js → tool-target-CKWkd6QI.js} +2 -2
- package/dist/{tool-target-D2mcYlSQ.js.map → tool-target-CKWkd6QI.js.map} +1 -1
- package/dist/{transcript-BTxJpW4R.js → transcript-BJrKfgRW.js} +13 -5
- package/dist/transcript-BJrKfgRW.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-DXBbP9Mk.d.ts → types-DVvW66ci.d.ts} +569 -895
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +548 -121
- package/dist/view/index.js +9 -9
- package/dist/{views-sXpGE6a2.js → views-B9STsLB4.js} +9 -10
- package/dist/{views-sXpGE6a2.js.map → views-B9STsLB4.js.map} +1 -1
- package/dist/views-Ca-rTixp.js +2 -0
- package/dist/{worker-entry-Duo9oAeT.js → worker-entry-C5bLCv6s.js} +48 -20
- package/dist/worker-entry-C5bLCv6s.js.map +1 -0
- package/dist/workspace-8uHtBSve.js +1 -0
- package/dist/{workspace-CDI68wtX.js → workspace-CBuI9Uds.js} +5 -5
- package/dist/{workspace-CDI68wtX.js.map → workspace-CBuI9Uds.js.map} +1 -1
- package/docker/seccomp-default.json +1 -0
- package/package.json +5 -1
- package/skills/eval-rubric/SKILL.md +46 -104
- package/dist/adapter-BloGVZ_a.js.map +0 -1
- package/dist/agent-m-_t-IuW.js.map +0 -1
- package/dist/app-files-C-jwhRDt.js.map +0 -1
- package/dist/assemble-BDEpelpi.js.map +0 -1
- package/dist/assemble-C5bfncHU.js +0 -1
- package/dist/capture-CVRwRTnb.js.map +0 -1
- package/dist/capture-CherbjzB.js +0 -1
- package/dist/cli-C_4gaJ50.js.map +0 -1
- package/dist/cli-spec-CpnbATtI.js.map +0 -1
- package/dist/client-D5hNjpEx.js.map +0 -1
- package/dist/command-BWAuwQNk.js +0 -91
- package/dist/command-BWAuwQNk.js.map +0 -1
- package/dist/command-BXLUfw4h.js.map +0 -1
- package/dist/command-Be6BSE2L.js +0 -335
- package/dist/command-Be6BSE2L.js.map +0 -1
- package/dist/command-Bxdy62dW.js.map +0 -1
- package/dist/commands-jzigbNFb.js.map +0 -1
- package/dist/config-BrdHA9bJ.js.map +0 -1
- package/dist/cost-D4elx933.js.map +0 -1
- package/dist/cost-DmnAod9x.js +0 -1
- package/dist/db-CAfksinX.js +0 -2
- package/dist/db-DO0p7xgP.js +0 -619
- package/dist/db-DO0p7xgP.js.map +0 -1
- package/dist/debug-D24NOQH1.js.map +0 -1
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-BnRlTpZA.js.map +0 -1
- package/dist/docker-D01slDF3.js.map +0 -1
- package/dist/doctor-DQ2p0euG.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-CO0P0soF.js.map +0 -1
- package/dist/evaluate-BUmsN7yG.js.map +0 -1
- package/dist/evaluate-C_sgRA-V.d.ts +0 -134
- package/dist/facts-DzOUTreB.js +0 -906
- package/dist/facts-DzOUTreB.js.map +0 -1
- package/dist/files-B-CJeeEh.js.map +0 -1
- package/dist/files-BAt0i00X.js +0 -1
- package/dist/files-Bp5j5BqI.js +0 -1
- package/dist/files-C2sjmXhk.js +0 -1
- package/dist/files-C_gQNVB-.js +0 -65
- package/dist/files-C_gQNVB-.js.map +0 -1
- package/dist/files-Ca0d6Uq3.js.map +0 -1
- package/dist/files-kUCrswI6.js.map +0 -1
- package/dist/generate-DZ7vp69F.js +0 -1647
- package/dist/generate-DZ7vp69F.js.map +0 -1
- package/dist/harness-COVXSwJ6.js +0 -354
- package/dist/harness-COVXSwJ6.js.map +0 -1
- package/dist/header-BLPP6aCU.js.map +0 -1
- package/dist/init-RgCQXZhb.js +0 -1
- package/dist/init-SRalM5xF.js.map +0 -1
- package/dist/insights-TyAvz9aV.js +0 -68
- package/dist/insights-TyAvz9aV.js.map +0 -1
- package/dist/invocations-DzyKSJBa.js.map +0 -1
- package/dist/judge-yLvjC5A3.js.map +0 -1
- package/dist/match-Bz0nSdzN.js.map +0 -1
- package/dist/matchers-D4eUqx4j.js.map +0 -1
- package/dist/model-1ZLr9Dcv.js.map +0 -1
- package/dist/model-B45lDUV1.js.map +0 -1
- package/dist/model-BTZq8HIZ.js +0 -1
- package/dist/model-CgnOpss4.js.map +0 -1
- package/dist/model-Ch8kX87u.js +0 -1
- package/dist/node-BSq5qHnQ.js.map +0 -1
- package/dist/object-CLf2-tNq.js.map +0 -1
- package/dist/opencode-providers-klkqgzvb.js +0 -3276
- package/dist/opencode-providers-klkqgzvb.js.map +0 -1
- package/dist/pairs-CHfZNRT6.js.map +0 -1
- package/dist/patch-DWf-I_Q_.js.map +0 -1
- package/dist/plan-BIwUzlB8.js.map +0 -1
- package/dist/plugins-Dkp_H3pB.js.map +0 -1
- package/dist/prices-BcLEXvdY.js.map +0 -1
- package/dist/query-Cvbwl804.js +0 -1
- package/dist/questions-bAudIc04.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-CXwSnSjR.js +0 -278
- package/dist/remarks-CXwSnSjR.js.map +0 -1
- package/dist/remarks-CyAu2DiB.js +0 -1
- package/dist/remarks-mujJv1Ey.js.map +0 -1
- package/dist/remote-CKcEKja4.js.map +0 -1
- package/dist/resolve-B5-uqmJ6.js +0 -1
- package/dist/resolve-CBoKzQMP.js.map +0 -1
- package/dist/results-CMgZEANy.js +0 -78
- package/dist/results-CMgZEANy.js.map +0 -1
- package/dist/resume-B9SOSnK9.js.map +0 -1
- package/dist/rules-VhftPOOT.js.map +0 -1
- package/dist/run-CwIEK16q.js +0 -88
- package/dist/run-CwIEK16q.js.map +0 -1
- package/dist/runtime-BgQRyCLl.js.map +0 -1
- package/dist/server-CHRSgKRp.js +0 -1
- package/dist/server-TGW4WyAz.js.map +0 -1
- package/dist/shared-BDUzStad.js.map +0 -1
- package/dist/source-B1WvBIQt.js +0 -40
- package/dist/source-B1WvBIQt.js.map +0 -1
- package/dist/source-DAzztZwU.js +0 -1
- package/dist/src-1jHbFaGW.js +0 -1
- package/dist/src-8zoX_qrb.js.map +0 -1
- package/dist/src-C_9wgowC.js +0 -2
- package/dist/src-ChhtI6BP.js.map +0 -1
- package/dist/src-D005Cbqz.js +0 -2
- package/dist/src-DIfRbp5a.js +0 -3
- package/dist/src-Dg8121Xy.js +0 -2
- package/dist/src-ElBqEN-S.js +0 -2
- package/dist/src-NSdQ9zI6.js.map +0 -1
- package/dist/src-YHGaUrGE.js +0 -2
- package/dist/src-mJVxw1Jm.js +0 -2
- package/dist/state-vznMh2wl.js +0 -1
- package/dist/suite-BsWGHzlG.js +0 -2
- package/dist/suite-WgT7m6Vs.js.map +0 -1
- package/dist/summary-DIGEVr08.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-DF3_p5pE.js +0 -1
- package/dist/validate-BhwqrVsm.js.map +0 -1
- package/dist/views-BW_C5eVH.js +0 -2
- package/dist/worker-entry-Duo9oAeT.js.map +0 -1
- package/dist/workspace-QZCacP8A.js +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"app-files-BMVBr079.js","names":[],"sources":["../src/studio/app-files.ts"],"mappings":";AAAA,MAAa,YAAY;CAAC,6BAA4B,EAAC,QAAO,6n4DAA4n4D;CAAE,4BAA2B,EAAC,QAAO,0z9pBAA0z9pB;CAAE,eAAc,EAAC,QAAO,yTAAwT;CAAE,cAAa,EAAC,QAAO,ojBAAkjB;AAAC"}
|
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import { r as isTimeout } from "./errors-B6BSMs03.js";
|
|
2
|
-
import {
|
|
2
|
+
import { D as profileSandbox } from "./environment-Cf5HMjib.js";
|
|
3
3
|
import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
4
|
-
import { B as
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { i as formatDuration } from "./text-
|
|
4
|
+
import { A as harnessId, B as sandboxId, I as plannedSandboxes, K as mergeJudge, L as profileJudge, M as majorityOf, ft as harnessEffort, lt as UNRECORDED_QUESTION_STATE, mt as servedEffortOf, pt as resolvedEffortOf, z as runsOf } from "./config-CneBZB84.js";
|
|
5
|
+
import { l as judgeModelName } from "./judge-Bg4v4e_K.js";
|
|
6
|
+
import { f as redactJsonText, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
7
|
+
import { i as formatDuration } from "./text-DTigcA64.js";
|
|
8
|
+
import { i as passRateOf, o as worstOutcome, r as gatePart, t as cellOutcome } from "./gate-BC6SdQZs.js";
|
|
9
|
+
import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-hfpuYsAM.js";
|
|
8
10
|
import { i as sumUsage, r as promptTokens } from "./usage-BNY1ABrU.js";
|
|
9
|
-
import { r as patchForJudge, t as collapseDependencies } from "./patch-
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-
|
|
13
|
-
import { t as checkVerdict } from "./contrasts-
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
11
|
+
import { r as patchForJudge, t as collapseDependencies } from "./patch-6T9XAipm.js";
|
|
12
|
+
import { a as sourcesPart, r as evalFile } from "./eval-source-CAE4y2Sr.js";
|
|
13
|
+
import { I as checkTries, st as effortSample } from "./model-Bz7-j71_.js";
|
|
14
|
+
import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-C_kdhp5v.js";
|
|
15
|
+
import { t as checkVerdict } from "./contrasts-lhFsekco.js";
|
|
16
|
+
import { n as isCompletedAttempt, o as previousAttempt } from "./resume-BmIM52MJ.js";
|
|
17
|
+
import { join } from "node:path";
|
|
16
18
|
import { readFile, stat } from "node:fs/promises";
|
|
17
19
|
//#region src/history/families.ts
|
|
18
20
|
const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
@@ -183,65 +185,27 @@ function familiesOf(evals, minLiftOf) {
|
|
|
183
185
|
});
|
|
184
186
|
}
|
|
185
187
|
//#endregion
|
|
186
|
-
//#region src/history/sources.ts
|
|
187
|
-
/** `from '…'`, `import '…'`, `import('…')` and `export … from '…'`, with a literal specifier. */
|
|
188
|
-
const SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+)(['"])((?:\.{1,2}\/|\/)[^'"]+)\1/g;
|
|
189
|
-
const CANDIDATES = (path) => {
|
|
190
|
-
return [
|
|
191
|
-
path,
|
|
192
|
-
.../\.(m?js|cjs)$/.exec(path) ? [path.replace(/\.(m?js|cjs)$/, ".ts"), path.replace(/\.(m?js|cjs)$/, ".tsx")] : [],
|
|
193
|
-
`${path}.ts`,
|
|
194
|
-
`${path}.tsx`,
|
|
195
|
-
`${path}.js`,
|
|
196
|
-
`${path}.mjs`,
|
|
197
|
-
join(path, "index.ts"),
|
|
198
|
-
join(path, "index.js")
|
|
199
|
-
];
|
|
200
|
-
};
|
|
201
|
-
async function isFile(path) {
|
|
202
|
-
return (await stat(path).catch(() => null))?.isFile() ?? false;
|
|
203
|
-
}
|
|
204
|
-
async function resolveImport(from, specifier) {
|
|
205
|
-
const base = specifier.startsWith("/") ? specifier : resolve(dirname(from), specifier);
|
|
206
|
-
for (const candidate of CANDIDATES(base)) if (await isFile(candidate)) return candidate;
|
|
207
|
-
return null;
|
|
208
|
-
}
|
|
209
|
-
/**
|
|
210
|
-
* The eval file and every local module it imports, transitively, by path from `root` and
|
|
211
|
-
* content hash, sorted by path. Only relative and absolute specifiers are followed: packages
|
|
212
|
-
* (`reeval`, `node:*`, anything in `node_modules`) aren't traced.
|
|
213
|
-
*/
|
|
214
|
-
async function traceSources(root, file) {
|
|
215
|
-
const seen = /* @__PURE__ */ new Map();
|
|
216
|
-
const queue = [resolve(root, file)];
|
|
217
|
-
while (queue.length) {
|
|
218
|
-
const path = queue.shift();
|
|
219
|
-
if (seen.has(path) || path.split(sep).includes("node_modules")) continue;
|
|
220
|
-
const bytes = await readFile(path).catch(() => null);
|
|
221
|
-
if (!bytes) continue;
|
|
222
|
-
seen.set(path, {
|
|
223
|
-
path: relative(root, path).split(sep).join("/"),
|
|
224
|
-
sha256: sha256(bytes)
|
|
225
|
-
});
|
|
226
|
-
for (const match of bytes.toString("utf8").matchAll(SPECIFIER)) {
|
|
227
|
-
const found = await resolveImport(path, match[2]);
|
|
228
|
-
if (found && !seen.has(found)) queue.push(found);
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
return [...seen.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
232
|
-
}
|
|
233
|
-
//#endregion
|
|
234
188
|
//#region src/history/assemble.ts
|
|
235
189
|
/** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
|
|
236
190
|
const TURN_TEXT_LIMIT = 8192;
|
|
237
191
|
const TURN_ERROR_LIMIT = 1024;
|
|
192
|
+
/**
|
|
193
|
+
* `text` masked when it's longer than `limit`, so a secret the cut would go through is masked
|
|
194
|
+
* whole. Text that fits is masked when the record is written.
|
|
195
|
+
*/
|
|
196
|
+
const maskedOver = (text, limit) => text.length > limit ? redactSecrets(text) : text;
|
|
238
197
|
/** A turn's first error event, such as the provider's "Credit balance is too low". */
|
|
239
198
|
function turnError(transcript) {
|
|
240
199
|
const found = transcript?.find((e) => e.type === "error");
|
|
241
200
|
if (found?.type !== "error") return null;
|
|
242
|
-
const message = found.message.trim();
|
|
201
|
+
const message = maskedOver(found.message.trim(), TURN_ERROR_LIMIT);
|
|
243
202
|
return message.length > TURN_ERROR_LIMIT ? `${message.slice(0, TURN_ERROR_LIMIT)}…` : message;
|
|
244
203
|
}
|
|
204
|
+
/** A turn's final message, cut at `TURN_TEXT_LIMIT`. */
|
|
205
|
+
function turnText(text) {
|
|
206
|
+
const s = maskedOver(text, TURN_TEXT_LIMIT);
|
|
207
|
+
return s.length > TURN_TEXT_LIMIT ? `${s.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${s.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : s;
|
|
208
|
+
}
|
|
245
209
|
/** Tokens across every turn that reported them. */
|
|
246
210
|
function costOf(attempts) {
|
|
247
211
|
const usage = sumUsage(attempts.map((a) => a.usage));
|
|
@@ -263,7 +227,8 @@ const OUTPUT_CAP = 131072;
|
|
|
263
227
|
const HEAD = 98304;
|
|
264
228
|
const TAIL = 16384;
|
|
265
229
|
/** Keeps evidence readable without letting one `cat` of a lockfile dominate the record. */
|
|
266
|
-
function capText(
|
|
230
|
+
function capText(raw, cap = OUTPUT_CAP) {
|
|
231
|
+
const text = maskedOver(raw, cap);
|
|
267
232
|
if (text.length <= cap) return text;
|
|
268
233
|
const omitted = text.length - HEAD - TAIL;
|
|
269
234
|
return `${text.slice(0, HEAD)}\n[reeval: truncated ${omitted} of ${text.length} characters, sha256 ${sha256(text)}]\n${text.slice(-16384)}`;
|
|
@@ -296,18 +261,6 @@ function mergeOutcome(e, record) {
|
|
|
296
261
|
error
|
|
297
262
|
};
|
|
298
263
|
}
|
|
299
|
-
/** Any failure fails; then infra errors; then all-skipped; else pass. */
|
|
300
|
-
function worstOutcome(outcomes) {
|
|
301
|
-
if (outcomes.includes("fail")) return "fail";
|
|
302
|
-
if (outcomes.includes("infra_error")) return "infra_error";
|
|
303
|
-
if (outcomes.length > 0 && outcomes.every((o) => o === "skipped")) return "skipped";
|
|
304
|
-
return outcomes.length ? "pass" : "skipped";
|
|
305
|
-
}
|
|
306
|
-
const passRateOf = (outcomes) => {
|
|
307
|
-
const pass = outcomes.filter((o) => o === "pass").length;
|
|
308
|
-
const counted = pass + outcomes.filter((o) => o === "fail").length;
|
|
309
|
-
return counted ? pass / counted : null;
|
|
310
|
-
};
|
|
311
264
|
/**
|
|
312
265
|
* The harness as the measurement sees it: without the operational limits (`maxConcurrent`,
|
|
313
266
|
* `startTimeout`), which don't change results, and without the route, which `gateway` holds.
|
|
@@ -319,12 +272,24 @@ const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
|
|
|
319
272
|
/** The gateway's part of a fingerprint: none on `direct`, so a cell without a gateway keeps its fingerprint. */
|
|
320
273
|
const gatewayPart = (gateway) => gateway && gateway.name !== "direct" ? { gateway: gateway.name } : {};
|
|
321
274
|
/** `<index>:<question hash>`, which follows one verdict across runs. */
|
|
322
|
-
const
|
|
275
|
+
const judgmentKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
|
|
276
|
+
const JSON_FILE = /\.(?:json|jsonl|ndjson)$/;
|
|
323
277
|
/** Collects artifact files, redacting before anything is hashed. */
|
|
324
278
|
var Artifacts = class {
|
|
325
279
|
files = /* @__PURE__ */ new Map();
|
|
280
|
+
moves = /* @__PURE__ */ new Map();
|
|
281
|
+
/**
|
|
282
|
+
* A snapshot at `source` placed at `path`, or null when it isn't there: its name and size are
|
|
283
|
+
* already in the record, and its files are never masked again or read into memory.
|
|
284
|
+
*/
|
|
285
|
+
async move(path, source) {
|
|
286
|
+
if (!await stat(source).catch(() => null)) return null;
|
|
287
|
+
this.moves.set(path, source);
|
|
288
|
+
return path;
|
|
289
|
+
}
|
|
290
|
+
/** JSON and JSON Lines files are masked inside their strings only, so they stay valid. */
|
|
326
291
|
add(path, content) {
|
|
327
|
-
const bytes = Buffer.from(redactSecrets(content));
|
|
292
|
+
const bytes = Buffer.from(JSON_FILE.test(path) ? redactJsonText(content) : redactSecrets(content));
|
|
328
293
|
this.files.set(path, bytes);
|
|
329
294
|
return {
|
|
330
295
|
path,
|
|
@@ -367,7 +332,7 @@ function withInput(e, { n, answer, calls }, out) {
|
|
|
367
332
|
const doc = {
|
|
368
333
|
question: answer.question,
|
|
369
334
|
model: answer.model,
|
|
370
|
-
stateOptions: answer.stateOptions ??
|
|
335
|
+
stateOptions: answer.stateOptions ?? UNRECORDED_QUESTION_STATE,
|
|
371
336
|
...answer.artifacts !== void 0 ? { artifacts: answer.artifacts } : {},
|
|
372
337
|
calls: calls.map(call)
|
|
373
338
|
};
|
|
@@ -436,13 +401,14 @@ function transcriptLines(record) {
|
|
|
436
401
|
});
|
|
437
402
|
return `${lines.join("\n")}\n`;
|
|
438
403
|
}
|
|
439
|
-
/** Every stream part the harness sent, for debugging the
|
|
404
|
+
/** Every stream part the harness sent, for debugging the normalizer (`REEVAL_RAW_TRANSCRIPT=1`). */
|
|
440
405
|
function rawLines(record) {
|
|
441
406
|
if (!record.turns.some((t) => t.raw)) return null;
|
|
442
|
-
|
|
407
|
+
const lines = record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
|
|
443
408
|
turn: index,
|
|
444
409
|
...part
|
|
445
|
-
})))
|
|
410
|
+
})));
|
|
411
|
+
return `${redactSplitSecrets(lines).join("\n")}\n`;
|
|
446
412
|
}
|
|
447
413
|
function activity(record) {
|
|
448
414
|
const events = record.turns.flatMap((t) => t.transcript);
|
|
@@ -515,6 +481,8 @@ var Seen = class {
|
|
|
515
481
|
environments = /* @__PURE__ */ new Map();
|
|
516
482
|
harnesses = /* @__PURE__ */ new Map();
|
|
517
483
|
resolvedHarnesses = /* @__PURE__ */ new Map();
|
|
484
|
+
/** Keys of the attempts kept from the previous record on resume. */
|
|
485
|
+
kept = /* @__PURE__ */ new Set();
|
|
518
486
|
/** Registry digests found by the preflight, by image id. */
|
|
519
487
|
digests;
|
|
520
488
|
constructor(sandbox) {
|
|
@@ -547,7 +515,28 @@ var Seen = class {
|
|
|
547
515
|
for (const t of record?.turns ?? []) if (t.model) resolved.add(t.model);
|
|
548
516
|
this.resolvedHarnesses.set(cell.harness, resolved);
|
|
549
517
|
}
|
|
550
|
-
|
|
518
|
+
/**
|
|
519
|
+
* A kept attempt has no worker record left, so what `add` reads from one comes from the
|
|
520
|
+
* previous record instead: its image, environment, context files and resolved models.
|
|
521
|
+
*/
|
|
522
|
+
addKept(previous, kept, cell, environmentDigest, planned) {
|
|
523
|
+
this.kept.add(kept.key);
|
|
524
|
+
for (const c of kept.context) this.contexts.set(`${c.path}\0${c.sha256}`, c);
|
|
525
|
+
for (const j of kept.judgements) if (j.resolvedModel) this.resolvedJudges.add(j.resolvedModel);
|
|
526
|
+
const image = previous.stack.images.find((i) => i.id === kept.imageId);
|
|
527
|
+
if (image && !this.images.has(image.id)) this.images.set(image.id, image);
|
|
528
|
+
const environment = previous.environments.find((x) => x.digest === environmentDigest);
|
|
529
|
+
if (environment) this.environments.set(`${environment.name}\0${environment.digest}`, environment);
|
|
530
|
+
const harness = previous.stack.harnesses.find((h) => h.id === cell.harness);
|
|
531
|
+
if (!planned && harness && !this.harnesses.has(cell.harness)) {
|
|
532
|
+
const { resolvedModels: _, resolvedEffort: __, ...rest } = harness;
|
|
533
|
+
this.harnesses.set(cell.harness, rest);
|
|
534
|
+
}
|
|
535
|
+
const resolved = this.resolvedHarnesses.get(cell.harness) ?? /* @__PURE__ */ new Set();
|
|
536
|
+
for (const t of kept.turns) if (t.model) resolved.add(t.model);
|
|
537
|
+
this.resolvedHarnesses.set(cell.harness, resolved);
|
|
538
|
+
}
|
|
539
|
+
entry(e, sources, from) {
|
|
551
540
|
let entry = this.byEval.get(e.id);
|
|
552
541
|
if (!entry) {
|
|
553
542
|
const key = e.id;
|
|
@@ -563,6 +552,8 @@ var Seen = class {
|
|
|
563
552
|
...e.attempt?.family ? { family: familyOf(e.attempt.family) } : {},
|
|
564
553
|
fingerprint: "",
|
|
565
554
|
sources,
|
|
555
|
+
...from.source ? { source: from.source } : {},
|
|
556
|
+
...from.revision !== void 0 ? { revision: from.revision } : {},
|
|
566
557
|
outcome: "skipped",
|
|
567
558
|
todo: e.todo === true,
|
|
568
559
|
passRate: null,
|
|
@@ -589,6 +580,8 @@ const UNKNOWN_CELL = {
|
|
|
589
580
|
};
|
|
590
581
|
/** minLift per cell, kept aside while attempts are added; see `rollUp`. */
|
|
591
582
|
const minLifts = /* @__PURE__ */ new WeakMap();
|
|
583
|
+
/** Cells whose fingerprint came from an attempt that ran, not one that was skipped. */
|
|
584
|
+
const measured = /* @__PURE__ */ new WeakSet();
|
|
592
585
|
const EMPTY_PARTS = {
|
|
593
586
|
sources: "",
|
|
594
587
|
harness: "",
|
|
@@ -598,12 +591,13 @@ const EMPTY_PARTS = {
|
|
|
598
591
|
judge: "",
|
|
599
592
|
context: ""
|
|
600
593
|
};
|
|
601
|
-
function cellOf(entry, cell, gate, harness, gateway) {
|
|
594
|
+
function cellOf(entry, cell, gate, harness, gateway, bestOf) {
|
|
602
595
|
let found = entry.cells.find((c) => c.key === cell.key);
|
|
603
596
|
if (!found) {
|
|
604
597
|
found = {
|
|
605
598
|
...cell,
|
|
606
599
|
...gateway ? { gateway: gateway.name } : {},
|
|
600
|
+
...bestOf !== void 0 ? { bestOf } : {},
|
|
607
601
|
fingerprint: "",
|
|
608
602
|
fingerprintParts: EMPTY_PARTS,
|
|
609
603
|
environmentDigest: null,
|
|
@@ -631,17 +625,25 @@ async function copyRef(ref, from, out) {
|
|
|
631
625
|
return out.add(ref.path, Buffer.from(bytes).toString("utf8"));
|
|
632
626
|
}
|
|
633
627
|
/** A completed attempt from a previous assemble of this runId, kept on resume. */
|
|
634
|
-
async function addPreviousAttempt(e, kept, seen, out, judge,
|
|
635
|
-
const
|
|
628
|
+
async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts) {
|
|
629
|
+
const before = previous.evals.find((p) => p.id === e.id);
|
|
630
|
+
const beforeCell = before?.cells.find((c) => c.key === kept.cell);
|
|
631
|
+
const cell = e.attempt?.cell ?? (beforeCell && {
|
|
632
|
+
key: beforeCell.key,
|
|
633
|
+
harness: beforeCell.harness,
|
|
634
|
+
environment: beforeCell.environment,
|
|
635
|
+
sandbox: beforeCell.sandbox,
|
|
636
|
+
role: beforeCell.role,
|
|
637
|
+
...beforeCell.variant !== void 0 ? { variant: beforeCell.variant } : {}
|
|
638
|
+
}) ?? {
|
|
636
639
|
...UNKNOWN_CELL,
|
|
637
640
|
key: kept.cell,
|
|
638
641
|
harness: kept.cell.split("/")[0] ?? "unknown",
|
|
639
642
|
environment: kept.cell.split("/")[1] ?? "unknown",
|
|
640
643
|
sandbox: kept.cell.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
|
|
641
644
|
};
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
const resolved = sources ?? seen.sources.get(e.file) ?? [];
|
|
645
|
+
const sources = before?.sources ?? [];
|
|
646
|
+
const { source, revision } = e.attempt ?? before ?? {};
|
|
645
647
|
const stub = {
|
|
646
648
|
id: e.id,
|
|
647
649
|
key: e.key,
|
|
@@ -656,8 +658,21 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
|
|
|
656
658
|
meta: void 0
|
|
657
659
|
};
|
|
658
660
|
seen.add(null, cell, stub);
|
|
659
|
-
|
|
660
|
-
const
|
|
661
|
+
seen.addKept(previous, kept, cell, beforeCell?.environmentDigest ?? null, e.attempt !== void 0);
|
|
662
|
+
const entry = seen.entry(stub, sources, {
|
|
663
|
+
source,
|
|
664
|
+
revision
|
|
665
|
+
});
|
|
666
|
+
const created = !entry.cells.some((c) => c.key === cell.key);
|
|
667
|
+
const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? beforeCell?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway, e.attempt ? e.attempt.bestOf : beforeCell?.bestOf);
|
|
668
|
+
if (created && beforeCell && !e.attempt) {
|
|
669
|
+
cellRecord.harnessKind = beforeCell.harnessKind;
|
|
670
|
+
cellRecord.model = beforeCell.model;
|
|
671
|
+
cellRecord.effort = beforeCell.effort;
|
|
672
|
+
if (beforeCell.gateway !== void 0) cellRecord.gateway = beforeCell.gateway;
|
|
673
|
+
}
|
|
674
|
+
const minLift = e.attempt ? e.attempt.minLift : beforeCell?.lift?.minLift ?? previous.families?.find((f) => f.id === before?.family?.id)?.lifts.find((l) => l.variant === kept.cell)?.minLift;
|
|
675
|
+
if (minLift !== void 0) minLifts.set(cellRecord, minLift);
|
|
661
676
|
const files = {};
|
|
662
677
|
for (const [name, ref] of Object.entries(kept.files)) {
|
|
663
678
|
if (!ref) continue;
|
|
@@ -686,23 +701,37 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
|
|
|
686
701
|
...q,
|
|
687
702
|
input: await copyRef(q.input, previousArtifacts, out)
|
|
688
703
|
} : q);
|
|
704
|
+
const { snapshots: keptSnapshots, ...rest } = kept;
|
|
705
|
+
const snapshots = keptSnapshots ? await placeSnapshots(keptSnapshots, (s) => ({
|
|
706
|
+
to: s.path ?? "",
|
|
707
|
+
from: previousArtifacts && s.path && join(previousArtifacts, ...s.path.split("/"))
|
|
708
|
+
}), out) : [];
|
|
689
709
|
entry.attempts.push({
|
|
690
|
-
...
|
|
710
|
+
...rest,
|
|
691
711
|
seq: null,
|
|
692
712
|
files,
|
|
693
713
|
checks,
|
|
694
|
-
...answers.length ? { answers } : {}
|
|
714
|
+
...answers.length ? { answers } : {},
|
|
715
|
+
...snapshots.length ? { snapshots } : {}
|
|
695
716
|
});
|
|
696
|
-
if (
|
|
717
|
+
if (measured.has(cellRecord)) return;
|
|
718
|
+
measured.add(cellRecord);
|
|
719
|
+
if (beforeCell?.fingerprint) {
|
|
720
|
+
cellRecord.fingerprintParts = beforeCell.fingerprintParts;
|
|
721
|
+
cellRecord.fingerprint = beforeCell.fingerprint;
|
|
722
|
+
cellRecord.environmentDigest = beforeCell.environmentDigest;
|
|
723
|
+
if (cellRecord.overrides.length === 0) cellRecord.overrides = [...beforeCell.overrides];
|
|
724
|
+
} else {
|
|
697
725
|
const parts = {
|
|
698
|
-
sources:
|
|
726
|
+
sources: sourcesPart(source, revision, sources),
|
|
699
727
|
harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
|
|
700
728
|
environment: cellRecord.environmentDigest,
|
|
701
729
|
image: kept.imageId,
|
|
702
730
|
agentCli: null,
|
|
703
731
|
judge,
|
|
704
732
|
context: sha256(JSON.stringify(kept.context)),
|
|
705
|
-
...gatewayPart(e.attempt?.gateway)
|
|
733
|
+
...gatewayPart(e.attempt?.gateway),
|
|
734
|
+
...gatePart(cellRecord.gate)
|
|
706
735
|
};
|
|
707
736
|
cellRecord.fingerprintParts = parts;
|
|
708
737
|
cellRecord.fingerprint = fingerprintOf(parts);
|
|
@@ -713,32 +742,40 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
713
742
|
const cell = e.attempt?.cell ?? UNKNOWN_CELL;
|
|
714
743
|
const n = e.attempt?.n ?? 1;
|
|
715
744
|
const kept = previousAttempt(previous, e.key);
|
|
716
|
-
if (e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
|
|
717
|
-
await addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts);
|
|
745
|
+
if (previous && e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
|
|
746
|
+
await addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts);
|
|
718
747
|
return;
|
|
719
748
|
}
|
|
720
749
|
const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
|
|
721
750
|
const { outcome, error } = mergeOutcome(e, record);
|
|
722
|
-
if (!seen.sources.has(e.file)) seen.sources.set(e.file, await
|
|
751
|
+
if (!seen.sources.has(e.file)) seen.sources.set(e.file, await evalFile(ctx.config.root, e.file));
|
|
723
752
|
const sources = seen.sources.get(e.file) ?? [];
|
|
753
|
+
const source = e.attempt?.source;
|
|
754
|
+
const revision = e.attempt?.revision;
|
|
724
755
|
const context = record?.context ?? [];
|
|
725
756
|
seen.add(record, cell, e);
|
|
726
|
-
const entry = seen.entry(e, sources
|
|
727
|
-
|
|
757
|
+
const entry = seen.entry(e, sources, {
|
|
758
|
+
source,
|
|
759
|
+
revision
|
|
760
|
+
});
|
|
761
|
+
if (record) entry.sources = sources;
|
|
762
|
+
const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway, e.attempt?.bestOf);
|
|
728
763
|
if (e.attempt?.minLift !== void 0) minLifts.set(cellRecord, e.attempt.minLift);
|
|
729
764
|
if (record?.environment?.overrides.length && cellRecord.overrides.length === 0) cellRecord.overrides = [...record.environment.overrides];
|
|
730
765
|
const environment = record?.environment;
|
|
731
766
|
if (environment && environment.name !== "-") cellRecord.environmentDigest = environment.digest;
|
|
732
767
|
if (!cellRecord.fingerprint || record) {
|
|
768
|
+
if (record) measured.add(cellRecord);
|
|
733
769
|
const parts = {
|
|
734
|
-
sources:
|
|
770
|
+
sources: sourcesPart(source, revision, sources),
|
|
735
771
|
harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
|
|
736
772
|
environment: environment?.digest ?? null,
|
|
737
773
|
image: record?.image?.id ?? null,
|
|
738
774
|
agentCli: record?.image?.agentCli ?? null,
|
|
739
775
|
judge,
|
|
740
776
|
context: sha256(JSON.stringify(context)),
|
|
741
|
-
...gatewayPart(e.attempt?.gateway)
|
|
777
|
+
...gatewayPart(e.attempt?.gateway),
|
|
778
|
+
...gatePart(cellRecord.gate)
|
|
742
779
|
};
|
|
743
780
|
cellRecord.fingerprintParts = parts;
|
|
744
781
|
cellRecord.fingerprint = fingerprintOf(parts);
|
|
@@ -760,6 +797,8 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
760
797
|
egress: record?.egress ?? null,
|
|
761
798
|
servedModels: record?.servedModels ?? null,
|
|
762
799
|
...record?.askedModels?.length ? { askedModels: record.askedModels } : {},
|
|
800
|
+
...record?.upstreams?.length ? { upstreams: record.upstreams } : {},
|
|
801
|
+
...record?.reportedCostUsd !== void 0 ? { reportedCostUsd: record.reportedCostUsd } : {},
|
|
763
802
|
effort: e.attempt ? harnessEffort(e.attempt.harness) ?? null : null,
|
|
764
803
|
servedEffort: servedEffortOf([...record?.servedModels ?? [], ...(record?.turns ?? []).map((t) => t.model)]) ?? null,
|
|
765
804
|
outcome,
|
|
@@ -768,6 +807,32 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
768
807
|
...await attemptEvidence(record, `${entry.slug}/${evalSlug(cell.key)}/${n}`, out, join(ctx.stagingDir, "evals", recordFileName(e.key)))
|
|
769
808
|
});
|
|
770
809
|
}
|
|
810
|
+
/** A snapshot's file or directory name: `final`, or `final.tar.gz` compressed. */
|
|
811
|
+
const fileOf = (s) => s.form === "tar.gz" ? `${s.name}.tar.gz` : s.name;
|
|
812
|
+
/**
|
|
813
|
+
* The attempt's snapshots with their paths in the run's directory, each moved there from where
|
|
814
|
+
* `place` says it waits; one whose files are gone (a resumed run's, from a run that kept none)
|
|
815
|
+
* is dropped.
|
|
816
|
+
*/
|
|
817
|
+
async function placeSnapshots(snapshots, place, out) {
|
|
818
|
+
const placed = [];
|
|
819
|
+
for (const s of snapshots) {
|
|
820
|
+
if (s.skipped) {
|
|
821
|
+
placed.push({
|
|
822
|
+
...s,
|
|
823
|
+
path: null
|
|
824
|
+
});
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
const { to, from } = place(s);
|
|
828
|
+
const path = from ? await out.move(to, from) : null;
|
|
829
|
+
if (path) placed.push({
|
|
830
|
+
...s,
|
|
831
|
+
path
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
return placed;
|
|
835
|
+
}
|
|
771
836
|
/** Turns, timings, activity, verdicts, checks and changed files, with their artifact files. */
|
|
772
837
|
async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
773
838
|
const files = {};
|
|
@@ -798,9 +863,13 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
798
863
|
}
|
|
799
864
|
if (artifacts.length) files.artifacts = artifacts;
|
|
800
865
|
}
|
|
801
|
-
const
|
|
866
|
+
const snapshots = record?.snapshots ? await placeSnapshots(record.snapshots, (s) => ({
|
|
867
|
+
to: `${dir}/snapshots/${fileOf(s)}`,
|
|
868
|
+
from: stagingRecord && join(stagingRecord.replace(/\.json$/, ".snapshots"), fileOf(s))
|
|
869
|
+
}), out) : [];
|
|
870
|
+
const judgments = (record?.judgements ?? []).map(({ call, ...j }, i) => ({
|
|
802
871
|
...j,
|
|
803
|
-
key:
|
|
872
|
+
key: judgmentKey(call ?? i, j.question)
|
|
804
873
|
}));
|
|
805
874
|
return {
|
|
806
875
|
turns: (record?.turns ?? []).map((t) => ({
|
|
@@ -813,7 +882,7 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
813
882
|
unreported: t.unreported ?? [],
|
|
814
883
|
error: turnError(t.transcript),
|
|
815
884
|
...t.mcpStartup ? { mcpStartup: t.mcpStartup } : {},
|
|
816
|
-
text: t.text
|
|
885
|
+
text: turnText(t.text)
|
|
817
886
|
})),
|
|
818
887
|
usage: sumUsage((record?.turns ?? []).map((t) => t.usage)),
|
|
819
888
|
...record?.captureUse ? { captureUse: record.captureUse } : {},
|
|
@@ -833,8 +902,11 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
833
902
|
stalledTools: 0,
|
|
834
903
|
maxToolMs: null
|
|
835
904
|
},
|
|
836
|
-
judgements,
|
|
905
|
+
judgements: judgments,
|
|
837
906
|
...record?.startRetries?.length ? { startRetries: record.startRetries } : {},
|
|
907
|
+
...record?.setupProcesses?.length ? { setupProcesses: record.setupProcesses } : {},
|
|
908
|
+
...snapshots.length ? { snapshots } : {},
|
|
909
|
+
...record?.serves?.length ? { serves: record.serves } : {},
|
|
838
910
|
...record?.assertions?.length ? { assertions: record.assertions } : {},
|
|
839
911
|
checks: (record?.checks ?? []).map((c, i) => ({
|
|
840
912
|
command: c.command,
|
|
@@ -890,7 +962,7 @@ function liftOf(control, variant, pooled) {
|
|
|
890
962
|
/**
|
|
891
963
|
* Each cell's checks: every label any attempt of the eval recorded, tallied over the cell's
|
|
892
964
|
* counted attempts (pass or fail). An attempt that counted without reaching a label, because
|
|
893
|
-
*
|
|
965
|
+
* an `expect.require` or a thrown error before it ended the body, counts it `notRun`.
|
|
894
966
|
*/
|
|
895
967
|
function checkTallies(entry) {
|
|
896
968
|
const labels = [...new Set(entry.attempts.flatMap((a) => (a.assertions ?? []).map((x) => x.label)))];
|
|
@@ -927,20 +999,28 @@ function discrimination(control, variant) {
|
|
|
927
999
|
}
|
|
928
1000
|
}
|
|
929
1001
|
/**
|
|
930
|
-
*
|
|
931
|
-
*
|
|
1002
|
+
* Marks a best-of cell's attempts past its majority's first: `added` for one that ran, `skipped`
|
|
1003
|
+
* for one the majority before it left unrun.
|
|
932
1004
|
*/
|
|
933
|
-
function
|
|
934
|
-
if (
|
|
935
|
-
const
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
1005
|
+
function markBestOf(cell, attempts) {
|
|
1006
|
+
if (cell.bestOf === void 0) return;
|
|
1007
|
+
const majority = majorityOf(cell.bestOf);
|
|
1008
|
+
let pass = 0;
|
|
1009
|
+
let fail = 0;
|
|
1010
|
+
for (const a of attempts.filter((x) => x.cell === cell.key).sort((x, y) => x.n - y.n)) {
|
|
1011
|
+
if (a.n > majority) {
|
|
1012
|
+
if (a.outcome !== "skipped") a.bestOf = "added";
|
|
1013
|
+
else if (pass >= majority || fail >= majority) a.bestOf = "skipped";
|
|
1014
|
+
}
|
|
1015
|
+
if (a.outcome === "pass") pass++;
|
|
1016
|
+
if (a.outcome === "fail") fail++;
|
|
1017
|
+
}
|
|
939
1018
|
}
|
|
940
1019
|
/** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
|
|
941
1020
|
function rollUp(evals) {
|
|
942
1021
|
for (const entry of evals) {
|
|
943
1022
|
for (const c of entry.cells) {
|
|
1023
|
+
markBestOf(c, entry.attempts);
|
|
944
1024
|
const outcomes = entry.attempts.filter((a) => a.cell === c.key).map((a) => a.outcome);
|
|
945
1025
|
c.runs = outcomes.length;
|
|
946
1026
|
c.pass = outcomes.filter((o) => o === "pass").length;
|
|
@@ -994,7 +1074,7 @@ function gitState(input, out) {
|
|
|
994
1074
|
return state;
|
|
995
1075
|
}
|
|
996
1076
|
const byName = (key) => (a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0;
|
|
997
|
-
function stackOf(input, seen, judgeModel, threshold
|
|
1077
|
+
function stackOf(input, seen, judgeModel, threshold) {
|
|
998
1078
|
const packages = packageVersions(input.enginePackages);
|
|
999
1079
|
return {
|
|
1000
1080
|
reeval: input.reeval,
|
|
@@ -1017,8 +1097,7 @@ function stackOf(input, seen, judgeModel, threshold, stateOptions) {
|
|
|
1017
1097
|
model: judgeModel,
|
|
1018
1098
|
resolvedModels: [...seen.resolvedJudges].sort(),
|
|
1019
1099
|
threshold,
|
|
1020
|
-
version: judgeVersion(input.reeval, packages, judgeModel ?? "")
|
|
1021
|
-
...stateOptions ? { state: stateOptions } : {}
|
|
1100
|
+
version: judgeVersion(input.reeval, packages, judgeModel ?? "")
|
|
1022
1101
|
},
|
|
1023
1102
|
context: [...seen.contexts.values()].sort(byName((c) => c.path)),
|
|
1024
1103
|
fingerprintScope: FINGERPRINT_SCOPE
|
|
@@ -1029,8 +1108,6 @@ async function assembleRun(input) {
|
|
|
1029
1108
|
const { config, runId, profile } = ctx;
|
|
1030
1109
|
const out = new Artifacts();
|
|
1031
1110
|
const runJudge = profile ? mergeJudge(config.judge, profileJudge(profile)) : config.judge;
|
|
1032
|
-
const runStateOptions = runJudge.state && resolveStateOptions(runJudge.state);
|
|
1033
|
-
const ownStateOptions = runStateOptions && JSON.stringify(runStateOptions) !== JSON.stringify(resolveStateOptions({})) ? runStateOptions : void 0;
|
|
1034
1111
|
const judgeModel = runJudge.model === void 0 ? null : judgeModelName(runJudge.model);
|
|
1035
1112
|
const configHash = sha256(JSON.stringify({
|
|
1036
1113
|
include: config.include,
|
|
@@ -1046,8 +1123,7 @@ async function assembleRun(input) {
|
|
|
1046
1123
|
const judge = sha256(JSON.stringify({
|
|
1047
1124
|
model: judgeModel,
|
|
1048
1125
|
threshold: runJudge.threshold,
|
|
1049
|
-
version: judgeVersion(input.reeval, packageVersions(input.enginePackages), judgeModel ?? "")
|
|
1050
|
-
...ownStateOptions ? { state: ownStateOptions } : {}
|
|
1126
|
+
version: judgeVersion(input.reeval, packageVersions(input.enginePackages), judgeModel ?? "")
|
|
1051
1127
|
}));
|
|
1052
1128
|
for (const e of engine.evals) await addAttempt(e, ctx, seen, out, judge, input.previous, input.previousArtifacts);
|
|
1053
1129
|
if (input.previous) {
|
|
@@ -1066,7 +1142,7 @@ async function assembleRun(input) {
|
|
|
1066
1142
|
errors: [],
|
|
1067
1143
|
formattedErrors: [],
|
|
1068
1144
|
meta: void 0
|
|
1069
|
-
}, a, seen, out, judge, input.
|
|
1145
|
+
}, a, seen, out, judge, input.previous, input.previousArtifacts);
|
|
1070
1146
|
}
|
|
1071
1147
|
}
|
|
1072
1148
|
const evals = [...seen.byEval.values()];
|
|
@@ -1080,6 +1156,15 @@ async function assembleRun(input) {
|
|
|
1080
1156
|
for (const e of evals) {
|
|
1081
1157
|
const asked = input.answers?.evals.get(e.id);
|
|
1082
1158
|
if (asked?.length) e.answers = asked.map((q) => withInput(e, q, out));
|
|
1159
|
+
else if (e.attempts.length && e.attempts.every((a) => seen.kept.has(a.key))) {
|
|
1160
|
+
const before = input.previous?.evals.find((p) => p.id === e.id)?.answers ?? [];
|
|
1161
|
+
const answers = [];
|
|
1162
|
+
for (const q of before) answers.push(q.input ? {
|
|
1163
|
+
...q,
|
|
1164
|
+
input: await copyRef(q.input, input.previousArtifacts, out)
|
|
1165
|
+
} : q);
|
|
1166
|
+
if (answers.length) e.answers = answers;
|
|
1167
|
+
}
|
|
1083
1168
|
for (const a of e.attempts) {
|
|
1084
1169
|
const own = input.answers?.attempts.get(a.key);
|
|
1085
1170
|
if (own?.length) a.answers = own.map((q) => withAttemptInput(e, a, q, out));
|
|
@@ -1102,7 +1187,7 @@ async function assembleRun(input) {
|
|
|
1102
1187
|
git: gitState(input, out),
|
|
1103
1188
|
runner: input.runner,
|
|
1104
1189
|
dependencies: { lockfiles: input.lockfiles },
|
|
1105
|
-
stack: stackOf(input, seen, judgeModel, runJudge.threshold
|
|
1190
|
+
stack: stackOf(input, seen, judgeModel, runJudge.threshold),
|
|
1106
1191
|
environments: [...seen.environments.values()].sort(byName((e) => e.name)),
|
|
1107
1192
|
profile: profile ? {
|
|
1108
1193
|
name: profile.name,
|
|
@@ -1111,7 +1196,7 @@ async function assembleRun(input) {
|
|
|
1111
1196
|
harness: ids(profile.defaults.harness),
|
|
1112
1197
|
environment: profile.defaults.environment?.name,
|
|
1113
1198
|
sandbox: (profile.defaults.environment || profile.defaults.sandbox) && sandboxId(profileSandbox(config, profile.defaults)),
|
|
1114
|
-
runs
|
|
1199
|
+
...profile.defaults.runs !== void 0 ? runsOf(profile.defaults.runs) : {},
|
|
1115
1200
|
timeout: profile.defaults.timeout,
|
|
1116
1201
|
env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
|
|
1117
1202
|
})),
|
|
@@ -1128,7 +1213,7 @@ async function assembleRun(input) {
|
|
|
1128
1213
|
config: {
|
|
1129
1214
|
sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
|
|
1130
1215
|
environment: config.environment?.name ?? null,
|
|
1131
|
-
runs
|
|
1216
|
+
...config.runs !== void 0 ? runsOf(config.runs) : { runs: 1 },
|
|
1132
1217
|
judgeModel,
|
|
1133
1218
|
threshold: runJudge.threshold,
|
|
1134
1219
|
timeoutMs: config.timeoutMs,
|
|
@@ -1151,10 +1236,11 @@ async function assembleRun(input) {
|
|
|
1151
1236
|
...families.length ? { families } : {},
|
|
1152
1237
|
...engine.unhandledErrors.length ? { errors: engine.unhandledErrors.map((e) => redactSecrets(e)) } : {}
|
|
1153
1238
|
},
|
|
1154
|
-
artifacts: out.files
|
|
1239
|
+
artifacts: out.files,
|
|
1240
|
+
moves: out.moves
|
|
1155
1241
|
};
|
|
1156
1242
|
}
|
|
1157
1243
|
//#endregion
|
|
1158
|
-
export {
|
|
1244
|
+
export { gatewayPart as a, rawLines as c, turnText as d, fingerprintOf as i, transcriptLines as l, assembleRun as n, judgmentKey as o, capText as r, mergeOutcome as s, OUTPUT_CAP as t, turnError as u };
|
|
1159
1245
|
|
|
1160
|
-
//# sourceMappingURL=assemble-
|
|
1246
|
+
//# sourceMappingURL=assemble-BVbI6NW-.js.map
|