reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"app-files-ChbvRx3P.js","names":[],"sources":["../src/studio/app-files.ts"],"mappings":";AAAA,MAAa,YAAY;CAAC,6BAA4B,EAAC,QAAO,6n4DAA4n4D;CAAE,4BAA2B,EAAC,QAAO,m26pBAAm26pB;CAAE,eAAc,EAAC,QAAO,yTAAwT;CAAE,cAAa,EAAC,QAAO,ojBAAkjB;AAAC"}
|
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import { r as isTimeout } from "./errors-B6BSMs03.js";
|
|
2
|
-
import {
|
|
3
|
-
import { T as profileSandbox } from "./environment-B4IuURK1.js";
|
|
2
|
+
import { D as profileSandbox } from "./environment-BOFgzFDg.js";
|
|
4
3
|
import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import { i as
|
|
9
|
-
import { r as
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
4
|
+
import { A as harnessId, F as plannedSandboxes, G as resolveStateOptions, I as profileJudge, R as sandboxId, W as mergeJudge, ct as resolvedEffortOf, lt as servedEffortOf, ot as resolveQuestionState, st as harnessEffort } from "./config-k6m7XbAz.js";
|
|
5
|
+
import { l as judgeModelName } from "./judge-CKdgEXN7.js";
|
|
6
|
+
import { f as redactJsonText, h as redactSplitSecrets, m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
7
|
+
import { i as formatDuration } from "./text-DTigcA64.js";
|
|
8
|
+
import { a as mcnemar, d as round3, f as signedRank, l as pairedLiftInterval, n as containsZero, o as meanInterval, p as wilson, r as liftInterval, s as median } from "./stats-hfpuYsAM.js";
|
|
9
|
+
import { i as sumUsage, r as promptTokens } from "./usage-BNY1ABrU.js";
|
|
10
|
+
import { r as patchForJudge, t as collapseDependencies } from "./patch-DKJiC5Sq.js";
|
|
11
|
+
import { a as sourcesPart, r as evalFile } from "./eval-source-Dje3rERU.js";
|
|
12
|
+
import { Et as worstOutcome, I as checkTries, St as cellOutcome, Tt as passRateOf, st as effortSample, wt as gatePart } from "./model-CTnyvh7H.js";
|
|
13
|
+
import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-DrJ687ut.js";
|
|
14
|
+
import { t as checkVerdict } from "./contrasts-DLZF4dGF.js";
|
|
15
|
+
import { n as isCompletedAttempt, o as previousAttempt } from "./resume-j-rcIryi.js";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { readFile } from "node:fs/promises";
|
|
16
18
|
//#region src/history/families.ts
|
|
17
19
|
const isCounted = (a) => a.outcome === "pass" || a.outcome === "fail";
|
|
18
20
|
function sideOf(entry, cell) {
|
|
@@ -182,65 +184,27 @@ function familiesOf(evals, minLiftOf) {
|
|
|
182
184
|
});
|
|
183
185
|
}
|
|
184
186
|
//#endregion
|
|
185
|
-
//#region src/history/sources.ts
|
|
186
|
-
/** `from '…'`, `import '…'`, `import('…')` and `export … from '…'`, with a literal specifier. */
|
|
187
|
-
const SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+)(['"])((?:\.{1,2}\/|\/)[^'"]+)\1/g;
|
|
188
|
-
const CANDIDATES = (path) => {
|
|
189
|
-
return [
|
|
190
|
-
path,
|
|
191
|
-
.../\.(m?js|cjs)$/.exec(path) ? [path.replace(/\.(m?js|cjs)$/, ".ts"), path.replace(/\.(m?js|cjs)$/, ".tsx")] : [],
|
|
192
|
-
`${path}.ts`,
|
|
193
|
-
`${path}.tsx`,
|
|
194
|
-
`${path}.js`,
|
|
195
|
-
`${path}.mjs`,
|
|
196
|
-
join(path, "index.ts"),
|
|
197
|
-
join(path, "index.js")
|
|
198
|
-
];
|
|
199
|
-
};
|
|
200
|
-
async function isFile(path) {
|
|
201
|
-
return (await stat(path).catch(() => null))?.isFile() ?? false;
|
|
202
|
-
}
|
|
203
|
-
async function resolveImport(from, specifier) {
|
|
204
|
-
const base = specifier.startsWith("/") ? specifier : resolve(dirname(from), specifier);
|
|
205
|
-
for (const candidate of CANDIDATES(base)) if (await isFile(candidate)) return candidate;
|
|
206
|
-
return null;
|
|
207
|
-
}
|
|
208
|
-
/**
|
|
209
|
-
* The eval file and every local module it imports, transitively, by path from `root` and
|
|
210
|
-
* content hash, sorted by path. Only relative and absolute specifiers are followed: packages
|
|
211
|
-
* (`reeval`, `node:*`, anything in `node_modules`) aren't traced.
|
|
212
|
-
*/
|
|
213
|
-
async function traceSources(root, file) {
|
|
214
|
-
const seen = /* @__PURE__ */ new Map();
|
|
215
|
-
const queue = [resolve(root, file)];
|
|
216
|
-
while (queue.length) {
|
|
217
|
-
const path = queue.shift();
|
|
218
|
-
if (seen.has(path) || path.split(sep).includes("node_modules")) continue;
|
|
219
|
-
const bytes = await readFile(path).catch(() => null);
|
|
220
|
-
if (!bytes) continue;
|
|
221
|
-
seen.set(path, {
|
|
222
|
-
path: relative(root, path).split(sep).join("/"),
|
|
223
|
-
sha256: sha256(bytes)
|
|
224
|
-
});
|
|
225
|
-
for (const match of bytes.toString("utf8").matchAll(SPECIFIER)) {
|
|
226
|
-
const found = await resolveImport(path, match[2]);
|
|
227
|
-
if (found && !seen.has(found)) queue.push(found);
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
return [...seen.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
231
|
-
}
|
|
232
|
-
//#endregion
|
|
233
187
|
//#region src/history/assemble.ts
|
|
234
188
|
/** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
|
|
235
189
|
const TURN_TEXT_LIMIT = 8192;
|
|
236
190
|
const TURN_ERROR_LIMIT = 1024;
|
|
191
|
+
/**
|
|
192
|
+
* `text` masked when it's longer than `limit`, so a secret the cut would go through is masked
|
|
193
|
+
* whole. Text that fits is masked when the record is written.
|
|
194
|
+
*/
|
|
195
|
+
const maskedOver = (text, limit) => text.length > limit ? redactSecrets(text) : text;
|
|
237
196
|
/** A turn's first error event, such as the provider's "Credit balance is too low". */
|
|
238
197
|
function turnError(transcript) {
|
|
239
198
|
const found = transcript?.find((e) => e.type === "error");
|
|
240
199
|
if (found?.type !== "error") return null;
|
|
241
|
-
const message = found.message.trim();
|
|
200
|
+
const message = maskedOver(found.message.trim(), TURN_ERROR_LIMIT);
|
|
242
201
|
return message.length > TURN_ERROR_LIMIT ? `${message.slice(0, TURN_ERROR_LIMIT)}…` : message;
|
|
243
202
|
}
|
|
203
|
+
/** A turn's final message, cut at `TURN_TEXT_LIMIT`. */
|
|
204
|
+
function turnText(text) {
|
|
205
|
+
const s = maskedOver(text, TURN_TEXT_LIMIT);
|
|
206
|
+
return s.length > TURN_TEXT_LIMIT ? `${s.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${s.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : s;
|
|
207
|
+
}
|
|
244
208
|
/** Tokens across every turn that reported them. */
|
|
245
209
|
function costOf(attempts) {
|
|
246
210
|
const usage = sumUsage(attempts.map((a) => a.usage));
|
|
@@ -262,7 +226,8 @@ const OUTPUT_CAP = 131072;
|
|
|
262
226
|
const HEAD = 98304;
|
|
263
227
|
const TAIL = 16384;
|
|
264
228
|
/** Keeps evidence readable without letting one `cat` of a lockfile dominate the record. */
|
|
265
|
-
function capText(
|
|
229
|
+
function capText(raw, cap = OUTPUT_CAP) {
|
|
230
|
+
const text = maskedOver(raw, cap);
|
|
266
231
|
if (text.length <= cap) return text;
|
|
267
232
|
const omitted = text.length - HEAD - TAIL;
|
|
268
233
|
return `${text.slice(0, HEAD)}\n[reeval: truncated ${omitted} of ${text.length} characters, sha256 ${sha256(text)}]\n${text.slice(-16384)}`;
|
|
@@ -295,30 +260,25 @@ function mergeOutcome(e, record) {
|
|
|
295
260
|
error
|
|
296
261
|
};
|
|
297
262
|
}
|
|
298
|
-
/**
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
return outcomes.length ? "pass" : "skipped";
|
|
304
|
-
}
|
|
305
|
-
const passRateOf = (outcomes) => {
|
|
306
|
-
const pass = outcomes.filter((o) => o === "pass").length;
|
|
307
|
-
const counted = pass + outcomes.filter((o) => o === "fail").length;
|
|
308
|
-
return counted ? pass / counted : null;
|
|
309
|
-
};
|
|
310
|
-
/** Without the operational limits (`maxConcurrent`, `startTimeout`), which don't change results. */
|
|
311
|
-
const withoutCaps = ({ maxConcurrent: _, startTimeout: __, ...rest }) => rest;
|
|
263
|
+
/**
|
|
264
|
+
* The harness as the measurement sees it: without the operational limits (`maxConcurrent`,
|
|
265
|
+
* `startTimeout`), which don't change results, and without the route, which `gateway` holds.
|
|
266
|
+
*/
|
|
267
|
+
const withoutCaps = ({ maxConcurrent: _, startTimeout: __, gateway: ___, ...rest }) => rest;
|
|
312
268
|
const harnessModel = (h) => h.kind === "fake" ? null : h.model ?? null;
|
|
313
269
|
/** A cell fingerprint: the hash of its parts, so each part can be compared on its own. */
|
|
314
270
|
const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
|
|
271
|
+
/** The gateway's part of a fingerprint: none on `direct`, so a cell without a gateway keeps its fingerprint. */
|
|
272
|
+
const gatewayPart = (gateway) => gateway && gateway.name !== "direct" ? { gateway: gateway.name } : {};
|
|
315
273
|
/** `<index>:<question hash>`, which follows one verdict across runs. */
|
|
316
|
-
const
|
|
274
|
+
const judgmentKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
|
|
275
|
+
const JSON_FILE = /\.(?:json|jsonl|ndjson)$/;
|
|
317
276
|
/** Collects artifact files, redacting before anything is hashed. */
|
|
318
277
|
var Artifacts = class {
|
|
319
278
|
files = /* @__PURE__ */ new Map();
|
|
279
|
+
/** JSON and JSON Lines files are masked inside their strings only, so they stay valid. */
|
|
320
280
|
add(path, content) {
|
|
321
|
-
const bytes = Buffer.from(redactSecrets(content));
|
|
281
|
+
const bytes = Buffer.from(JSON_FILE.test(path) ? redactJsonText(content) : redactSecrets(content));
|
|
322
282
|
this.files.set(path, bytes);
|
|
323
283
|
return {
|
|
324
284
|
path,
|
|
@@ -340,26 +300,28 @@ var Artifacts = class {
|
|
|
340
300
|
return this.add(path, `${JSON.stringify(value, null, 2)}\n`);
|
|
341
301
|
}
|
|
342
302
|
};
|
|
303
|
+
/** A screenshot with its attempt's id resolved to the capture's path in the record. */
|
|
304
|
+
function screenshotRef(s, byId) {
|
|
305
|
+
if ("missing" in s) return s;
|
|
306
|
+
const { attempt, ...shot } = s;
|
|
307
|
+
const ref = byId.get(attempt)?.files.artifacts?.find((a) => a.name === s.name);
|
|
308
|
+
return ref ? {
|
|
309
|
+
...shot,
|
|
310
|
+
path: ref.path
|
|
311
|
+
} : shot;
|
|
312
|
+
}
|
|
343
313
|
/** The answer, with what its model was sent saved as `questions/<n>.json` beside the eval's attempts. */
|
|
344
|
-
function withInput(e, { answer, calls },
|
|
314
|
+
function withInput(e, { n, answer, calls }, out) {
|
|
345
315
|
if (!calls.length) return answer;
|
|
346
316
|
const byId = new Map(e.attempts.map((a) => [a.id, a]));
|
|
347
317
|
const call = ({ screenshots, ...rest }) => ({
|
|
348
318
|
...rest,
|
|
349
|
-
...screenshots ? { screenshots: screenshots.map((s) => {
|
|
350
|
-
if ("missing" in s) return s;
|
|
351
|
-
const { attempt, ...shot } = s;
|
|
352
|
-
const ref = byId.get(attempt)?.files.artifacts?.find((a) => a.name === s.name);
|
|
353
|
-
return ref ? {
|
|
354
|
-
...shot,
|
|
355
|
-
path: ref.path
|
|
356
|
-
} : shot;
|
|
357
|
-
}) } : {}
|
|
319
|
+
...screenshots ? { screenshots: screenshots.map((s) => screenshotRef(s, byId)) } : {}
|
|
358
320
|
});
|
|
359
321
|
const doc = {
|
|
360
322
|
question: answer.question,
|
|
361
323
|
model: answer.model,
|
|
362
|
-
stateOptions: answer.stateOptions ?? resolveQuestionState(void 0),
|
|
324
|
+
stateOptions: answer.stateOptions ?? resolveQuestionState(void 0, true),
|
|
363
325
|
...answer.artifacts !== void 0 ? { artifacts: answer.artifacts } : {},
|
|
364
326
|
calls: calls.map(call)
|
|
365
327
|
};
|
|
@@ -368,6 +330,26 @@ function withInput(e, { answer, calls }, n, out) {
|
|
|
368
330
|
input: out.json(`${e.slug}/questions/${n}.json`, doc)
|
|
369
331
|
};
|
|
370
332
|
}
|
|
333
|
+
/** An attempt's own answer, with what its model was sent saved as `questions/<n>.json` in its directory. */
|
|
334
|
+
function withAttemptInput(e, a, { n, answer, call }, out) {
|
|
335
|
+
if (!call) return answer;
|
|
336
|
+
const { screenshots, ...rest } = call;
|
|
337
|
+
const byId = /* @__PURE__ */ new Map([[a.id, a]]);
|
|
338
|
+
const doc = {
|
|
339
|
+
question: answer.question,
|
|
340
|
+
model: answer.model,
|
|
341
|
+
stateOptions: answer.stateOptions,
|
|
342
|
+
...answer.artifacts !== void 0 ? { artifacts: answer.artifacts } : {},
|
|
343
|
+
...answer.choices ? { choices: answer.choices } : {},
|
|
344
|
+
...rest,
|
|
345
|
+
...screenshots ? { screenshots: screenshots.map((s) => screenshotRef(s, byId)) } : {}
|
|
346
|
+
};
|
|
347
|
+
const dir = `${e.slug}/${evalSlug(a.cell)}/${a.n}`;
|
|
348
|
+
return {
|
|
349
|
+
...answer,
|
|
350
|
+
input: out.json(`${dir}/questions/${n}.json`, doc)
|
|
351
|
+
};
|
|
352
|
+
}
|
|
371
353
|
const endOf = (startedAt, durationMs) => new Date(Date.parse(startedAt) + durationMs).toISOString();
|
|
372
354
|
/** One JSON object per line: each turn's prompt, its events, then a `turn-end`. */
|
|
373
355
|
function transcriptLines(record) {
|
|
@@ -408,13 +390,14 @@ function transcriptLines(record) {
|
|
|
408
390
|
});
|
|
409
391
|
return `${lines.join("\n")}\n`;
|
|
410
392
|
}
|
|
411
|
-
/** Every stream part the harness sent, for debugging the
|
|
393
|
+
/** Every stream part the harness sent, for debugging the normalizer (`REEVAL_RAW_TRANSCRIPT=1`). */
|
|
412
394
|
function rawLines(record) {
|
|
413
395
|
if (!record.turns.some((t) => t.raw)) return null;
|
|
414
|
-
|
|
396
|
+
const lines = record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
|
|
415
397
|
turn: index,
|
|
416
398
|
...part
|
|
417
|
-
})))
|
|
399
|
+
})));
|
|
400
|
+
return `${redactSplitSecrets(lines).join("\n")}\n`;
|
|
418
401
|
}
|
|
419
402
|
function activity(record) {
|
|
420
403
|
const events = record.turns.flatMap((t) => t.transcript);
|
|
@@ -440,7 +423,15 @@ const block = (label, text) => {
|
|
|
440
423
|
const body = capText(text);
|
|
441
424
|
return `--- ${label}\n${body}${body.endsWith("\n") || !body ? "" : "\n"}`;
|
|
442
425
|
};
|
|
443
|
-
const
|
|
426
|
+
const output = (r) => `${block("stdout", r.stdout)}${block("stderr", r.stderr)}`;
|
|
427
|
+
/** The command, its exit and output; with `retries`, each try's exit and output in turn. */
|
|
428
|
+
function checkLog(c) {
|
|
429
|
+
const head = `$ ${c.command}\nexit ${c.exitCode}`;
|
|
430
|
+
if (!c.retries || !c.tries?.length) return `${head}\n${output(c)}`;
|
|
431
|
+
const of = c.retries + 1;
|
|
432
|
+
const tries = c.tries.map((t, i) => `\ntry ${i + 1} of ${of}: exit ${t.exitCode} after ${formatDuration(t.durationMs)}\n${output(t)}`);
|
|
433
|
+
return `${head}, ${checkTries(c) ?? `passed on try 1 of ${of}`}\n${tries.join("")}`;
|
|
434
|
+
}
|
|
444
435
|
/** Dependency and binary noise dropped, and each remaining file's diff capped. */
|
|
445
436
|
function historyPatch(patch) {
|
|
446
437
|
return patchForJudge(patch).split(/(?=^diff --git )/m).map((part) => capText(part)).join("");
|
|
@@ -479,6 +470,8 @@ var Seen = class {
|
|
|
479
470
|
environments = /* @__PURE__ */ new Map();
|
|
480
471
|
harnesses = /* @__PURE__ */ new Map();
|
|
481
472
|
resolvedHarnesses = /* @__PURE__ */ new Map();
|
|
473
|
+
/** Keys of the attempts kept from the previous record on resume. */
|
|
474
|
+
kept = /* @__PURE__ */ new Set();
|
|
482
475
|
/** Registry digests found by the preflight, by image id. */
|
|
483
476
|
digests;
|
|
484
477
|
constructor(sandbox) {
|
|
@@ -511,7 +504,28 @@ var Seen = class {
|
|
|
511
504
|
for (const t of record?.turns ?? []) if (t.model) resolved.add(t.model);
|
|
512
505
|
this.resolvedHarnesses.set(cell.harness, resolved);
|
|
513
506
|
}
|
|
514
|
-
|
|
507
|
+
/**
|
|
508
|
+
* A kept attempt has no worker record left, so what `add` reads from one comes from the
|
|
509
|
+
* previous record instead: its image, environment, context files and resolved models.
|
|
510
|
+
*/
|
|
511
|
+
addKept(previous, kept, cell, environmentDigest, planned) {
|
|
512
|
+
this.kept.add(kept.key);
|
|
513
|
+
for (const c of kept.context) this.contexts.set(`${c.path}\0${c.sha256}`, c);
|
|
514
|
+
for (const j of kept.judgements) if (j.resolvedModel) this.resolvedJudges.add(j.resolvedModel);
|
|
515
|
+
const image = previous.stack.images.find((i) => i.id === kept.imageId);
|
|
516
|
+
if (image && !this.images.has(image.id)) this.images.set(image.id, image);
|
|
517
|
+
const environment = previous.environments.find((x) => x.digest === environmentDigest);
|
|
518
|
+
if (environment) this.environments.set(`${environment.name}\0${environment.digest}`, environment);
|
|
519
|
+
const harness = previous.stack.harnesses.find((h) => h.id === cell.harness);
|
|
520
|
+
if (!planned && harness && !this.harnesses.has(cell.harness)) {
|
|
521
|
+
const { resolvedModels: _, resolvedEffort: __, ...rest } = harness;
|
|
522
|
+
this.harnesses.set(cell.harness, rest);
|
|
523
|
+
}
|
|
524
|
+
const resolved = this.resolvedHarnesses.get(cell.harness) ?? /* @__PURE__ */ new Set();
|
|
525
|
+
for (const t of kept.turns) if (t.model) resolved.add(t.model);
|
|
526
|
+
this.resolvedHarnesses.set(cell.harness, resolved);
|
|
527
|
+
}
|
|
528
|
+
entry(e, sources, from) {
|
|
515
529
|
let entry = this.byEval.get(e.id);
|
|
516
530
|
if (!entry) {
|
|
517
531
|
const key = e.id;
|
|
@@ -527,6 +541,8 @@ var Seen = class {
|
|
|
527
541
|
...e.attempt?.family ? { family: familyOf(e.attempt.family) } : {},
|
|
528
542
|
fingerprint: "",
|
|
529
543
|
sources,
|
|
544
|
+
...from.source ? { source: from.source } : {},
|
|
545
|
+
...from.revision !== void 0 ? { revision: from.revision } : {},
|
|
530
546
|
outcome: "skipped",
|
|
531
547
|
todo: e.todo === true,
|
|
532
548
|
passRate: null,
|
|
@@ -553,6 +569,8 @@ const UNKNOWN_CELL = {
|
|
|
553
569
|
};
|
|
554
570
|
/** minLift per cell, kept aside while attempts are added; see `rollUp`. */
|
|
555
571
|
const minLifts = /* @__PURE__ */ new WeakMap();
|
|
572
|
+
/** Cells whose fingerprint came from an attempt that ran, not one that was skipped. */
|
|
573
|
+
const measured = /* @__PURE__ */ new WeakSet();
|
|
556
574
|
const EMPTY_PARTS = {
|
|
557
575
|
sources: "",
|
|
558
576
|
harness: "",
|
|
@@ -562,11 +580,12 @@ const EMPTY_PARTS = {
|
|
|
562
580
|
judge: "",
|
|
563
581
|
context: ""
|
|
564
582
|
};
|
|
565
|
-
function cellOf(entry, cell, gate, harness) {
|
|
583
|
+
function cellOf(entry, cell, gate, harness, gateway) {
|
|
566
584
|
let found = entry.cells.find((c) => c.key === cell.key);
|
|
567
585
|
if (!found) {
|
|
568
586
|
found = {
|
|
569
587
|
...cell,
|
|
588
|
+
...gateway ? { gateway: gateway.name } : {},
|
|
570
589
|
fingerprint: "",
|
|
571
590
|
fingerprintParts: EMPTY_PARTS,
|
|
572
591
|
environmentDigest: null,
|
|
@@ -594,17 +613,25 @@ async function copyRef(ref, from, out) {
|
|
|
594
613
|
return out.add(ref.path, Buffer.from(bytes).toString("utf8"));
|
|
595
614
|
}
|
|
596
615
|
/** A completed attempt from a previous assemble of this runId, kept on resume. */
|
|
597
|
-
async function addPreviousAttempt(e, kept, seen, out, judge,
|
|
598
|
-
const
|
|
616
|
+
async function addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts) {
|
|
617
|
+
const before = previous.evals.find((p) => p.id === e.id);
|
|
618
|
+
const beforeCell = before?.cells.find((c) => c.key === kept.cell);
|
|
619
|
+
const cell = e.attempt?.cell ?? (beforeCell && {
|
|
620
|
+
key: beforeCell.key,
|
|
621
|
+
harness: beforeCell.harness,
|
|
622
|
+
environment: beforeCell.environment,
|
|
623
|
+
sandbox: beforeCell.sandbox,
|
|
624
|
+
role: beforeCell.role,
|
|
625
|
+
...beforeCell.variant !== void 0 ? { variant: beforeCell.variant } : {}
|
|
626
|
+
}) ?? {
|
|
599
627
|
...UNKNOWN_CELL,
|
|
600
628
|
key: kept.cell,
|
|
601
629
|
harness: kept.cell.split("/")[0] ?? "unknown",
|
|
602
630
|
environment: kept.cell.split("/")[1] ?? "unknown",
|
|
603
631
|
sandbox: kept.cell.split("/")[2]?.replace(/#\d+$/, "") ?? "unknown"
|
|
604
632
|
};
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
const resolved = sources ?? seen.sources.get(e.file) ?? [];
|
|
633
|
+
const sources = before?.sources ?? [];
|
|
634
|
+
const { source, revision } = e.attempt ?? before ?? {};
|
|
608
635
|
const stub = {
|
|
609
636
|
id: e.id,
|
|
610
637
|
key: e.key,
|
|
@@ -619,8 +646,21 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
|
|
|
619
646
|
meta: void 0
|
|
620
647
|
};
|
|
621
648
|
seen.add(null, cell, stub);
|
|
622
|
-
|
|
623
|
-
const
|
|
649
|
+
seen.addKept(previous, kept, cell, beforeCell?.environmentDigest ?? null, e.attempt !== void 0);
|
|
650
|
+
const entry = seen.entry(stub, sources, {
|
|
651
|
+
source,
|
|
652
|
+
revision
|
|
653
|
+
});
|
|
654
|
+
const created = !entry.cells.some((c) => c.key === cell.key);
|
|
655
|
+
const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? beforeCell?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway);
|
|
656
|
+
if (created && beforeCell && !e.attempt) {
|
|
657
|
+
cellRecord.harnessKind = beforeCell.harnessKind;
|
|
658
|
+
cellRecord.model = beforeCell.model;
|
|
659
|
+
cellRecord.effort = beforeCell.effort;
|
|
660
|
+
if (beforeCell.gateway !== void 0) cellRecord.gateway = beforeCell.gateway;
|
|
661
|
+
}
|
|
662
|
+
const minLift = e.attempt ? e.attempt.minLift : beforeCell?.lift?.minLift ?? previous.families?.find((f) => f.id === before?.family?.id)?.lifts.find((l) => l.variant === kept.cell)?.minLift;
|
|
663
|
+
if (minLift !== void 0) minLifts.set(cellRecord, minLift);
|
|
624
664
|
const files = {};
|
|
625
665
|
for (const [name, ref] of Object.entries(kept.files)) {
|
|
626
666
|
if (!ref) continue;
|
|
@@ -644,21 +684,36 @@ async function addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts,
|
|
|
644
684
|
...c,
|
|
645
685
|
log: c.log ? await copyRef(c.log, previousArtifacts, out) : c.log
|
|
646
686
|
});
|
|
687
|
+
const answers = [];
|
|
688
|
+
for (const q of kept.answers ?? []) answers.push(q.input ? {
|
|
689
|
+
...q,
|
|
690
|
+
input: await copyRef(q.input, previousArtifacts, out)
|
|
691
|
+
} : q);
|
|
647
692
|
entry.attempts.push({
|
|
648
693
|
...kept,
|
|
649
694
|
seq: null,
|
|
650
695
|
files,
|
|
651
|
-
checks
|
|
696
|
+
checks,
|
|
697
|
+
...answers.length ? { answers } : {}
|
|
652
698
|
});
|
|
653
|
-
if (
|
|
699
|
+
if (measured.has(cellRecord)) return;
|
|
700
|
+
measured.add(cellRecord);
|
|
701
|
+
if (beforeCell?.fingerprint) {
|
|
702
|
+
cellRecord.fingerprintParts = beforeCell.fingerprintParts;
|
|
703
|
+
cellRecord.fingerprint = beforeCell.fingerprint;
|
|
704
|
+
cellRecord.environmentDigest = beforeCell.environmentDigest;
|
|
705
|
+
if (cellRecord.overrides.length === 0) cellRecord.overrides = [...beforeCell.overrides];
|
|
706
|
+
} else {
|
|
654
707
|
const parts = {
|
|
655
|
-
sources:
|
|
708
|
+
sources: sourcesPart(source, revision, sources),
|
|
656
709
|
harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
|
|
657
710
|
environment: cellRecord.environmentDigest,
|
|
658
711
|
image: kept.imageId,
|
|
659
712
|
agentCli: null,
|
|
660
713
|
judge,
|
|
661
|
-
context: sha256(JSON.stringify(kept.context))
|
|
714
|
+
context: sha256(JSON.stringify(kept.context)),
|
|
715
|
+
...gatewayPart(e.attempt?.gateway),
|
|
716
|
+
...gatePart(cellRecord.gate)
|
|
662
717
|
};
|
|
663
718
|
cellRecord.fingerprintParts = parts;
|
|
664
719
|
cellRecord.fingerprint = fingerprintOf(parts);
|
|
@@ -669,31 +724,40 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
669
724
|
const cell = e.attempt?.cell ?? UNKNOWN_CELL;
|
|
670
725
|
const n = e.attempt?.n ?? 1;
|
|
671
726
|
const kept = previousAttempt(previous, e.key);
|
|
672
|
-
if (e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
|
|
673
|
-
await addPreviousAttempt(e, kept, seen, out, judge, previousArtifacts);
|
|
727
|
+
if (previous && e.state === "skipped" && kept && isCompletedAttempt(kept.outcome, kept.error)) {
|
|
728
|
+
await addPreviousAttempt(e, kept, seen, out, judge, previous, previousArtifacts);
|
|
674
729
|
return;
|
|
675
730
|
}
|
|
676
731
|
const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
|
|
677
732
|
const { outcome, error } = mergeOutcome(e, record);
|
|
678
|
-
if (!seen.sources.has(e.file)) seen.sources.set(e.file, await
|
|
733
|
+
if (!seen.sources.has(e.file)) seen.sources.set(e.file, await evalFile(ctx.config.root, e.file));
|
|
679
734
|
const sources = seen.sources.get(e.file) ?? [];
|
|
735
|
+
const source = e.attempt?.source;
|
|
736
|
+
const revision = e.attempt?.revision;
|
|
680
737
|
const context = record?.context ?? [];
|
|
681
738
|
seen.add(record, cell, e);
|
|
682
|
-
const entry = seen.entry(e, sources
|
|
683
|
-
|
|
739
|
+
const entry = seen.entry(e, sources, {
|
|
740
|
+
source,
|
|
741
|
+
revision
|
|
742
|
+
});
|
|
743
|
+
if (record) entry.sources = sources;
|
|
744
|
+
const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness, e.attempt?.gateway);
|
|
684
745
|
if (e.attempt?.minLift !== void 0) minLifts.set(cellRecord, e.attempt.minLift);
|
|
685
746
|
if (record?.environment?.overrides.length && cellRecord.overrides.length === 0) cellRecord.overrides = [...record.environment.overrides];
|
|
686
747
|
const environment = record?.environment;
|
|
687
748
|
if (environment && environment.name !== "-") cellRecord.environmentDigest = environment.digest;
|
|
688
749
|
if (!cellRecord.fingerprint || record) {
|
|
750
|
+
if (record) measured.add(cellRecord);
|
|
689
751
|
const parts = {
|
|
690
|
-
sources:
|
|
752
|
+
sources: sourcesPart(source, revision, sources),
|
|
691
753
|
harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
|
|
692
754
|
environment: environment?.digest ?? null,
|
|
693
755
|
image: record?.image?.id ?? null,
|
|
694
756
|
agentCli: record?.image?.agentCli ?? null,
|
|
695
757
|
judge,
|
|
696
|
-
context: sha256(JSON.stringify(context))
|
|
758
|
+
context: sha256(JSON.stringify(context)),
|
|
759
|
+
...gatewayPart(e.attempt?.gateway),
|
|
760
|
+
...gatePart(cellRecord.gate)
|
|
697
761
|
};
|
|
698
762
|
cellRecord.fingerprintParts = parts;
|
|
699
763
|
cellRecord.fingerprint = fingerprintOf(parts);
|
|
@@ -714,6 +778,9 @@ async function addAttempt(e, ctx, seen, out, judge, previous, previousArtifacts)
|
|
|
714
778
|
context,
|
|
715
779
|
egress: record?.egress ?? null,
|
|
716
780
|
servedModels: record?.servedModels ?? null,
|
|
781
|
+
...record?.askedModels?.length ? { askedModels: record.askedModels } : {},
|
|
782
|
+
...record?.upstreams?.length ? { upstreams: record.upstreams } : {},
|
|
783
|
+
...record?.reportedCostUsd !== void 0 ? { reportedCostUsd: record.reportedCostUsd } : {},
|
|
717
784
|
effort: e.attempt ? harnessEffort(e.attempt.harness) ?? null : null,
|
|
718
785
|
servedEffort: servedEffortOf([...record?.servedModels ?? [], ...(record?.turns ?? []).map((t) => t.model)]) ?? null,
|
|
719
786
|
outcome,
|
|
@@ -752,9 +819,9 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
752
819
|
}
|
|
753
820
|
if (artifacts.length) files.artifacts = artifacts;
|
|
754
821
|
}
|
|
755
|
-
const
|
|
822
|
+
const judgments = (record?.judgements ?? []).map(({ call, ...j }, i) => ({
|
|
756
823
|
...j,
|
|
757
|
-
key:
|
|
824
|
+
key: judgmentKey(call ?? i, j.question)
|
|
758
825
|
}));
|
|
759
826
|
return {
|
|
760
827
|
turns: (record?.turns ?? []).map((t) => ({
|
|
@@ -767,7 +834,7 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
767
834
|
unreported: t.unreported ?? [],
|
|
768
835
|
error: turnError(t.transcript),
|
|
769
836
|
...t.mcpStartup ? { mcpStartup: t.mcpStartup } : {},
|
|
770
|
-
text: t.text
|
|
837
|
+
text: turnText(t.text)
|
|
771
838
|
})),
|
|
772
839
|
usage: sumUsage((record?.turns ?? []).map((t) => t.usage)),
|
|
773
840
|
...record?.captureUse ? { captureUse: record.captureUse } : {},
|
|
@@ -787,14 +854,22 @@ async function attemptEvidence(record, dir, out, stagingRecord) {
|
|
|
787
854
|
stalledTools: 0,
|
|
788
855
|
maxToolMs: null
|
|
789
856
|
},
|
|
790
|
-
judgements,
|
|
857
|
+
judgements: judgments,
|
|
791
858
|
...record?.startRetries?.length ? { startRetries: record.startRetries } : {},
|
|
859
|
+
...record?.setupProcesses?.length ? { setupProcesses: record.setupProcesses } : {},
|
|
792
860
|
...record?.assertions?.length ? { assertions: record.assertions } : {},
|
|
793
861
|
checks: (record?.checks ?? []).map((c, i) => ({
|
|
794
862
|
command: c.command,
|
|
795
863
|
exitCode: c.exitCode,
|
|
796
864
|
durationMs: c.durationMs,
|
|
797
|
-
log: out.add(`${dir}/checks/${i + 1}.log`, checkLog(c))
|
|
865
|
+
log: out.add(`${dir}/checks/${i + 1}.log`, checkLog(c)),
|
|
866
|
+
...c.retries && c.tries ? {
|
|
867
|
+
retries: c.retries,
|
|
868
|
+
tries: c.tries.map((t) => ({
|
|
869
|
+
exitCode: t.exitCode,
|
|
870
|
+
durationMs: t.durationMs
|
|
871
|
+
}))
|
|
872
|
+
} : {}
|
|
798
873
|
})),
|
|
799
874
|
changedFiles: changedFiles(record?.diff?.files ?? []),
|
|
800
875
|
files
|
|
@@ -873,17 +948,6 @@ function discrimination(control, variant) {
|
|
|
873
948
|
if (verdict) check.verdict = verdict;
|
|
874
949
|
}
|
|
875
950
|
}
|
|
876
|
-
/**
|
|
877
|
-
* A cell's outcome by its gate. `'all'` matches `{ passRate: 1 }`: every counted attempt must
|
|
878
|
-
* pass, and it is an infra error only when nothing counted. `'none'` takes the worst attempt.
|
|
879
|
-
*/
|
|
880
|
-
function cellOutcome(outcomes, gate) {
|
|
881
|
-
if (gate === "none") return worstOutcome(outcomes);
|
|
882
|
-
const required = gate === "all" ? 1 : gate.passRate;
|
|
883
|
-
const rate = passRateOf(outcomes);
|
|
884
|
-
if (rate === null) return worstOutcome(outcomes);
|
|
885
|
-
return rate >= required ? "pass" : "fail";
|
|
886
|
-
}
|
|
887
951
|
/** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
|
|
888
952
|
function rollUp(evals) {
|
|
889
953
|
for (const entry of evals) {
|
|
@@ -1013,7 +1077,7 @@ async function assembleRun(input) {
|
|
|
1013
1077
|
errors: [],
|
|
1014
1078
|
formattedErrors: [],
|
|
1015
1079
|
meta: void 0
|
|
1016
|
-
}, a, seen, out, judge, input.
|
|
1080
|
+
}, a, seen, out, judge, input.previous, input.previousArtifacts);
|
|
1017
1081
|
}
|
|
1018
1082
|
}
|
|
1019
1083
|
const evals = [...seen.byEval.values()];
|
|
@@ -1025,8 +1089,21 @@ async function assembleRun(input) {
|
|
|
1025
1089
|
const families = familiesOf(evals, (cell) => minLifts.get(cell));
|
|
1026
1090
|
sequence(evals);
|
|
1027
1091
|
for (const e of evals) {
|
|
1028
|
-
const asked = input.answers?.get(e.id);
|
|
1029
|
-
if (asked?.length) e.answers = asked.map((q
|
|
1092
|
+
const asked = input.answers?.evals.get(e.id);
|
|
1093
|
+
if (asked?.length) e.answers = asked.map((q) => withInput(e, q, out));
|
|
1094
|
+
else if (e.attempts.length && e.attempts.every((a) => seen.kept.has(a.key))) {
|
|
1095
|
+
const before = input.previous?.evals.find((p) => p.id === e.id)?.answers ?? [];
|
|
1096
|
+
const answers = [];
|
|
1097
|
+
for (const q of before) answers.push(q.input ? {
|
|
1098
|
+
...q,
|
|
1099
|
+
input: await copyRef(q.input, input.previousArtifacts, out)
|
|
1100
|
+
} : q);
|
|
1101
|
+
if (answers.length) e.answers = answers;
|
|
1102
|
+
}
|
|
1103
|
+
for (const a of e.attempts) {
|
|
1104
|
+
const own = input.answers?.attempts.get(a.key);
|
|
1105
|
+
if (own?.length) a.answers = own.map((q) => withAttemptInput(e, a, q, out));
|
|
1106
|
+
}
|
|
1030
1107
|
}
|
|
1031
1108
|
const attempts = evals.flatMap((e) => e.attempts);
|
|
1032
1109
|
const count = (o) => attempts.filter((a) => a.outcome === o).length;
|
|
@@ -1071,7 +1148,7 @@ async function assembleRun(input) {
|
|
|
1071
1148
|
config: {
|
|
1072
1149
|
sandbox: plannedSandboxes(input.plan.sandboxes, profileSandbox(config, profile?.defaults)),
|
|
1073
1150
|
environment: config.environment?.name ?? null,
|
|
1074
|
-
runs: config.runs,
|
|
1151
|
+
runs: config.runs ?? 1,
|
|
1075
1152
|
judgeModel,
|
|
1076
1153
|
threshold: runJudge.threshold,
|
|
1077
1154
|
timeoutMs: config.timeoutMs,
|
|
@@ -1098,6 +1175,6 @@ async function assembleRun(input) {
|
|
|
1098
1175
|
};
|
|
1099
1176
|
}
|
|
1100
1177
|
//#endregion
|
|
1101
|
-
export {
|
|
1178
|
+
export { gatewayPart as a, rawLines as c, turnText as d, fingerprintOf as i, transcriptLines as l, assembleRun as n, judgmentKey as o, capText as r, mergeOutcome as s, OUTPUT_CAP as t, turnError as u };
|
|
1102
1179
|
|
|
1103
|
-
//# sourceMappingURL=assemble-
|
|
1180
|
+
//# sourceMappingURL=assemble-SKQGW_QF.js.map
|