reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -0,0 +1,861 @@
|
|
|
1
|
+
import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
|
|
2
|
+
import { it as STOPPED_FIRST, ot as resolveQuestionState, rt as DEFAULT_QUESTION_STATE } from "./config-k6m7XbAz.js";
|
|
3
|
+
import { m as redactSecrets } from "./redact-DVIaahLv.js";
|
|
4
|
+
import { n as clip } from "./text-DTigcA64.js";
|
|
5
|
+
import { c as ratesOf, d as slots, i as pairFactors, m as voteReason, n as checksHeld, o as pairsOf, p as voteOf, t as acrossOf, u as sideName } from "./pairs-puYi-zj2.js";
|
|
6
|
+
import { a as isFake, i as hostModel, r as cannedModel } from "./model-Bq3G0OG1.js";
|
|
7
|
+
import { n as judgeDiff } from "./patch-DKJiC5Sq.js";
|
|
8
|
+
import { t as transcriptForJudge } from "./transcript-DxPaWHTZ.js";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
import { readFile } from "node:fs/promises";
|
|
12
|
+
import { Output, generateText } from "ai";
|
|
13
|
+
//#region src/compare/questions.ts
|
|
14
|
+
const DEFAULT_SHOWS = { state: DEFAULT_QUESTION_STATE };
|
|
15
|
+
const isOn = (value) => value !== false && !(Array.isArray(value) && !value.length);
|
|
16
|
+
/** `a`, `a and b`, `a, b and c`. */
|
|
17
|
+
const listed = (parts) => parts.length < 2 ? parts[0] ?? "" : `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}`;
|
|
18
|
+
/** The parts an attempt is shown as, for the system prompt. */
|
|
19
|
+
function shownParts(shows, taskEach) {
|
|
20
|
+
const { state } = shows;
|
|
21
|
+
return [
|
|
22
|
+
...taskEach && state.task ? ["the prompts it was sent"] : [],
|
|
23
|
+
...state.output ? ["its final message"] : [],
|
|
24
|
+
...isOn(state.transcript) ? ["its transcript"] : [],
|
|
25
|
+
...state.files ? ["the files it changed"] : [],
|
|
26
|
+
...isOn(state.diff) ? ["the changes it made to the project, as a unified diff"] : [],
|
|
27
|
+
...isOn(state.checks) ? ["the commands run on its work, with their exit codes and output"] : [],
|
|
28
|
+
...shows.artifacts !== void 0 ? ["screenshots of its work"] : []
|
|
29
|
+
];
|
|
30
|
+
}
|
|
31
|
+
/** The system prompt, naming the parts each attempt is shown as. */
|
|
32
|
+
function questionSystem(shows = DEFAULT_SHOWS, taskEach = false) {
|
|
33
|
+
const parts = shownParts(shows, taskEach);
|
|
34
|
+
return `You compare two attempts by coding agents at the same task and answer one question about them.
|
|
35
|
+
Each attempt is shown as ${listed(parts)}, labeled A or B. The labels are assigned at random: which attempt comes first says nothing about it.
|
|
36
|
+
Judge only by the question, from what is shown. Answer A or B for the attempt the question favors, or tie when what is shown doesn't separate them on it. Give a short reason for the pick.`;
|
|
37
|
+
}
|
|
38
|
+
const voteSchema = z.object({
|
|
39
|
+
winner: z.enum([
|
|
40
|
+
"A",
|
|
41
|
+
"B",
|
|
42
|
+
"tie"
|
|
43
|
+
]),
|
|
44
|
+
reason: z.string().optional()
|
|
45
|
+
});
|
|
46
|
+
/** Roughly four characters a token: each side gets half of `maxInputTokens`. */
|
|
47
|
+
const CHARS_PER_TOKEN = 4;
|
|
48
|
+
/** The turns a side shows: the last, or every one. */
|
|
49
|
+
const shownTurns = (side, state) => {
|
|
50
|
+
const turns = side.turns ?? [];
|
|
51
|
+
return state.turns === "all" ? turns : turns.slice(-1);
|
|
52
|
+
};
|
|
53
|
+
/** The task both sides were given, when they were given the same prompts. */
|
|
54
|
+
function sharedTask(first, second, state) {
|
|
55
|
+
if (!state.task) return null;
|
|
56
|
+
const a = shownTurns(first, state).map((t) => t.prompt);
|
|
57
|
+
const b = shownTurns(second, state).map((t) => t.prompt);
|
|
58
|
+
if (!a.length || a.length !== b.length || a.some((p, i) => p !== b[i])) return null;
|
|
59
|
+
return a.join("\n\n");
|
|
60
|
+
}
|
|
61
|
+
/** What a screenshot is labelled: its place on its side of a pair, or on the one attempt shown. */
|
|
62
|
+
const screenshotLabel = (side, i, of) => side ? `Attempt ${side}, screenshot ${i} of ${of}` : `Screenshot ${i} of ${of}`;
|
|
63
|
+
function buildSide(attempt, label, shows, taskShared) {
|
|
64
|
+
const { state } = shows;
|
|
65
|
+
const turns = shownTurns(attempt, state).map((t) => ({
|
|
66
|
+
...state.task && !taskShared ? { task: redactSecrets(t.prompt) } : {},
|
|
67
|
+
...state.output ? { output: redactSecrets(t.text ?? "") } : {},
|
|
68
|
+
...state.transcript ? { transcript: redactSecrets(transcriptForJudge(t.transcript ?? [], state.transcript).join("\n")) } : {}
|
|
69
|
+
}));
|
|
70
|
+
const shown = judgeDiff({
|
|
71
|
+
patch: attempt.diff,
|
|
72
|
+
files: attempt.files ?? []
|
|
73
|
+
}, Array.isArray(state.diff) ? { patterns: state.diff } : {});
|
|
74
|
+
const patch = redactSecrets(shown.patch);
|
|
75
|
+
const side = {
|
|
76
|
+
...state.turns === "all" ? turns.some((t) => Object.keys(t).length) ? { turns } : {} : turns[0] ?? {},
|
|
77
|
+
...state.files ? { files: shown.files } : {},
|
|
78
|
+
...state.diff !== false ? { diff: patch.trim() ? patch : "(no changes)" } : {}
|
|
79
|
+
};
|
|
80
|
+
const checks = checksOf(attempt, state.checks);
|
|
81
|
+
if (checks) side.checks = checks;
|
|
82
|
+
const images = [];
|
|
83
|
+
const sent = [];
|
|
84
|
+
if (shows.artifacts !== void 0) {
|
|
85
|
+
const captures = attempt.captures ?? [];
|
|
86
|
+
const names = shows.artifacts === true ? captures.map((c) => c.name) : shows.artifacts;
|
|
87
|
+
side.artifacts = names.map((name, i) => {
|
|
88
|
+
const at = screenshotLabel(label, i + 1, names.length);
|
|
89
|
+
const capture = captures.find((c) => c.name === name);
|
|
90
|
+
const side = label ?? "A";
|
|
91
|
+
if (!capture) {
|
|
92
|
+
images.push({
|
|
93
|
+
label: `${at}: not captured`,
|
|
94
|
+
data: null,
|
|
95
|
+
side,
|
|
96
|
+
name
|
|
97
|
+
});
|
|
98
|
+
return {
|
|
99
|
+
label: at,
|
|
100
|
+
missing: true
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
sent.push(name);
|
|
104
|
+
images.push({
|
|
105
|
+
label: at,
|
|
106
|
+
data: capture.data,
|
|
107
|
+
side,
|
|
108
|
+
name
|
|
109
|
+
});
|
|
110
|
+
return {
|
|
111
|
+
label: at,
|
|
112
|
+
mediaType: "image/png",
|
|
113
|
+
bytes: capture.data.byteLength
|
|
114
|
+
};
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
side,
|
|
119
|
+
images,
|
|
120
|
+
sent
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
function checksOf(attempt, which) {
|
|
124
|
+
if (which === false) return void 0;
|
|
125
|
+
const ran = attempt.checks ?? [];
|
|
126
|
+
const shown = ({ command, exitCode, stdout, stderr }) => ({
|
|
127
|
+
command,
|
|
128
|
+
exitCode,
|
|
129
|
+
stdout: redactSecrets(stdout),
|
|
130
|
+
stderr: redactSecrets(stderr)
|
|
131
|
+
});
|
|
132
|
+
if (which === true) return ran.map(shown);
|
|
133
|
+
return which.flatMap((command) => {
|
|
134
|
+
const runs = ran.filter((r) => r.command === command);
|
|
135
|
+
return runs.length ? runs.map(shown) : [{
|
|
136
|
+
command,
|
|
137
|
+
notRun: true
|
|
138
|
+
}];
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
const tag = (name, body) => [
|
|
142
|
+
`<${name}>`,
|
|
143
|
+
body,
|
|
144
|
+
`</${name}>`
|
|
145
|
+
];
|
|
146
|
+
/** A side's text in the prompt: each part it shows, in its own tag. */
|
|
147
|
+
function sideText(side) {
|
|
148
|
+
const turnLines = (t) => [
|
|
149
|
+
...t.task !== void 0 ? tag("task", t.task) : [],
|
|
150
|
+
...t.output !== void 0 ? tag("output", t.output) : [],
|
|
151
|
+
...t.transcript !== void 0 ? tag("transcript", t.transcript) : []
|
|
152
|
+
];
|
|
153
|
+
return [
|
|
154
|
+
...side.turns ? side.turns.flatMap((t, i) => [
|
|
155
|
+
`<turn n="${i + 1}">`,
|
|
156
|
+
...turnLines(t),
|
|
157
|
+
"</turn>"
|
|
158
|
+
]) : turnLines(side),
|
|
159
|
+
...side.files ? tag("files", side.files.join("\n") || "(none)") : [],
|
|
160
|
+
...side.diff !== void 0 ? tag("diff", side.diff) : [],
|
|
161
|
+
...side.checks ? tag("checks", side.checks.map((c) => "notRun" in c ? `$ ${c.command} → not run` : [
|
|
162
|
+
`$ ${c.command} → exit ${c.exitCode}`,
|
|
163
|
+
...c.stdout ? [c.stdout.trimEnd()] : [],
|
|
164
|
+
...c.stderr ? [`stderr: ${c.stderr.trimEnd()}`] : []
|
|
165
|
+
].join("\n")).join("\n")) : [],
|
|
166
|
+
...side.artifacts ? tag("screenshots", side.artifacts.map((a) => "missing" in a ? `${a.label}: not captured` : a.label).join("\n")) : []
|
|
167
|
+
].join("\n");
|
|
168
|
+
}
|
|
169
|
+
/** A side's text, cut to `chars` when it's longer. */
|
|
170
|
+
function cut(text, chars) {
|
|
171
|
+
if (text.length <= chars) return text;
|
|
172
|
+
return `${text.slice(0, chars)}\n[reeval: attempt cut at ${chars} of ${text.length} characters]`;
|
|
173
|
+
}
|
|
174
|
+
/** What the model is shown for one order of a pair: no harness, arm, run, attempt id or capture name. */
|
|
175
|
+
function blindInput(question, pair, swapped, maxInputTokens, shows = DEFAULT_SHOWS) {
|
|
176
|
+
const [first, second] = slots(pair, swapped);
|
|
177
|
+
const chars = Math.floor(maxInputTokens * CHARS_PER_TOKEN / 2);
|
|
178
|
+
const task = sharedTask(first, second, shows.state);
|
|
179
|
+
const a = buildSide(first, "A", shows, task !== null);
|
|
180
|
+
const b = buildSide(second, "B", shows, task !== null);
|
|
181
|
+
const prompt = [
|
|
182
|
+
`Question: ${question}`,
|
|
183
|
+
...task ? [
|
|
184
|
+
"",
|
|
185
|
+
"The task both attempts were given:",
|
|
186
|
+
"<task>",
|
|
187
|
+
task,
|
|
188
|
+
"</task>"
|
|
189
|
+
] : [],
|
|
190
|
+
"",
|
|
191
|
+
"<attempt label=\"A\">",
|
|
192
|
+
cut(sideText(a.side), chars),
|
|
193
|
+
"</attempt>",
|
|
194
|
+
"",
|
|
195
|
+
"<attempt label=\"B\">",
|
|
196
|
+
cut(sideText(b.side), chars),
|
|
197
|
+
"</attempt>"
|
|
198
|
+
].join("\n");
|
|
199
|
+
const [sentFirst, sentSecond] = swapped ? [b.sent, a.sent] : [a.sent, b.sent];
|
|
200
|
+
return {
|
|
201
|
+
blind: {
|
|
202
|
+
question,
|
|
203
|
+
prompt: redactSecrets(prompt),
|
|
204
|
+
a: a.side,
|
|
205
|
+
b: b.side
|
|
206
|
+
},
|
|
207
|
+
system: questionSystem(shows, task === null),
|
|
208
|
+
images: [...a.images, ...b.images],
|
|
209
|
+
sent: {
|
|
210
|
+
a: sentFirst,
|
|
211
|
+
b: sentSecond
|
|
212
|
+
}
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
function fakeAnswer(fake, pair) {
|
|
216
|
+
const vote = fake.vote ?? "tie";
|
|
217
|
+
if (typeof vote === "object" && "error" in vote) return cannedModel({ error: vote.error });
|
|
218
|
+
const pick = typeof vote === "function" ? vote(pair) : vote;
|
|
219
|
+
const winner = typeof pick === "string" ? pick : pick.winner;
|
|
220
|
+
const reason = typeof pick === "string" ? void 0 : pick.reason;
|
|
221
|
+
return cannedModel({ text: JSON.stringify({
|
|
222
|
+
winner,
|
|
223
|
+
...reason !== void 0 ? { reason } : {}
|
|
224
|
+
}) });
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* The model call's system prompt and input: the prompt as text, or, with screenshots, one user
|
|
228
|
+
* message of the prompt then each screenshot's label and image part.
|
|
229
|
+
*/
|
|
230
|
+
function questionRequest(input) {
|
|
231
|
+
const { blind, images } = input;
|
|
232
|
+
if (!images.length) return {
|
|
233
|
+
system: input.system,
|
|
234
|
+
prompt: blind.prompt
|
|
235
|
+
};
|
|
236
|
+
return {
|
|
237
|
+
system: input.system,
|
|
238
|
+
messages: [{
|
|
239
|
+
role: "user",
|
|
240
|
+
content: [{
|
|
241
|
+
type: "text",
|
|
242
|
+
text: blind.prompt
|
|
243
|
+
}, ...images.flatMap((image) => [{
|
|
244
|
+
type: "text",
|
|
245
|
+
text: image.label
|
|
246
|
+
}, ...image.data ? [{
|
|
247
|
+
type: "file",
|
|
248
|
+
mediaType: "image/png",
|
|
249
|
+
data: image.data
|
|
250
|
+
}] : []])]
|
|
251
|
+
}]
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
async function ask(model, gateway, settings, input, maxOutputTokens, signal) {
|
|
255
|
+
const result = await generateText({
|
|
256
|
+
model: isFake(model) ? fakeAnswer(model, input.blind) : await hostModel(model, gateway, "question model", settings),
|
|
257
|
+
...questionRequest(input),
|
|
258
|
+
output: Output.object({
|
|
259
|
+
schema: voteSchema,
|
|
260
|
+
name: "vote"
|
|
261
|
+
}),
|
|
262
|
+
maxOutputTokens,
|
|
263
|
+
maxRetries: 3,
|
|
264
|
+
...signal ? { abortSignal: signal } : {}
|
|
265
|
+
});
|
|
266
|
+
const { winner, reason } = result.output;
|
|
267
|
+
return {
|
|
268
|
+
pick: winner,
|
|
269
|
+
reason: voteReason(reason),
|
|
270
|
+
resolvedModel: isFake(model) ? null : result.response?.modelId ?? null
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
/** Runs `work` over `items`, at most `limit` at once, keeping their order. */
|
|
274
|
+
async function pooled(items, limit, work) {
|
|
275
|
+
const out = new Array(items.length);
|
|
276
|
+
let next = 0;
|
|
277
|
+
const lanes = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
278
|
+
while (next < items.length) {
|
|
279
|
+
const i = next++;
|
|
280
|
+
out[i] = await work(items[i]);
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
await Promise.all(lanes);
|
|
284
|
+
return out;
|
|
285
|
+
}
|
|
286
|
+
function askedCall(pair, swapped, input, error) {
|
|
287
|
+
const [first, second] = slots(pair, swapped);
|
|
288
|
+
const shown = {
|
|
289
|
+
A: first,
|
|
290
|
+
B: second
|
|
291
|
+
};
|
|
292
|
+
return {
|
|
293
|
+
cut: pair.cut,
|
|
294
|
+
...pair.cut === "within" ? { harness: pair.harness } : { arm: pair.arm },
|
|
295
|
+
run: pair.run,
|
|
296
|
+
a: sideName(pair.cut, pair.a),
|
|
297
|
+
b: sideName(pair.cut, pair.b),
|
|
298
|
+
swapped,
|
|
299
|
+
labels: {
|
|
300
|
+
A: sideName(pair.cut, first),
|
|
301
|
+
B: sideName(pair.cut, second)
|
|
302
|
+
},
|
|
303
|
+
system: input.system,
|
|
304
|
+
prompt: input.blind.prompt,
|
|
305
|
+
...input.images.length ? { screenshots: input.images.map((image) => {
|
|
306
|
+
const label = image.label.replace(/: not captured$/, "");
|
|
307
|
+
return image.data ? {
|
|
308
|
+
label,
|
|
309
|
+
side: sideName(pair.cut, shown[image.side]),
|
|
310
|
+
name: image.name,
|
|
311
|
+
attempt: shown[image.side].id ?? ""
|
|
312
|
+
} : {
|
|
313
|
+
label,
|
|
314
|
+
missing: true
|
|
315
|
+
};
|
|
316
|
+
}) } : {},
|
|
317
|
+
...error !== void 0 ? { error } : {}
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
const errorText = (error) => redactSecrets((error instanceof Error ? error.message : String(error)).split("\n")[0] ?? "");
|
|
321
|
+
/**
|
|
322
|
+
* Why the question can't be asked: a command or capture it names that none of `sides` has, so
|
|
323
|
+
* each would show it missing. Undefined when each one is on some side.
|
|
324
|
+
*/
|
|
325
|
+
function missingEverywhere(question, shows, sides, who = "paired attempt") {
|
|
326
|
+
const { checks } = shows.state;
|
|
327
|
+
if (Array.isArray(checks)) {
|
|
328
|
+
const command = checks.find((c) => !sides.some((s) => s.checks?.some((r) => r.command === c)));
|
|
329
|
+
if (command !== void 0) return `question "${question}" state.checks names a command no ${who} ran: ${JSON.stringify(command)}`;
|
|
330
|
+
}
|
|
331
|
+
if (Array.isArray(shows.artifacts)) {
|
|
332
|
+
const name = shows.artifacts.find((n) => !sides.some((s) => s.captures?.some((c) => c.name === n)));
|
|
333
|
+
if (name !== void 0) return `question "${question}" artifacts names a capture no ${who} has: ${JSON.stringify(name)}`;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Asks one question of every pair, each twice with the order swapped. Votes that came back are
|
|
338
|
+
* kept when a call fails, and the first failure is the answer's `error`.
|
|
339
|
+
*/
|
|
340
|
+
async function answerQuestion(options) {
|
|
341
|
+
const { question, model, pairs } = options;
|
|
342
|
+
const shows = {
|
|
343
|
+
state: resolveQuestionState(options.state, true),
|
|
344
|
+
...options.artifacts !== void 0 ? { artifacts: options.artifacts } : {}
|
|
345
|
+
};
|
|
346
|
+
const head = {
|
|
347
|
+
question,
|
|
348
|
+
model: isFake(model) ? "fake" : model
|
|
349
|
+
};
|
|
350
|
+
const asked = {
|
|
351
|
+
stateOptions: shows.state,
|
|
352
|
+
...shows.artifacts !== void 0 ? { artifacts: shows.artifacts } : {}
|
|
353
|
+
};
|
|
354
|
+
const missing = missingEverywhere(question, shows, [...new Set(pairs.flatMap((p) => [p.a, p.b]))]);
|
|
355
|
+
if (missing) return {
|
|
356
|
+
...head,
|
|
357
|
+
...asked,
|
|
358
|
+
votes: [],
|
|
359
|
+
rates: [],
|
|
360
|
+
error: missing
|
|
361
|
+
};
|
|
362
|
+
const calls = pairs.flatMap((pair) => [false, true].map((swapped) => ({
|
|
363
|
+
pair,
|
|
364
|
+
swapped
|
|
365
|
+
}))).map((call, at) => ({
|
|
366
|
+
...call,
|
|
367
|
+
at
|
|
368
|
+
}));
|
|
369
|
+
const served = /* @__PURE__ */ new Set();
|
|
370
|
+
let error;
|
|
371
|
+
const made = new Array(calls.length).fill(null);
|
|
372
|
+
const votes = await pooled(calls, options.concurrency ?? 4, async ({ pair, swapped, at }) => {
|
|
373
|
+
if (error !== void 0) return null;
|
|
374
|
+
const input = blindInput(question, pair, swapped, options.maxInputTokens, shows);
|
|
375
|
+
try {
|
|
376
|
+
const { pick, reason, resolvedModel } = await ask(model, options.gateway, {
|
|
377
|
+
fallbacks: !!options.fallbacks,
|
|
378
|
+
...options.vertex ? { vertex: options.vertex } : {}
|
|
379
|
+
}, input, options.maxOutputTokens, options.signal);
|
|
380
|
+
made[at] = askedCall(pair, swapped, input);
|
|
381
|
+
if (resolvedModel) served.add(resolvedModel);
|
|
382
|
+
const vote = voteOf(question, pair, swapped, pick, reason);
|
|
383
|
+
return shows.artifacts !== void 0 ? {
|
|
384
|
+
...vote,
|
|
385
|
+
artifacts: input.sent
|
|
386
|
+
} : vote;
|
|
387
|
+
} catch (e) {
|
|
388
|
+
const why = errorText(e);
|
|
389
|
+
made[at] = askedCall(pair, swapped, input, why);
|
|
390
|
+
error ??= why;
|
|
391
|
+
return null;
|
|
392
|
+
}
|
|
393
|
+
});
|
|
394
|
+
options.calls?.push(...made.filter((c) => c !== null));
|
|
395
|
+
const kept = votes.filter((v) => v !== null);
|
|
396
|
+
return {
|
|
397
|
+
...head,
|
|
398
|
+
...served.size ? { resolvedModel: [...served].sort().join(", ") } : {},
|
|
399
|
+
...asked,
|
|
400
|
+
votes: kept,
|
|
401
|
+
rates: ratesOf(kept),
|
|
402
|
+
...error !== void 0 ? { error } : {}
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
/** Stored answers to a question asked of one attempt are at most this long. */
|
|
406
|
+
const ANSWER_TEXT_CAP = 1e3;
|
|
407
|
+
/** The system prompt for a question asked of one attempt, naming the parts it's shown as. */
|
|
408
|
+
function attemptQuestionSystem(shows, choices) {
|
|
409
|
+
const how = choices ? "Pick the one of the choices that answers it, and give a short reason for the pick." : "Answer it in a few sentences.";
|
|
410
|
+
return `You answer one question about an attempt by a coding agent at a task.
|
|
411
|
+
The attempt is shown as ${listed(shownParts(shows, true))}.
|
|
412
|
+
Answer only from what is shown. ${how}`;
|
|
413
|
+
}
|
|
414
|
+
function attemptInput(question, attempt, maxInputTokens, shows, choices) {
|
|
415
|
+
const built = buildSide(attempt, null, shows, false);
|
|
416
|
+
const prompt = [
|
|
417
|
+
`Question: ${question}`,
|
|
418
|
+
...choices ? [
|
|
419
|
+
"",
|
|
420
|
+
"Choices:",
|
|
421
|
+
...choices.map((c) => `- ${c}`)
|
|
422
|
+
] : [],
|
|
423
|
+
"",
|
|
424
|
+
"<attempt>",
|
|
425
|
+
cut(sideText(built.side), maxInputTokens * CHARS_PER_TOKEN),
|
|
426
|
+
"</attempt>"
|
|
427
|
+
].join("\n");
|
|
428
|
+
return {
|
|
429
|
+
blind: {
|
|
430
|
+
question,
|
|
431
|
+
prompt: redactSecrets(prompt),
|
|
432
|
+
...choices ? { choices } : {},
|
|
433
|
+
attempt: built.side
|
|
434
|
+
},
|
|
435
|
+
system: attemptQuestionSystem(shows, choices),
|
|
436
|
+
images: built.images,
|
|
437
|
+
sent: built.sent
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
function fakeAttemptAnswer(fake, attempt) {
|
|
441
|
+
const answer = fake.answer;
|
|
442
|
+
if (answer && typeof answer === "object" && "error" in answer) return cannedModel({ error: answer.error });
|
|
443
|
+
const given = answer === void 0 ? {} : typeof answer === "function" ? answer(attempt) : answer;
|
|
444
|
+
const { choice, text } = typeof given === "string" ? {
|
|
445
|
+
text: given,
|
|
446
|
+
choice: void 0
|
|
447
|
+
} : given;
|
|
448
|
+
return cannedModel({ text: JSON.stringify({
|
|
449
|
+
...attempt.choices ? { choice: choice ?? attempt.choices[0] } : {},
|
|
450
|
+
answer: text ?? "fake answer"
|
|
451
|
+
}) });
|
|
452
|
+
}
|
|
453
|
+
const answerSchema = (choices) => choices ? z.object({
|
|
454
|
+
choice: z.enum(choices),
|
|
455
|
+
answer: z.string()
|
|
456
|
+
}) : z.object({ answer: z.string() });
|
|
457
|
+
/** Asks one question of one attempt. A failed call is the answer's `error`, with empty `text`. */
|
|
458
|
+
async function answerAttempt(options) {
|
|
459
|
+
const { question, model, attempt, choices } = options;
|
|
460
|
+
const shows = {
|
|
461
|
+
state: resolveQuestionState(options.state, false),
|
|
462
|
+
...options.artifacts !== void 0 ? { artifacts: options.artifacts } : {}
|
|
463
|
+
};
|
|
464
|
+
const head = {
|
|
465
|
+
question,
|
|
466
|
+
model: isFake(model) ? "fake" : model,
|
|
467
|
+
stateOptions: shows.state,
|
|
468
|
+
...shows.artifacts !== void 0 ? { artifacts: shows.artifacts } : {},
|
|
469
|
+
...choices ? { choices } : {}
|
|
470
|
+
};
|
|
471
|
+
let input;
|
|
472
|
+
try {
|
|
473
|
+
input = attemptInput(question, attempt, options.maxInputTokens, shows, choices);
|
|
474
|
+
const result = await generateText({
|
|
475
|
+
model: isFake(model) ? fakeAttemptAnswer(model, input.blind) : await hostModel(model, options.gateway, "question model", {
|
|
476
|
+
fallbacks: !!options.fallbacks,
|
|
477
|
+
...options.vertex ? { vertex: options.vertex } : {}
|
|
478
|
+
}),
|
|
479
|
+
...questionRequest(input),
|
|
480
|
+
output: Output.object({
|
|
481
|
+
schema: answerSchema(choices),
|
|
482
|
+
name: "answer"
|
|
483
|
+
}),
|
|
484
|
+
maxOutputTokens: options.maxOutputTokens,
|
|
485
|
+
maxRetries: 3,
|
|
486
|
+
...options.signal ? { abortSignal: options.signal } : {}
|
|
487
|
+
});
|
|
488
|
+
const out = result.output;
|
|
489
|
+
const served = isFake(model) ? null : result.response?.modelId ?? null;
|
|
490
|
+
return {
|
|
491
|
+
answer: {
|
|
492
|
+
...head,
|
|
493
|
+
...served ? { resolvedModel: served } : {},
|
|
494
|
+
...out.choice !== void 0 ? { choice: out.choice } : {},
|
|
495
|
+
text: clip(redactSecrets(out.answer.trim()), ANSWER_TEXT_CAP)
|
|
496
|
+
},
|
|
497
|
+
call: attemptCall(input, attempt)
|
|
498
|
+
};
|
|
499
|
+
} catch (e) {
|
|
500
|
+
const why = errorText(e);
|
|
501
|
+
return {
|
|
502
|
+
answer: {
|
|
503
|
+
...head,
|
|
504
|
+
text: "",
|
|
505
|
+
error: why
|
|
506
|
+
},
|
|
507
|
+
...input ? { call: attemptCall(input, attempt, why) } : {}
|
|
508
|
+
};
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
/** The call a question asked of one attempt made, as its `questions/<n>.json` keeps it. */
|
|
512
|
+
const attemptCall = (input, attempt, error) => ({
|
|
513
|
+
system: input.system,
|
|
514
|
+
prompt: input.blind.prompt,
|
|
515
|
+
...input.images.length ? { screenshots: input.images.map((image) => {
|
|
516
|
+
const label = image.label.replace(/: not captured$/, "");
|
|
517
|
+
return image.data ? {
|
|
518
|
+
label,
|
|
519
|
+
side: "attempt",
|
|
520
|
+
name: image.name,
|
|
521
|
+
attempt: attempt.id ?? ""
|
|
522
|
+
} : {
|
|
523
|
+
label,
|
|
524
|
+
missing: true
|
|
525
|
+
};
|
|
526
|
+
}) } : {},
|
|
527
|
+
...error !== void 0 ? { error } : {}
|
|
528
|
+
});
|
|
529
|
+
/** Ran to a pass or a fail: what a question asked of one attempt is asked of. */
|
|
530
|
+
const answerable = (e, record) => record !== null && (e.state === "passed" || e.state === "failed" && record.error?.kind !== "infra" && record.error?.kind !== "aborted");
|
|
531
|
+
/** Runs `work` at most `limit` at a time, in the order it was asked. */
|
|
532
|
+
function limiter(limit) {
|
|
533
|
+
let running = 0;
|
|
534
|
+
const waiting = [];
|
|
535
|
+
return async (work) => {
|
|
536
|
+
if (running >= limit) await new Promise((go) => waiting.push(go));
|
|
537
|
+
else running++;
|
|
538
|
+
try {
|
|
539
|
+
return await work();
|
|
540
|
+
} finally {
|
|
541
|
+
const next = waiting.shift();
|
|
542
|
+
if (next) next();
|
|
543
|
+
else running--;
|
|
544
|
+
}
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* A staged attempt as a pair side: its record's turns, diff and checks, and whether its checks
|
|
549
|
+
* held. One a resume skipped as already completed is judged by its staged outcome.
|
|
550
|
+
*/
|
|
551
|
+
function stagedAttempt(e, record) {
|
|
552
|
+
const cell = e.attempt?.cell;
|
|
553
|
+
if (!cell) return null;
|
|
554
|
+
const outcome = e.state === "skipped" ? record?.outcome : e.state === "passed" ? "pass" : "fail";
|
|
555
|
+
const checksPassed = record !== null && outcome !== void 0 && checksHeld(outcome, record.error);
|
|
556
|
+
return {
|
|
557
|
+
harness: cell.harness,
|
|
558
|
+
arm: cell.environment,
|
|
559
|
+
run: e.attempt?.n ?? 1,
|
|
560
|
+
diff: record?.diff?.patch ?? "",
|
|
561
|
+
files: record?.diff?.files ?? [],
|
|
562
|
+
checksPassed,
|
|
563
|
+
control: cell.role === "control",
|
|
564
|
+
turns: record?.turns.map((t) => ({
|
|
565
|
+
prompt: t.prompt,
|
|
566
|
+
text: t.text,
|
|
567
|
+
transcript: t.transcript
|
|
568
|
+
})) ?? [],
|
|
569
|
+
checks: record?.checks.map(({ command, exitCode, stdout, stderr }) => ({
|
|
570
|
+
command,
|
|
571
|
+
exitCode,
|
|
572
|
+
stdout,
|
|
573
|
+
stderr
|
|
574
|
+
})) ?? [],
|
|
575
|
+
id: `${cell.key}#${e.attempt?.n ?? 1}`
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
/** The last try's output in a check log (`--- stdout` then `--- stderr`, as assembly writes it). */
|
|
579
|
+
function logOutput(log) {
|
|
580
|
+
const out = log?.lastIndexOf("--- stdout\n") ?? -1;
|
|
581
|
+
const err = log?.lastIndexOf("--- stderr\n") ?? -1;
|
|
582
|
+
if (!log || out < 0 || err < out) return {
|
|
583
|
+
stdout: "",
|
|
584
|
+
stderr: ""
|
|
585
|
+
};
|
|
586
|
+
return {
|
|
587
|
+
stdout: log.slice(out + 11, err),
|
|
588
|
+
stderr: log.slice(err + 11)
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* An attempt a resume kept from the previous record, as a pair side: its turns, `diff.patch`,
|
|
593
|
+
* transcript, check logs and captures, read back from the previous record's artifact files.
|
|
594
|
+
*/
|
|
595
|
+
async function keptAttempt(e, kept, artifacts, captures) {
|
|
596
|
+
const cell = e.attempt?.cell;
|
|
597
|
+
if (!cell) return null;
|
|
598
|
+
const path = (ref) => join(artifacts ?? "", ...ref.path.split("/"));
|
|
599
|
+
const text = (ref) => ref && artifacts ? readFile(path(ref), "utf8").catch(() => null) : Promise.resolve(null);
|
|
600
|
+
const events = /* @__PURE__ */ new Map();
|
|
601
|
+
for (const line of (await text(kept.files.transcript))?.split("\n") ?? []) {
|
|
602
|
+
if (!line) continue;
|
|
603
|
+
const { turn, ...event } = JSON.parse(line);
|
|
604
|
+
if (event.type === "prompt" || event.type === "turn-end") continue;
|
|
605
|
+
events.set(turn, [...events.get(turn) ?? [], event]);
|
|
606
|
+
}
|
|
607
|
+
const checks = [];
|
|
608
|
+
for (const c of kept.checks) checks.push({
|
|
609
|
+
command: c.command,
|
|
610
|
+
exitCode: c.exitCode,
|
|
611
|
+
...logOutput(await text(c.log))
|
|
612
|
+
});
|
|
613
|
+
const read = captures && artifacts ? kept.files.artifacts ?? [] : [];
|
|
614
|
+
const shots = await Promise.all(read.map(async (a) => {
|
|
615
|
+
const data = await readFile(path(a)).catch(() => null);
|
|
616
|
+
return data ? [{
|
|
617
|
+
name: a.name,
|
|
618
|
+
data: new Uint8Array(data)
|
|
619
|
+
}] : [];
|
|
620
|
+
}));
|
|
621
|
+
return {
|
|
622
|
+
harness: cell.harness,
|
|
623
|
+
arm: cell.environment,
|
|
624
|
+
run: e.attempt?.n ?? kept.n,
|
|
625
|
+
diff: await text(kept.files.diff) ?? "",
|
|
626
|
+
checksPassed: checksHeld(kept.outcome, kept.error),
|
|
627
|
+
control: cell.role === "control",
|
|
628
|
+
turns: kept.turns.map((t, i) => {
|
|
629
|
+
const said = events.get(i) ?? [];
|
|
630
|
+
const last = t.text.includes("… [reeval: truncated ") ? said.findLast((x) => x.type === "text") : void 0;
|
|
631
|
+
return {
|
|
632
|
+
prompt: t.prompt,
|
|
633
|
+
text: last?.type === "text" ? last.text : t.text,
|
|
634
|
+
transcript: said
|
|
635
|
+
};
|
|
636
|
+
}),
|
|
637
|
+
checks,
|
|
638
|
+
...captures ? { captures: shots.flat() } : {},
|
|
639
|
+
id: `${cell.key}#${e.attempt?.n ?? kept.n}`
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
const readStaged = async (stagingDir, key) => JSON.parse(await readFile(join(stagingDir, "evals", recordFileName(key)), "utf8").catch(() => "null"));
|
|
643
|
+
/** A staged attempt's captures, read from the PNGs beside its record; a missing file is left out. */
|
|
644
|
+
async function stagedCaptures(stagingDir, key, record) {
|
|
645
|
+
const dir = join(stagingDir, "evals", recordFileName(key).replace(/\.json$/, ".artifacts"));
|
|
646
|
+
return (await Promise.all((record?.artifacts ?? []).map(async ({ name }) => {
|
|
647
|
+
const data = await readFile(join(dir, `${name}.png`)).catch(() => null);
|
|
648
|
+
return data ? [{
|
|
649
|
+
name,
|
|
650
|
+
data: new Uint8Array(data)
|
|
651
|
+
}] : [];
|
|
652
|
+
}))).flat();
|
|
653
|
+
}
|
|
654
|
+
/** A per-attempt question's answer when no model answered it. */
|
|
655
|
+
const unanswered = (q, error) => ({
|
|
656
|
+
n: q.n,
|
|
657
|
+
answer: {
|
|
658
|
+
question: q.question,
|
|
659
|
+
model: isFake(q.model) ? "fake" : q.model,
|
|
660
|
+
stateOptions: resolveQuestionState(q.state, false),
|
|
661
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
|
|
662
|
+
...q.choices ? { choices: q.choices } : {},
|
|
663
|
+
text: "",
|
|
664
|
+
error
|
|
665
|
+
}
|
|
666
|
+
});
|
|
667
|
+
/**
|
|
668
|
+
* Asks each attempt its eval's per-attempt questions as soon as it ends, and an eval's pairwise
|
|
669
|
+
* questions as soon as its last planned attempt ends, while the rest of the run goes on; `settle`
|
|
670
|
+
* starts any pairwise ones left (an eval a filter or budget cut short) and waits for all.
|
|
671
|
+
*/
|
|
672
|
+
function questionRunner(options) {
|
|
673
|
+
const byId = new Map(options.evals.map((e) => [e.row.id, e]));
|
|
674
|
+
const ended = /* @__PURE__ */ new Map();
|
|
675
|
+
const started = /* @__PURE__ */ new Map();
|
|
676
|
+
const perAttempt = /* @__PURE__ */ new Map();
|
|
677
|
+
const done = /* @__PURE__ */ new Map();
|
|
678
|
+
const shown = /* @__PURE__ */ new Map();
|
|
679
|
+
const asking = /* @__PURE__ */ new Map();
|
|
680
|
+
const controller = new AbortController();
|
|
681
|
+
const live = () => !controller.signal.aborted;
|
|
682
|
+
const limit = limiter(options.concurrency ?? 4);
|
|
683
|
+
const pairwiseOf = (e) => e.questions.filter((q) => q.across !== void 0);
|
|
684
|
+
const eachOf = (e) => e.questions.filter((q) => q.across === void 0);
|
|
685
|
+
const keptAttempts = new Map((options.resumed?.previous?.evals ?? []).flatMap((e) => e.attempts.map((a) => [a.key, a])));
|
|
686
|
+
const isCompleted = (a) => a.state === "skipped" && options.resumed?.completed.has(a.key) === true;
|
|
687
|
+
const side = async (a, record, captures) => {
|
|
688
|
+
const kept = isCompleted(a) ? keptAttempts.get(a.key) : void 0;
|
|
689
|
+
if (kept) return keptAttempt(a, kept, options.resumed?.artifacts ?? null, captures);
|
|
690
|
+
const found = stagedAttempt(a, record);
|
|
691
|
+
if (!found || !captures) return found;
|
|
692
|
+
return {
|
|
693
|
+
...found,
|
|
694
|
+
captures: await stagedCaptures(options.stagingDir, a.key, record)
|
|
695
|
+
};
|
|
696
|
+
};
|
|
697
|
+
const askEach = (id, e, result) => {
|
|
698
|
+
const questions = eachOf(e);
|
|
699
|
+
if (!questions.length || perAttempt.has(result.key) || !live()) return;
|
|
700
|
+
const asked = (async () => {
|
|
701
|
+
const record = await readStaged(options.stagingDir, result.key);
|
|
702
|
+
if (!answerable(result, record)) return [];
|
|
703
|
+
const attempt = await side(result, record, questions.some((q) => q.artifacts !== void 0));
|
|
704
|
+
if (!attempt) return [];
|
|
705
|
+
shown.set(result.key, attempt);
|
|
706
|
+
asking.set(result.key, questions);
|
|
707
|
+
return Promise.all(questions.map(async (q) => {
|
|
708
|
+
const got = await limit(() => answerAttempt({
|
|
709
|
+
question: q.question,
|
|
710
|
+
model: q.model,
|
|
711
|
+
...q.gateway ? { gateway: q.gateway } : {},
|
|
712
|
+
...q.fallbacks ? { fallbacks: q.fallbacks } : {},
|
|
713
|
+
...q.vertex ? { vertex: q.vertex } : {},
|
|
714
|
+
attempt,
|
|
715
|
+
maxInputTokens: options.maxInputTokens,
|
|
716
|
+
maxOutputTokens: options.maxOutputTokens,
|
|
717
|
+
...q.state ? { state: q.state } : {},
|
|
718
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
|
|
719
|
+
...q.choices ? { choices: q.choices } : {},
|
|
720
|
+
signal: controller.signal
|
|
721
|
+
}));
|
|
722
|
+
if (live()) options.onAttemptAnswer?.(id, attempt.id ?? result.key, got.answer);
|
|
723
|
+
return {
|
|
724
|
+
n: q.n,
|
|
725
|
+
...got
|
|
726
|
+
};
|
|
727
|
+
}));
|
|
728
|
+
})().catch((error) => questions.map((q) => {
|
|
729
|
+
const asked = unanswered(q, errorText(error));
|
|
730
|
+
if (live()) options.onAttemptAnswer?.(id, result.key, asked.answer);
|
|
731
|
+
return asked;
|
|
732
|
+
}));
|
|
733
|
+
perAttempt.set(result.key, asked);
|
|
734
|
+
asked.then((got) => {
|
|
735
|
+
if (live() && got.length) done.set(result.key, got);
|
|
736
|
+
});
|
|
737
|
+
};
|
|
738
|
+
const unshown = (e, results) => {
|
|
739
|
+
const sides = results.flatMap((r) => shown.get(r.key) ?? []);
|
|
740
|
+
for (const q of eachOf(e)) {
|
|
741
|
+
const shows = {
|
|
742
|
+
state: resolveQuestionState(q.state, false),
|
|
743
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
|
|
744
|
+
};
|
|
745
|
+
const missing = sides.length && missingEverywhere(q.question, shows, sides, "attempt");
|
|
746
|
+
if (!missing) continue;
|
|
747
|
+
options.onUnshown?.(e.row.id, missing);
|
|
748
|
+
for (const r of results) {
|
|
749
|
+
const got = done.get(r.key);
|
|
750
|
+
if (!got) continue;
|
|
751
|
+
done.set(r.key, got.map((x) => {
|
|
752
|
+
if (x.n !== q.n) return x;
|
|
753
|
+
const { choice: _, resolvedModel: __, ...head } = x.answer;
|
|
754
|
+
return {
|
|
755
|
+
...x,
|
|
756
|
+
answer: {
|
|
757
|
+
...head,
|
|
758
|
+
text: "",
|
|
759
|
+
error: missing
|
|
760
|
+
}
|
|
761
|
+
};
|
|
762
|
+
}));
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
const start = (id) => {
|
|
767
|
+
const e = byId.get(id);
|
|
768
|
+
if (!e || started.has(id)) return;
|
|
769
|
+
const questions = pairwiseOf(e);
|
|
770
|
+
if (!questions.length) return;
|
|
771
|
+
const attempts = ended.get(id) ?? [];
|
|
772
|
+
const wantsCaptures = questions.some((q) => q.artifacts !== void 0);
|
|
773
|
+
const answers = (async () => {
|
|
774
|
+
const staged = (await Promise.all(attempts.map(async (a) => side(a, await readStaged(options.stagingDir, a.key), wantsCaptures)))).filter((s) => s !== null).sort((x, y) => (x.id ?? "") < (y.id ?? "") ? -1 : 1);
|
|
775
|
+
return Promise.all(questions.map(async (q) => {
|
|
776
|
+
const calls = [];
|
|
777
|
+
const answer = await answerQuestion({
|
|
778
|
+
question: q.question,
|
|
779
|
+
model: q.model,
|
|
780
|
+
...q.gateway ? { gateway: q.gateway } : {},
|
|
781
|
+
...q.fallbacks ? { fallbacks: q.fallbacks } : {},
|
|
782
|
+
...q.vertex ? { vertex: q.vertex } : {},
|
|
783
|
+
pairs: pairsOf(staged, acrossOf(pairFactors(q.across))),
|
|
784
|
+
maxInputTokens: options.maxInputTokens,
|
|
785
|
+
maxOutputTokens: options.maxOutputTokens,
|
|
786
|
+
...q.state ? { state: q.state } : {},
|
|
787
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
|
|
788
|
+
signal: controller.signal,
|
|
789
|
+
calls
|
|
790
|
+
});
|
|
791
|
+
if (live()) options.onAnswer?.(id, answer);
|
|
792
|
+
return {
|
|
793
|
+
n: q.n,
|
|
794
|
+
answer,
|
|
795
|
+
calls
|
|
796
|
+
};
|
|
797
|
+
}));
|
|
798
|
+
})().catch((error) => questions.map((q) => {
|
|
799
|
+
const answer = {
|
|
800
|
+
question: q.question,
|
|
801
|
+
model: isFake(q.model) ? "fake" : q.model,
|
|
802
|
+
stateOptions: resolveQuestionState(q.state, true),
|
|
803
|
+
...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {},
|
|
804
|
+
votes: [],
|
|
805
|
+
rates: [],
|
|
806
|
+
error: errorText(error)
|
|
807
|
+
};
|
|
808
|
+
if (live()) options.onAnswer?.(id, answer);
|
|
809
|
+
return {
|
|
810
|
+
n: q.n,
|
|
811
|
+
answer,
|
|
812
|
+
calls: []
|
|
813
|
+
};
|
|
814
|
+
}));
|
|
815
|
+
started.set(id, answers);
|
|
816
|
+
};
|
|
817
|
+
return {
|
|
818
|
+
onEvalEnd(result) {
|
|
819
|
+
const e = byId.get(result.id);
|
|
820
|
+
if (!e) return;
|
|
821
|
+
askEach(result.id, e, result);
|
|
822
|
+
const list = ended.get(result.id) ?? [];
|
|
823
|
+
list.push(result);
|
|
824
|
+
ended.set(result.id, list);
|
|
825
|
+
if (list.filter((r) => !isCompleted(r)).length >= e.row.plan.attempts) start(result.id);
|
|
826
|
+
},
|
|
827
|
+
/** Stops calls in flight and asks nothing more: the run was canceled. */
|
|
828
|
+
cancel() {
|
|
829
|
+
controller.abort();
|
|
830
|
+
},
|
|
831
|
+
/**
|
|
832
|
+
* What a canceled run keeps: each attempt whose answers all came back before the cancel,
|
|
833
|
+
* and, for an attempt the cancel cut short, each question as an error, so the record shows
|
|
834
|
+
* it went unanswered. Waits for the aborted calls to unwind.
|
|
835
|
+
*/
|
|
836
|
+
async answered() {
|
|
837
|
+
await Promise.all(perAttempt.values());
|
|
838
|
+
const attempts = new Map(done);
|
|
839
|
+
for (const [key, questions] of asking) if (!attempts.has(key)) attempts.set(key, questions.map((q) => unanswered(q, STOPPED_FIRST)));
|
|
840
|
+
return {
|
|
841
|
+
evals: /* @__PURE__ */ new Map(),
|
|
842
|
+
attempts
|
|
843
|
+
};
|
|
844
|
+
},
|
|
845
|
+
async settle() {
|
|
846
|
+
for (const id of byId.keys()) if (ended.has(id)) start(id);
|
|
847
|
+
const evals = /* @__PURE__ */ new Map();
|
|
848
|
+
for (const [id, answers] of started) evals.set(id, await answers);
|
|
849
|
+
for (const answers of perAttempt.values()) await answers;
|
|
850
|
+
for (const [id, e] of byId) unshown(e, ended.get(id) ?? []);
|
|
851
|
+
return {
|
|
852
|
+
evals,
|
|
853
|
+
attempts: new Map(done)
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
//#endregion
|
|
859
|
+
export { questionRunner };
|
|
860
|
+
|
|
861
|
+
//# sourceMappingURL=questions-BRhU8MzB.js.map
|