reeval 0.13.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/THIRD_PARTY_NOTICES.md +190 -190
- package/assets/gateway-models.mjs +74 -0
- package/dist/{adapter-IKKo4sZj.js → adapter-CbqbMR7y.js} +55 -14
- package/dist/adapter-CbqbMR7y.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-Zgn-kiqq.js} +105 -35
- package/dist/agent-Zgn-kiqq.js.map +1 -0
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-ChbvRx3P.js +11 -0
- package/dist/app-files-ChbvRx3P.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-SKQGW_QF.js} +213 -136
- package/dist/assemble-SKQGW_QF.js.map +1 -0
- package/dist/assemble-XBWuK3Ed.js +1 -0
- package/dist/{attempt-jI0OWxlz.js → attempt-Ba2QerOK.js} +10 -10
- package/dist/{attempt-jI0OWxlz.js.map → attempt-Ba2QerOK.js.map} +1 -1
- package/dist/attempt-host-BOp8JklD.js +295 -0
- package/dist/attempt-host-BOp8JklD.js.map +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-BM_3100g.js} +600 -53
- package/dist/capture-BM_3100g.js.map +1 -0
- package/dist/capture-DSK8O5bA.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-PdYrcoKM.js} +177 -249
- package/dist/cli-PdYrcoKM.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-D0qyrPZi.js} +9 -21
- package/dist/cli-spec-D0qyrPZi.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-Ct6s6OnF.js} +106 -3
- package/dist/client-Ct6s6OnF.js.map +1 -0
- package/dist/{command-CANun0m0.js → command-4pJYVCT4.js} +34 -21
- package/dist/command-4pJYVCT4.js.map +1 -0
- package/dist/{validate-DWjkshl1.js → command-9xm5ImqI.js} +420 -60
- package/dist/command-9xm5ImqI.js.map +1 -0
- package/dist/{command-D_kEnYcX.js → command-BqBxLgTc.js} +33 -31
- package/dist/command-BqBxLgTc.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-DbSMcxZL.js} +35 -15
- package/dist/command-DbSMcxZL.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-BU6YJCz9.js} +183 -71
- package/dist/commands-BU6YJCz9.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-Bi2SEQtr.js} +1 -1
- package/dist/config-k6m7XbAz.js +3777 -0
- package/dist/config-k6m7XbAz.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DLZF4dGF.js} +3 -2
- package/dist/contrasts-DLZF4dGF.js.map +1 -0
- package/dist/cost-C5RBTTJr.js +1 -0
- package/dist/cost-ChPa8-Fp.js +359 -0
- package/dist/cost-ChPa8-Fp.js.map +1 -0
- package/dist/db-CpsifVln.js +1530 -0
- package/dist/db-CpsifVln.js.map +1 -0
- package/dist/db-DkD-F-X-.js +2 -0
- package/dist/{debug-D24NOQH1.js → debug-BhizgAsU.js} +2 -1
- package/dist/{debug-D24NOQH1.js.map → debug-BhizgAsU.js.map} +1 -1
- package/dist/debug-DeM5pswK.js +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-51WvhN_X.js} +5 -4
- package/dist/docker-51WvhN_X.js.map +1 -0
- package/dist/{docker-CQxucRPe.js → docker-B4-rx4dA.js} +48 -14
- package/dist/docker-B4-rx4dA.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-YFj4H1Cr.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-BuQnfz4e.js} +390 -53
- package/dist/doctor-BuQnfz4e.js.map +1 -0
- package/dist/{env-BQqj93gu.js → env-CARj5Axq.js} +6 -3
- package/dist/env-CARj5Axq.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-BOFgzFDg.js} +117 -297
- package/dist/environment-BOFgzFDg.js.map +1 -0
- package/dist/{errors-r025B0Zb.js → errors-UjcLWelF.js} +2 -2
- package/dist/{errors-r025B0Zb.js.map → errors-UjcLWelF.js.map} +1 -1
- package/dist/eval-source-Dje3rERU.js +145 -0
- package/dist/eval-source-Dje3rERU.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BPP9s3s6.js} +406 -130
- package/dist/evaluate-BPP9s3s6.js.map +1 -0
- package/dist/evaluate-dqXccn4k.d.ts +44 -0
- package/dist/{files-CN1QfenC.js → files-Bi461Be2.js} +202 -91
- package/dist/files-Bi461Be2.js.map +1 -0
- package/dist/files-C2h2Cw4F.js +1 -0
- package/dist/files-Clipx4OW.js +37 -0
- package/dist/files-Clipx4OW.js.map +1 -0
- package/dist/files-DmUrRmmL.js +1 -0
- package/dist/{files-D27nUk0K.js → files-H24pz0hL.js} +20 -9
- package/dist/files-H24pz0hL.js.map +1 -0
- package/dist/gateway.d.ts +30 -0
- package/dist/gateway.js +39 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-CsVZkNNL.js} +3 -3
- package/dist/{git-rPfl7AD1.js.map → git-CsVZkNNL.js.map} +1 -1
- package/dist/harness-3NVyTSGK.js +392 -0
- package/dist/harness-3NVyTSGK.js.map +1 -0
- package/dist/harness.d.ts +33 -21
- package/dist/harness.js +1 -1
- package/dist/{images-m784ueGK.js → images-4mwhNkvh.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-4mwhNkvh.js.map} +1 -1
- package/dist/index.d.ts +34 -35
- package/dist/index.js +5 -6
- package/dist/init-BDEOopK7.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-cOk3D44b.js} +12 -15
- package/dist/init-cOk3D44b.js.map +1 -0
- package/dist/{invocations-D_ePcwyL.js → invocations-DW-lTk2f.js} +10 -6
- package/dist/invocations-DW-lTk2f.js.map +1 -0
- package/dist/judge-CKdgEXN7.js +575 -0
- package/dist/judge-CKdgEXN7.js.map +1 -0
- package/dist/judge.d.ts +8 -9
- package/dist/judge.js +7 -9
- package/dist/judge.js.map +1 -1
- package/dist/{live-DYd2QEyR.js → live-XJQ6nqK4.js} +2 -2
- package/dist/{live-DYd2QEyR.js.map → live-XJQ6nqK4.js.map} +1 -1
- package/dist/{matchers-Ddmc2z7H.js → matchers-DT2TaGix.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-DT2TaGix.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-Bq3G0OG1.js} +52 -45
- package/dist/model-Bq3G0OG1.js.map +1 -0
- package/dist/{model-UgHDMhI1.js → model-CTnyvh7H.js} +345 -382
- package/dist/model-CTnyvh7H.js.map +1 -0
- package/dist/model-D-SWrLZG.js +1 -0
- package/dist/model-g3XnzUv0.js +1 -0
- package/dist/model-vRiUycWh.js +77 -0
- package/dist/model-vRiUycWh.js.map +1 -0
- package/dist/{node-CGZYh4Fj.js → node-BVbbnFtv.js} +80 -51
- package/dist/node-BVbbnFtv.js.map +1 -0
- package/dist/opencode-providers-Bz0l7OZI.js +3276 -0
- package/dist/opencode-providers-Bz0l7OZI.js.map +1 -0
- package/dist/openrouter-models-CWRwLX-G.js +34 -0
- package/dist/openrouter-models-CWRwLX-G.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-puYi-zj2.js} +17 -9
- package/dist/pairs-puYi-zj2.js.map +1 -0
- package/dist/patch-CquGAGX5.js +76 -0
- package/dist/patch-CquGAGX5.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DKJiC5Sq.js} +5 -7
- package/dist/patch-DKJiC5Sq.js.map +1 -0
- package/dist/{plan-CZYgXnZh.js → plan-DdWkM6SW.js} +118 -46
- package/dist/plan-DdWkM6SW.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-D-CP6mMd.js} +254 -18
- package/dist/plugins-D-CP6mMd.js.map +1 -0
- package/dist/{preflight-D3HvKeAx.js → preflight-9ixuXzsx.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-9ixuXzsx.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-DTiVv0gY.js} +1 -1
- package/dist/{prepare-Dx7dOnJB.js → prepare-BfyikQit.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-BfyikQit.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-tmBBf4W8.js} +2 -2
- package/dist/prices-QaS-kjSQ.js +253 -0
- package/dist/prices-QaS-kjSQ.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-BizwUn3h.js} +4 -4
- package/dist/{provenance-C_-8y-x3.js.map → provenance-BizwUn3h.js.map} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-B6b9Tnyi.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-B6b9Tnyi.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-DkOupEBl.js} +1 -1
- package/dist/query-Clm2mNSm.js +1 -0
- package/dist/{query-DXthKmq0.js → query-DyM-VExS.js} +3 -3
- package/dist/{query-DXthKmq0.js.map → query-DyM-VExS.js.map} +1 -1
- package/dist/questions-BRhU8MzB.js +861 -0
- package/dist/questions-BRhU8MzB.js.map +1 -0
- package/dist/{record-kk9M1hj-.js → record-DZBCjuwm.js} +6 -6
- package/dist/{record-kk9M1hj-.js.map → record-DZBCjuwm.js.map} +1 -1
- package/dist/redact-DVIaahLv.js +658 -0
- package/dist/redact-DVIaahLv.js.map +1 -0
- package/dist/remarks-BwJ8pcTQ.js +1 -0
- package/dist/remarks-C7i2LMRb.js +596 -0
- package/dist/remarks-C7i2LMRb.js.map +1 -0
- package/dist/{header-BVzbRnNv.js → remarks-D_3qhBWU.js} +34 -9
- package/dist/remarks-D_3qhBWU.js.map +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-dA8LyoPc.js} +12 -6
- package/dist/remarks-dA8LyoPc.js.map +1 -0
- package/dist/remote-DFIzcx_6.js +273 -0
- package/dist/remote-DFIzcx_6.js.map +1 -0
- package/dist/resolve--rQZ8u6Z.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-DWSJpKHX.js} +48 -33
- package/dist/resolve-DWSJpKHX.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-BC3pcG_Z.js} +15 -30
- package/dist/results-BC3pcG_Z.js.map +1 -0
- package/dist/{resume-8-nnW8iO.js → resume-j-rcIryi.js} +47 -17
- package/dist/resume-j-rcIryi.js.map +1 -0
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-d56Gg3J5.js} +42 -13
- package/dist/runtime-d56Gg3J5.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-Cmln11gU.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-Cmln11gU.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/{server-De8oH3bt.js → server-BOoH10J8.js} +69 -52
- package/dist/server-BOoH10J8.js.map +1 -0
- package/dist/server-DpuVAbrO.js +1 -0
- package/dist/{shared-CA61BuPJ.js → shared-B_lvGqWz.js} +20 -12
- package/dist/shared-B_lvGqWz.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/source-BFd6f0-R.js +1 -0
- package/dist/{source-C686eZpY.js → source-wD04-rRG.js} +15 -10
- package/dist/source-wD04-rRG.js.map +1 -0
- package/dist/{src-CzQrAcF8.js → src-1lpTocbd.js} +152 -75
- package/dist/src-1lpTocbd.js.map +1 -0
- package/dist/{src-DrJSnoMR.js → src-6_zz7oJs.js} +1 -1
- package/dist/{src-cbCb-MPy.js → src-8yTF251P.js} +68 -22
- package/dist/{src-cbCb-MPy.js.map → src-8yTF251P.js.map} +1 -1
- package/dist/{src-Xj_BZKbq.js → src-BO8AYqJ5.js} +45 -18
- package/dist/src-BO8AYqJ5.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-BYDOXw_Y.js} +26 -10
- package/dist/{src-DeY52Gyf.js.map → src-BYDOXw_Y.js.map} +1 -1
- package/dist/src-BYPMM1x7.js +2 -0
- package/dist/src-Bp4aN3rA.js +2 -0
- package/dist/src-C7upnZSd.js +2 -0
- package/dist/{src-CIpKL45U.js → src-D91Hd6hT.js} +566 -1070
- package/dist/src-D91Hd6hT.js.map +1 -0
- package/dist/src-DH-9IO7f.js +1 -0
- package/dist/{src-djNNYIdP.js → src-DSZyg0Uq.js} +178 -29
- package/dist/src-DSZyg0Uq.js.map +1 -0
- package/dist/{src-DIL0jST0.js → src-DgHuStEi.js} +159 -31
- package/dist/src-DgHuStEi.js.map +1 -0
- package/dist/src-HSxBazcg.js +3 -0
- package/dist/src-LD1BAJFa.js +2 -0
- package/dist/src-dlX-Lmxg.js +2 -0
- package/dist/{src-C7W6kg81.js → src-gn7YlnRg.js} +64 -17
- package/dist/{src-C7W6kg81.js.map → src-gn7YlnRg.js.map} +1 -1
- package/dist/src-kYeaJJrl.js +2 -0
- package/dist/state-B5PLm5qY.js +1 -0
- package/dist/{state-C0i_ujQZ.js → state-DrJ687ut.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-DrJ687ut.js.map} +1 -1
- package/dist/stats-hfpuYsAM.js +315 -0
- package/dist/stats-hfpuYsAM.js.map +1 -0
- package/dist/suite-22Kt9Grx.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-Co8cwelb.js} +92 -42
- package/dist/suite-Co8cwelb.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DLFHVoux.js} +131 -280
- package/dist/summary-DLFHVoux.js.map +1 -0
- package/dist/testing.d.ts +449 -2
- package/dist/testing.js +16 -3
- package/dist/testing.js.map +1 -0
- package/dist/{text-CrVy5ioL.js → text-DTigcA64.js} +9 -5
- package/dist/text-DTigcA64.js.map +1 -0
- package/dist/{transcript-BTxJpW4R.js → transcript-DxPaWHTZ.js} +12 -4
- package/dist/transcript-DxPaWHTZ.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-coMWHaU2.d.ts} +493 -745
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/vertex-Bv7vhMgs.js +1 -0
- package/dist/vertex-C7PI-WPc.d.ts +3 -0
- package/dist/vertex-DypYjx71.js +456 -0
- package/dist/vertex-DypYjx71.js.map +1 -0
- package/dist/view/index.d.ts +505 -43
- package/dist/view/index.js +10 -9
- package/dist/{views-Dk8MCQKr.js → views-BFO1BLKu.js} +14 -13
- package/dist/views-BFO1BLKu.js.map +1 -0
- package/dist/views-BOf8qEL8.js +2 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-CJtdsKBP.js} +21 -14
- package/dist/worker-entry-CJtdsKBP.js.map +1 -0
- package/dist/{workspace-CLeOybkr.js → workspace-BRxjfa8N.js} +5 -5
- package/dist/{workspace-CLeOybkr.js.map → workspace-BRxjfa8N.js.map} +1 -1
- package/dist/workspace-DDHzkMjm.js +1 -0
- package/docker/seccomp-default.json +1 -0
- package/package.json +10 -2
- package/skills/eval-rubric/SKILL.md +43 -103
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/agent-ClE-g9VL.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/cli-spec-BXIBUlaK.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-C6vwmjS7.js +0 -91
- package/dist/command-C6vwmjS7.js.map +0 -1
- package/dist/command-CANun0m0.js.map +0 -1
- package/dist/command-CBaHukyl.js +0 -335
- package/dist/command-CBaHukyl.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js +0 -602
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/debug-D7bEv_UV.js +0 -1
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/docker-DuHcQidD.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/env-BQqj93gu.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-C9-ritA5.d.ts +0 -92
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js +0 -898
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-Bs6H3HiG.js +0 -65
- package/dist/files-Bs6H3HiG.js.map +0 -1
- package/dist/files-CN1QfenC.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D27nUk0K.js.map +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js +0 -1646
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js +0 -323
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/header-BVzbRnNv.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/insights-B2Xmhz9o.js +0 -64
- package/dist/insights-B2Xmhz9o.js.map +0 -1
- package/dist/invocations-D_ePcwyL.js.map +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-B8H3gL69.js +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js +0 -3305
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/patch-uMq9rCl_.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/plugins-P-uh54VF.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js +0 -523
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/redact-CRGdfV2s.js +0 -240
- package/dist/redact-CRGdfV2s.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remarks-Ml6bcSdV.js +0 -278
- package/dist/remarks-Ml6bcSdV.js.map +0 -1
- package/dist/remote-Dkeghk8k.js +0 -378
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/results-D-naai_Q.js.map +0 -1
- package/dist/resume-8-nnW8iO.js.map +0 -1
- package/dist/rules-CcV3zLDR.js +0 -318
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/run-Bel7Vde4.js +0 -88
- package/dist/run-Bel7Vde4.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/server-De8oH3bt.js.map +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-C686eZpY.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-CIpKL45U.js.map +0 -1
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-Xj_BZKbq.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/src-djNNYIdP.js.map +0 -1
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/text-CrVy5ioL.js.map +0 -1
- package/dist/transcript-BTxJpW4R.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/worker-entry-CqrzGPF4.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -1,67 +1,22 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { a as fromModel } from "./insights-B2Xmhz9o.js";
|
|
7
|
-
import { o as warnings } from "./state-C0i_ujQZ.js";
|
|
1
|
+
import { a as plural, i as formatDuration, n as clip, r as fenced } from "./text-DTigcA64.js";
|
|
2
|
+
import { n as formatTokens } from "./usage-BNY1ABrU.js";
|
|
3
|
+
import { B as commandLine, C as isUngatedMiss, F as checkLabels, I as checkTries, S as isNotGatedPass, U as labeledJudgments, ft as formatSeconds, k as runtimeLabel, u as evalMarkOutcome, ut as formatCount, v as harnessLabel, w as languageLabel } from "./model-CTnyvh7H.js";
|
|
4
|
+
import { c as isStudy, r as countedOf, s as isNotGatedResult, t as canceledOf, u as suiteResult } from "./suite-Co8cwelb.js";
|
|
5
|
+
import { o as warnings } from "./state-DrJ687ut.js";
|
|
8
6
|
import { t as colors } from "./colors-QV9dFVhN.js";
|
|
9
|
-
import { r as remarkLines } from "./remarks-
|
|
10
|
-
//#region src/insights/hash.ts
|
|
11
|
-
/** JSON with every object's keys sorted. */
|
|
12
|
-
function canonicalJson(value) {
|
|
13
|
-
return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
|
|
14
|
-
}
|
|
15
|
-
const factsHash = (suite) => `sha256:${sha256(canonicalJson(suite))}`;
|
|
16
|
-
//#endregion
|
|
7
|
+
import { r as remarkLines } from "./remarks-dA8LyoPc.js";
|
|
17
8
|
//#region src/ui/layout.ts
|
|
18
|
-
/** The block for an eval: a pure function of its
|
|
19
|
-
function layout(
|
|
9
|
+
/** The block for an eval: a pure function of its shape. */
|
|
10
|
+
function layout(across, isFamily, isCompare) {
|
|
20
11
|
if (isCompare) return {
|
|
21
12
|
kind: "regressions",
|
|
22
13
|
family: false
|
|
23
14
|
};
|
|
24
15
|
return {
|
|
25
|
-
kind:
|
|
16
|
+
kind: across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
|
|
26
17
|
family: isFamily
|
|
27
18
|
};
|
|
28
19
|
}
|
|
29
|
-
const moved = (c) => c.effect === "better" || c.effect === "worse";
|
|
30
|
-
/** A cell the agent failed, or that errored; not one whose only fails the interrupt cancelled. */
|
|
31
|
-
const failedCell = (c) => c.outcome === "fail" && c.failed > cancelledOf(c) || c.outcome === "infra_error";
|
|
32
|
-
/**
|
|
33
|
-
* The insight slots an eval's layout opens: which rows take a sentence (a failing case; an arm
|
|
34
|
-
* that failed or moved; a failing harness), whether it has findings (a choose or finish eval that
|
|
35
|
-
* measures a change), and whether it has named tools (a choose eval). An `other` eval takes only
|
|
36
|
-
* its answer, unless it compares harnesses, which takes a sentence on any job. What the run's
|
|
37
|
-
* interrupt cancelled opens nothing: the agent didn't fail it, and its effects aren't measured.
|
|
38
|
-
*/
|
|
39
|
-
function slotsOf(job, e) {
|
|
40
|
-
const known = job !== "other";
|
|
41
|
-
const measured = e.outcome !== "cancelled";
|
|
42
|
-
const harnesses = e.across === "harness" || e.across === "both";
|
|
43
|
-
const rows = [];
|
|
44
|
-
if (known || harnesses) {
|
|
45
|
-
if (e.cases) {
|
|
46
|
-
for (const k of e.cases) if (k.outcome !== "pass" && k.outcome !== "cancelled") rows.push(k.path);
|
|
47
|
-
for (const p of e.pooled ?? []) {
|
|
48
|
-
const pooledMoved = measured && (p.effect === "better" || p.effect === "worse");
|
|
49
|
-
if (p.outcome === "fail" || pooledMoved) rows.push(p.path);
|
|
50
|
-
}
|
|
51
|
-
} else if (e.across === "harness") {
|
|
52
|
-
for (const c of e.cells ?? []) if (failedCell(c)) rows.push(c.path);
|
|
53
|
-
} else if (e.across === "change" || e.across === "both") {
|
|
54
|
-
for (const c of e.cells ?? []) if (failedCell(c) || measured && moved(c)) rows.push(c.path);
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
return {
|
|
58
|
-
rows,
|
|
59
|
-
findings: known && e.across === "change" && measured,
|
|
60
|
-
namedTools: job === "choose"
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
/** The last line of a run's output while no analysis model writes its answers. */
|
|
64
|
-
const NO_MODEL_HINT = "Set analysis.model in reeval.config.ts for answers to what each eval asks.";
|
|
65
20
|
/** The narrowest a row's label column is, so short labels line up under the name. */
|
|
66
21
|
const LABEL = 17;
|
|
67
22
|
const ASSERTION_MAX = 160;
|
|
@@ -72,7 +27,7 @@ const VERDICT = {
|
|
|
72
27
|
pass: "pass",
|
|
73
28
|
fail: "fail",
|
|
74
29
|
infra_error: "infra error",
|
|
75
|
-
|
|
30
|
+
canceled: "canceled",
|
|
76
31
|
skipped: "skipped"
|
|
77
32
|
};
|
|
78
33
|
const EFFECT = {
|
|
@@ -108,26 +63,23 @@ function table(items, width = LABEL) {
|
|
|
108
63
|
}).join(" ")}`.trimEnd();
|
|
109
64
|
});
|
|
110
65
|
}
|
|
111
|
-
const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "
|
|
112
|
-
/** A study (every cell `gate: 'none'`) passes whatever it saw, so it never reads `pass`. */
|
|
66
|
+
const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "canceled" ? c.yellow : c.dim;
|
|
113
67
|
function verdict(c, e) {
|
|
114
68
|
if (e.todo) return t("todo", c.dim);
|
|
115
|
-
|
|
116
|
-
if (e.outcome === "pass" && cells.length && cells.every((x) => !x.gated)) return t("not gated", c.dim);
|
|
69
|
+
if (e.outcome === "pass" && isStudy(e.cells ?? [])) return t("not gated", c.dim);
|
|
117
70
|
return t(VERDICT[e.outcome], paintOutcome(c, e.outcome));
|
|
118
71
|
}
|
|
119
|
-
const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
120
72
|
/**
|
|
121
73
|
* `3 of 4`, with the attempts that didn't count named: `2 of 3 · 1 infra error · 1 skipped`. A
|
|
122
|
-
*
|
|
74
|
+
* canceled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart.
|
|
123
75
|
*/
|
|
124
76
|
function cellCount(cell) {
|
|
125
|
-
const
|
|
77
|
+
const canceled = canceledOf(cell);
|
|
126
78
|
const counted = countedOf(cell);
|
|
127
79
|
const parts = [
|
|
128
80
|
counted > 0 ? `${cell.passed} of ${counted}` : "",
|
|
129
81
|
cell.infraError ? plural(cell.infraError, "infra error") : "",
|
|
130
|
-
|
|
82
|
+
canceled ? `${canceled} canceled` : "",
|
|
131
83
|
cell.skipped ? `${cell.skipped} skipped` : ""
|
|
132
84
|
].filter(Boolean);
|
|
133
85
|
if (!parts.length) return "not run";
|
|
@@ -154,7 +106,7 @@ const ERROR_KIND = {
|
|
|
154
106
|
assertion: "error",
|
|
155
107
|
judge: "judge",
|
|
156
108
|
timeout: "timeout",
|
|
157
|
-
aborted: "
|
|
109
|
+
aborted: "canceled",
|
|
158
110
|
infra: "infra error",
|
|
159
111
|
turn: "turn error"
|
|
160
112
|
};
|
|
@@ -175,22 +127,32 @@ function evidence(c, cell, indent, more = true) {
|
|
|
175
127
|
indent: indent + 2,
|
|
176
128
|
free: t(assertion, c.dim)
|
|
177
129
|
});
|
|
130
|
+
if (k.retried) items.push({
|
|
131
|
+
indent: indent + 2,
|
|
132
|
+
free: t(k.retried, c.dim)
|
|
133
|
+
});
|
|
178
134
|
}
|
|
179
135
|
for (const err of cell.errors) items.push({
|
|
180
136
|
indent,
|
|
181
137
|
cells: [errorKind(err), t(errorText(err), c.dim)]
|
|
182
138
|
});
|
|
183
|
-
|
|
139
|
+
items.push(...retriedItems(c, cell, indent));
|
|
140
|
+
const passed = cell.checks.filter((k) => !failing(k) && !k.retried).length;
|
|
184
141
|
if (more && passed && items.length) items.push({
|
|
185
142
|
indent,
|
|
186
143
|
free: t(`+${passed} more passed`, c.dim)
|
|
187
144
|
});
|
|
188
145
|
return items;
|
|
189
146
|
}
|
|
147
|
+
/** Passing checks that needed their `c.exec` retries: `npm test passed on try 2 of 3`. */
|
|
148
|
+
const retriedItems = (c, cell, indent) => cell.checks.filter((k) => !failing(k) && k.retried).map((k) => ({
|
|
149
|
+
indent,
|
|
150
|
+
cells: [checkLabel(k.label), t(k.retried ?? "", c.dim)]
|
|
151
|
+
}));
|
|
190
152
|
/** A control with no environment of its own is the baseline the variants change. */
|
|
191
153
|
const armOf = (cell) => cell.variant ?? (cell.environment === "-" ? "baseline" : cell.environment);
|
|
192
154
|
/** Each cell's label: the parts that vary between them (harness, arm, sandbox). */
|
|
193
|
-
function
|
|
155
|
+
function labeler(cells, parts) {
|
|
194
156
|
const vary = parts.filter((p) => {
|
|
195
157
|
const of = (cell) => p === "harness" ? cell.harness : p === "arm" ? armOf(cell) : cell.sandbox;
|
|
196
158
|
return new Set(cells.map(of)).size > 1;
|
|
@@ -269,73 +231,9 @@ function effortTable(c, cells, label, indent, failed, width) {
|
|
|
269
231
|
]
|
|
270
232
|
}))], width);
|
|
271
233
|
}
|
|
272
|
-
const NO_SLOTS = {
|
|
273
|
-
row: () => null,
|
|
274
|
-
named: () => [],
|
|
275
|
-
subject: null
|
|
276
|
-
};
|
|
277
|
-
const triesOf = (cells) => cells.reduce((n, x) => n + countedOf(x), 0);
|
|
278
|
-
/** `named`, then a row per tool with the tries that named it. */
|
|
279
|
-
function namedItems(c, slots, cells, indent) {
|
|
280
|
-
const named = slots.named(cells);
|
|
281
|
-
if (!named.length) return [];
|
|
282
|
-
const of = triesOf(cells);
|
|
283
|
-
return [{
|
|
284
|
-
indent,
|
|
285
|
-
free: t("named", c.dim)
|
|
286
|
-
}, ...named.map((n) => ({
|
|
287
|
-
indent: indent + 2,
|
|
288
|
-
cells: [n.tool, `${n.tries} of ${of}`]
|
|
289
|
-
}))];
|
|
290
|
-
}
|
|
291
|
-
/** The tries that named the subject, over the cells' tries: `0 of 4`; empty with no subject. */
|
|
292
|
-
function subjectTries(slots, cells) {
|
|
293
|
-
if (!slots.subject) return "";
|
|
294
|
-
const subject = slots.subject.toLowerCase();
|
|
295
|
-
return `${slots.named(cells).find((x) => x.tool.toLowerCase() === subject)?.tries ?? 0} of ${triesOf(cells)}`;
|
|
296
|
-
}
|
|
297
|
-
/** Web searches or fetches over the tries, when every try reported a number. */
|
|
298
|
-
function webItems(cells, indent) {
|
|
299
|
-
const row = (name, counts) => counts.length && counts.every((x) => x !== null) ? [{
|
|
300
|
-
indent,
|
|
301
|
-
cells: [name, String(counts.reduce((a, b) => (a ?? 0) + (b ?? 0), 0))]
|
|
302
|
-
}] : [];
|
|
303
|
-
return [...row("searches", cells.flatMap((x) => x.searches)), ...row("fetches", cells.flatMap((x) => x.fetches))];
|
|
304
|
-
}
|
|
305
|
-
/** A choose case that missed: its failed checks, the tools named, its searches, then `+N more passed`. */
|
|
306
|
-
function chooseDetail(c, cells, indent, slots) {
|
|
307
|
-
const checks = cells.flatMap((cell) => evidence(c, cell, indent, false));
|
|
308
|
-
const passed = cells.flatMap((x) => x.checks.filter((k) => !failing(k))).length;
|
|
309
|
-
return [
|
|
310
|
-
...checks,
|
|
311
|
-
...namedItems(c, slots, cells, indent),
|
|
312
|
-
...webItems(cells, indent),
|
|
313
|
-
...passed ? [{
|
|
314
|
-
indent,
|
|
315
|
-
free: t(`+${passed} more passed`, c.dim)
|
|
316
|
-
}] : []
|
|
317
|
-
];
|
|
318
|
-
}
|
|
319
|
-
/** Choose a tool, on a single eval: the subject's count, and on a miss the detail. */
|
|
320
|
-
function chooseBody(c, cells, indent, slots) {
|
|
321
|
-
const items = slots.subject ? [{
|
|
322
|
-
indent,
|
|
323
|
-
cells: [slots.subject, subjectTries(slots, cells)]
|
|
324
|
-
}] : [];
|
|
325
|
-
if (!cells.every(clean)) items.push(...chooseDetail(c, cells, indent, slots));
|
|
326
|
-
return table(items);
|
|
327
|
-
}
|
|
328
|
-
/** A row's sentence, as a line under it. */
|
|
329
|
-
const rowLine = (slots, path, indent) => {
|
|
330
|
-
const text = slots.row(path);
|
|
331
|
-
return text ? [{
|
|
332
|
-
indent,
|
|
333
|
-
free: text
|
|
334
|
-
}] : [];
|
|
335
|
-
};
|
|
336
234
|
/** Measure a change: a row per arm with its effect, the check matrix and the effort table. */
|
|
337
235
|
function measureBody(c, cells, indent, o) {
|
|
338
|
-
const label =
|
|
236
|
+
const label = labeler(cells, ["arm", "sandbox"]);
|
|
339
237
|
const arms = [{
|
|
340
238
|
indent,
|
|
341
239
|
cells: ["arm", t("pass", c.dim)]
|
|
@@ -350,7 +248,6 @@ function measureBody(c, cells, indent, o) {
|
|
|
350
248
|
...effect.text ? [effect] : []
|
|
351
249
|
]
|
|
352
250
|
});
|
|
353
|
-
arms.push(...rowLine(o.slots, cell.path, indent + 2), ...namedItems(c, o.slots, [cell], indent + 2));
|
|
354
251
|
const stats = o.verbose && o.effects ? statsLine(cell, cells) : null;
|
|
355
252
|
if (stats) arms.push({
|
|
356
253
|
indent: indent + 2,
|
|
@@ -371,8 +268,8 @@ function measureBody(c, cells, indent, o) {
|
|
|
371
268
|
];
|
|
372
269
|
}
|
|
373
270
|
/** A row per cell (harness, or sandbox), the ones that didn't pass outright with their evidence. */
|
|
374
|
-
function cellRows(c, cells, heading, indent
|
|
375
|
-
const label =
|
|
271
|
+
function cellRows(c, cells, heading, indent) {
|
|
272
|
+
const label = labeler(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
|
|
376
273
|
const items = [{
|
|
377
274
|
indent,
|
|
378
275
|
cells: [heading, t("pass", c.dim)]
|
|
@@ -382,14 +279,13 @@ function cellRows(c, cells, heading, indent, slots) {
|
|
|
382
279
|
indent,
|
|
383
280
|
cells: [label(cell), cellCount(cell)]
|
|
384
281
|
});
|
|
385
|
-
items.push(...
|
|
386
|
-
if (!clean(cell)) items.push(...namedItems(c, slots, [cell], indent + 2), ...evidence(c, cell, indent + 2));
|
|
282
|
+
if (!clean(cell)) items.push(...evidence(c, cell, indent + 2));
|
|
387
283
|
}
|
|
388
284
|
return table(items);
|
|
389
285
|
}
|
|
390
286
|
/** Compare harnesses with arms: a heading per harness, its arms as Measure a change under it. */
|
|
391
287
|
function bothBody(c, cells, indent, o) {
|
|
392
|
-
const harnessOf =
|
|
288
|
+
const harnessOf = labeler(cells, ["harness", "sandbox"]);
|
|
393
289
|
const groups = /* @__PURE__ */ new Map();
|
|
394
290
|
for (const cell of cells) groups.set(harnessOf(cell), [...groups.get(harnessOf(cell)) ?? [], cell]);
|
|
395
291
|
return [...groups].flatMap(([harness, group]) => [`${" ".repeat(indent)}${harness}`, ...measureBody(c, group, indent + 2, o)]);
|
|
@@ -397,15 +293,14 @@ function bothBody(c, cells, indent, o) {
|
|
|
397
293
|
/** The rows under an eval's head (or a family case's), by what its cells compare. */
|
|
398
294
|
function cellsBody(c, kind, cells, indent, o) {
|
|
399
295
|
if (kind === "measure") return measureBody(c, cells, indent, o);
|
|
400
|
-
if (kind === "harnesses") return cellRows(c, cells, "harness", indent
|
|
296
|
+
if (kind === "harnesses") return cellRows(c, cells, "harness", indent);
|
|
401
297
|
if (kind === "both") return bothBody(c, cells, indent, o);
|
|
402
|
-
if (
|
|
403
|
-
if (cells.length > 1) return cellRows(c, cells, "sandbox", indent, o.slots);
|
|
298
|
+
if (cells.length > 1) return cellRows(c, cells, "sandbox", indent);
|
|
404
299
|
return resultsBody(c, cells[0], indent);
|
|
405
300
|
}
|
|
406
301
|
/** A single eval with one cell: its tries, and on anything but a clean pass, the evidence. */
|
|
407
302
|
function resultsBody(c, cell, indent) {
|
|
408
|
-
if (clean(cell)) return
|
|
303
|
+
if (clean(cell)) return table(retriedItems(c, cell, indent));
|
|
409
304
|
const items = [];
|
|
410
305
|
if (cell.attempts > 1) items.push({
|
|
411
306
|
indent,
|
|
@@ -413,8 +308,34 @@ function resultsBody(c, cell, indent) {
|
|
|
413
308
|
});
|
|
414
309
|
return table([...items, ...evidence(c, cell, indent)]);
|
|
415
310
|
}
|
|
311
|
+
/** Free-text answers shown per question; the rest are in the record. */
|
|
312
|
+
const ANSWERS_SHOWN = 3;
|
|
313
|
+
function attemptQuestionLines(c, q, indent) {
|
|
314
|
+
const pad = " ".repeat(indent);
|
|
315
|
+
const head = `${pad}${c.dim("question")} ${clip(q.question, 100)}`;
|
|
316
|
+
const failed = q.error === null ? [] : [`${pad} ${c.yellow(`${q.failed} of ${q.asked} not answered: ${clip(q.error, 200)}`)}`];
|
|
317
|
+
if (q.choices) return [
|
|
318
|
+
head,
|
|
319
|
+
`${pad} ${q.choices.map((t) => `${t.choice} ${t.n}`).join(" · ")}`,
|
|
320
|
+
...failed
|
|
321
|
+
];
|
|
322
|
+
const shown = q.answers.slice(0, ANSWERS_SHOWN);
|
|
323
|
+
const rows = table(shown.map((a) => ({
|
|
324
|
+
indent: indent + 2,
|
|
325
|
+
cells: [a.attempt, clip(a.text.replace(/\s+/g, " "), 100)]
|
|
326
|
+
})));
|
|
327
|
+
const more = q.answers.length - shown.length;
|
|
328
|
+
const rest = more ? [`${pad} ${c.dim(`${more} more: reeval query attempt_answers`)}`] : [];
|
|
329
|
+
return [
|
|
330
|
+
head,
|
|
331
|
+
...rows,
|
|
332
|
+
...rest,
|
|
333
|
+
...failed
|
|
334
|
+
];
|
|
335
|
+
}
|
|
416
336
|
function questionLines(c, questions, indent) {
|
|
417
337
|
return questions.flatMap((q) => {
|
|
338
|
+
if (q.kind === "attempts") return attemptQuestionLines(c, q, indent);
|
|
418
339
|
const pad = " ".repeat(indent);
|
|
419
340
|
const head = `${pad}${c.dim("question")} ${clip(q.question, 100)}`;
|
|
420
341
|
if (q.votes === 0 && q.error === null) return [head, `${pad} ${c.dim("no pairs to judge: tries that failed their checks are left out")}`];
|
|
@@ -457,10 +378,9 @@ function pooledStats(p, label) {
|
|
|
457
378
|
/**
|
|
458
379
|
* A family: its template, then a row per case. With one cell a case is a row with its count;
|
|
459
380
|
* with more, the case is outer and its cells inner, laid out as a single eval's are, and a
|
|
460
|
-
* pooled `all cases` row sums each cell's counts with its variant's pooled effect.
|
|
461
|
-
* family's case row carries the subject's count instead, and a missed case the tools named.
|
|
381
|
+
* pooled `all cases` row sums each cell's counts with its variant's pooled effect.
|
|
462
382
|
*/
|
|
463
|
-
function familyBlock(c, e, kind, verbose
|
|
383
|
+
function familyBlock(c, e, kind, verbose) {
|
|
464
384
|
const cases = e.cases ?? [];
|
|
465
385
|
const columns = [...new Map(cases.flatMap((k) => k.cells).map((cell) => [cell.key, cell])).values()];
|
|
466
386
|
const pooled = e.pooled ?? [];
|
|
@@ -470,7 +390,7 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
470
390
|
todo: false,
|
|
471
391
|
cells: k.cells
|
|
472
392
|
});
|
|
473
|
-
const pooledWord = t(VERDICT[pooledOutcome], paintOutcome(c, pooledOutcome));
|
|
393
|
+
const pooledWord = isNotGatedResult(e) ? t("not gated", c.dim) : t(VERDICT[pooledOutcome], paintOutcome(c, pooledOutcome));
|
|
474
394
|
const words = [...cases.map(caseWord), ...pooled.length ? [pooledWord] : []];
|
|
475
395
|
const wordWidth = Math.max(...words.map((w) => w.text.length));
|
|
476
396
|
const lead = (word, name) => ({
|
|
@@ -479,38 +399,6 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
479
399
|
});
|
|
480
400
|
const head = `${" ".repeat(17)}${e.template ?? e.name}`;
|
|
481
401
|
const questions = questionLines(c, e.questions, 2);
|
|
482
|
-
if (kind === "choose") {
|
|
483
|
-
const subject = slots.subject ? [t(slots.subject, c.dim)] : [];
|
|
484
|
-
const items = [{
|
|
485
|
-
indent: 2,
|
|
486
|
-
cells: [
|
|
487
|
-
"case",
|
|
488
|
-
t("pass", c.dim),
|
|
489
|
-
...subject
|
|
490
|
-
]
|
|
491
|
-
}];
|
|
492
|
-
for (const k of cases) {
|
|
493
|
-
const passed = k.cells.reduce((n, x) => n + x.passed, 0);
|
|
494
|
-
const tries = triesOf(k.cells);
|
|
495
|
-
const count = k.cells.length === 1 && k.cells[0] ? cellCount(k.cells[0]) : tries ? `${passed} of ${tries}` : "not run";
|
|
496
|
-
const named = slots.subject && tries ? [subjectTries(slots, k.cells)] : [];
|
|
497
|
-
items.push({
|
|
498
|
-
indent: 2,
|
|
499
|
-
cells: [
|
|
500
|
-
lead(caseWord(k), k.label),
|
|
501
|
-
count,
|
|
502
|
-
...named
|
|
503
|
-
]
|
|
504
|
-
});
|
|
505
|
-
items.push(...rowLine(slots, k.path, 4));
|
|
506
|
-
if (!k.cells.every(clean)) items.push(...chooseDetail(c, k.cells, 4, slots));
|
|
507
|
-
}
|
|
508
|
-
return [
|
|
509
|
-
head,
|
|
510
|
-
...table(items),
|
|
511
|
-
...questions
|
|
512
|
-
];
|
|
513
|
-
}
|
|
514
402
|
if (columns.length <= 1) {
|
|
515
403
|
const items = [];
|
|
516
404
|
for (const k of cases) {
|
|
@@ -519,7 +407,6 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
519
407
|
indent: 2,
|
|
520
408
|
cells: [lead(caseWord(k), k.label), cell ? cellCount(cell) : ""]
|
|
521
409
|
});
|
|
522
|
-
items.push(...rowLine(slots, k.path, 4));
|
|
523
410
|
if (cell && !clean(cell)) items.push(...evidence(c, cell, 4));
|
|
524
411
|
}
|
|
525
412
|
return [
|
|
@@ -531,18 +418,15 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
531
418
|
const out = [head];
|
|
532
419
|
for (const k of cases) {
|
|
533
420
|
out.push(` ${show(lead(caseWord(k), k.label))}`);
|
|
534
|
-
const sentence = slots.row(k.path);
|
|
535
|
-
if (sentence) out.push(` ${sentence}`);
|
|
536
421
|
const failed = k.outcome !== "pass";
|
|
537
422
|
out.push(...cellsBody(c, kind, k.cells, 4, {
|
|
538
423
|
failed,
|
|
539
424
|
verbose,
|
|
540
|
-
effects: false
|
|
541
|
-
slots
|
|
425
|
+
effects: false
|
|
542
426
|
}));
|
|
543
427
|
}
|
|
544
428
|
if (pooled.length) {
|
|
545
|
-
const label =
|
|
429
|
+
const label = labeler(columns, [
|
|
546
430
|
"harness",
|
|
547
431
|
"arm",
|
|
548
432
|
"sandbox"
|
|
@@ -563,7 +447,6 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
563
447
|
...effect.text ? [effect] : []
|
|
564
448
|
]
|
|
565
449
|
});
|
|
566
|
-
if (p) items.push(...rowLine(slots, p.path, 6));
|
|
567
450
|
if (p && verbose) items.push({
|
|
568
451
|
indent: 6,
|
|
569
452
|
free: t(pooledStats(p, label(col)), c.dim)
|
|
@@ -573,43 +456,40 @@ function familyBlock(c, e, kind, verbose, slots) {
|
|
|
573
456
|
}
|
|
574
457
|
return [...out, ...questions];
|
|
575
458
|
}
|
|
576
|
-
/**
|
|
577
|
-
function
|
|
578
|
-
|
|
579
|
-
const
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
}
|
|
459
|
+
/** `Anthropic 12 · Amazon Bedrock 3`: each upstream provider and how many calls it served, summed. */
|
|
460
|
+
function upstreamList(counts) {
|
|
461
|
+
const total = /* @__PURE__ */ new Map();
|
|
462
|
+
for (const u of counts) total.set(u.provider, (total.get(u.provider) ?? 0) + u.calls);
|
|
463
|
+
return [...total].map(([provider, calls]) => `${provider} ${calls}`).join(" · ");
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Which upstream providers a gateway said served each harness's calls and its judge's, a line per
|
|
467
|
+
* harness, so a run OpenRouter spread over several shows it. None where the gateway named none.
|
|
468
|
+
*/
|
|
469
|
+
function upstreamLines(c, cells, indent) {
|
|
470
|
+
const by = /* @__PURE__ */ new Map();
|
|
471
|
+
for (const cell of cells) {
|
|
472
|
+
if (!cell.upstreams && !cell.judgeUpstreams) continue;
|
|
473
|
+
const seen = by.get(cell.harness) ?? {
|
|
474
|
+
agent: [],
|
|
475
|
+
judge: []
|
|
476
|
+
};
|
|
477
|
+
seen.agent.push(...cell.upstreams ?? []);
|
|
478
|
+
seen.judge.push(...cell.judgeUpstreams ?? []);
|
|
479
|
+
by.set(cell.harness, seen);
|
|
480
|
+
}
|
|
481
|
+
return [...by].map(([harness, { agent, judge }]) => {
|
|
482
|
+
const parts = [...agent.length ? [`served by ${upstreamList(agent)}`] : [], ...judge.length ? [`judge served by ${upstreamList(judge)}`] : []];
|
|
483
|
+
const label = by.size > 1 ? `${harness} ` : "";
|
|
484
|
+
return `${" ".repeat(indent)}${c.dim(`${label}${parts.join("; ")}`)}`;
|
|
485
|
+
});
|
|
598
486
|
}
|
|
599
|
-
/** One eval's block, laid out as `layout()` says for its
|
|
600
|
-
function evalBlock(c, e, verbose = false
|
|
601
|
-
const shape = layout(
|
|
602
|
-
const slots = slotsFor(insight);
|
|
603
|
-
const answer = insight?.answer ? [` ${oneLine(insight.answer.text)}`] : [];
|
|
604
|
-
const findings = (insight?.findings ?? []).map((f) => ` ${oneLine(f.text)}`);
|
|
487
|
+
/** One eval's block, laid out as `layout()` says for its shape. */
|
|
488
|
+
function evalBlock(c, e, verbose = false) {
|
|
489
|
+
const shape = layout(e.across, e.family, false);
|
|
605
490
|
if (shape.family) {
|
|
606
|
-
const
|
|
607
|
-
return [
|
|
608
|
-
head,
|
|
609
|
-
...answer,
|
|
610
|
-
...rest,
|
|
611
|
-
...findings
|
|
612
|
-
];
|
|
491
|
+
const served = upstreamLines(c, (e.cases ?? []).flatMap((k) => k.cells), 2);
|
|
492
|
+
return [...familyBlock(c, e, shape.kind, verbose), ...served];
|
|
613
493
|
}
|
|
614
494
|
const v = verdict(c, e);
|
|
615
495
|
const name = `${e.group.length ? `${e.group.join(" › ")} › ` : ""}${e.name}`;
|
|
@@ -617,40 +497,37 @@ function evalBlock(c, e, verbose = false, insight = null) {
|
|
|
617
497
|
const cells = e.cells ?? [];
|
|
618
498
|
if (e.todo || e.outcome === "skipped" || !cells.length) return [head];
|
|
619
499
|
const failed = e.outcome !== "pass";
|
|
620
|
-
const body = cellsBody(c, shape.kind, cells, 2, {
|
|
621
|
-
failed,
|
|
622
|
-
verbose,
|
|
623
|
-
effects: true,
|
|
624
|
-
slots
|
|
625
|
-
});
|
|
626
500
|
return [
|
|
627
501
|
head,
|
|
628
|
-
...
|
|
629
|
-
|
|
630
|
-
|
|
502
|
+
...cellsBody(c, shape.kind, cells, 2, {
|
|
503
|
+
failed,
|
|
504
|
+
verbose,
|
|
505
|
+
effects: true
|
|
506
|
+
}),
|
|
507
|
+
...upstreamLines(c, cells, 2),
|
|
631
508
|
...questionLines(c, e.questions, 2)
|
|
632
509
|
];
|
|
633
510
|
}
|
|
634
511
|
/**
|
|
635
|
-
* `5 fail · 2 pass · 1
|
|
636
|
-
* one, zeros left out.
|
|
512
|
+
* `5 fail · 2 pass · 1 not gated · 1 canceled`: errors outside any eval and fails first, every
|
|
513
|
+
* case counting one, zeros left out.
|
|
637
514
|
*/
|
|
638
515
|
function tallyLine(c, tally) {
|
|
639
516
|
const parts = [
|
|
640
517
|
tally.errors ? c.red(plural(tally.errors, "run error")) : "",
|
|
641
518
|
tally.fail ? c.red(`${tally.fail} fail`) : "",
|
|
642
519
|
tally.pass ? c.green(`${tally.pass} pass`) : "",
|
|
520
|
+
tally.notGated ? c.dim(`${tally.notGated} not gated`) : "",
|
|
643
521
|
tally.infraError ? c.yellow(plural(tally.infraError, "infra error")) : "",
|
|
644
|
-
tally.
|
|
522
|
+
tally.canceled ? c.yellow(`${tally.canceled} canceled`) : "",
|
|
645
523
|
tally.skipped ? c.dim(`${tally.skipped} skipped`) : ""
|
|
646
524
|
].filter(Boolean);
|
|
647
525
|
return parts.length ? parts.join(c.dim(" · ")) : "no evals ran";
|
|
648
526
|
}
|
|
649
527
|
/** Every eval's block in file-path order, then source order; a blank line around each longer one. */
|
|
650
|
-
function renderBlocks(suite, opts
|
|
528
|
+
function renderBlocks(suite, opts) {
|
|
651
529
|
const c = colors(opts.color);
|
|
652
|
-
const
|
|
653
|
-
const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose, byEval.get(e.path) ?? null)));
|
|
530
|
+
const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose)));
|
|
654
531
|
const out = [];
|
|
655
532
|
blocks.forEach((block, i) => {
|
|
656
533
|
const prev = blocks[i - 1];
|
|
@@ -659,51 +536,20 @@ function renderBlocks(suite, opts, insights = null) {
|
|
|
659
536
|
});
|
|
660
537
|
return out;
|
|
661
538
|
}
|
|
662
|
-
/** Insights for these facts: null when there are none, they failed, or they were written for other facts. */
|
|
663
|
-
const freshInsights = (suite, insights) => insights && !insights.error && insights.factsHash === factsHash(suite) ? insights : null;
|
|
664
|
-
/** Why a run's insights didn't print, or with `-v` what wrote them. */
|
|
665
|
-
function insightNotes(c, suite, insights, fresh, verbose) {
|
|
666
|
-
if (!insights || !fromModel(insights)) return [];
|
|
667
|
-
if (insights.error) return [c.yellow(`insights failed (${clip(insights.error, 200)})`)];
|
|
668
|
-
if (!fresh) return [c.dim(`insights left out: insights.json was written for other facts than this run's; delete it and run reeval analyze ${suite.runId} --insights`)];
|
|
669
|
-
const skipped = insights.budget?.skipped.length ?? 0;
|
|
670
|
-
const failed = insights.failed ?? [];
|
|
671
|
-
const stopped = insights.stopped;
|
|
672
|
-
const finish = `reeval analyze ${suite.runId} --insights to finish`;
|
|
673
|
-
const budget = [
|
|
674
|
-
...skipped ? [c.yellow(`insights skipped for ${plural(skipped, "eval")}: analysis.maxTokens (${formatTokens(insights.budget?.maxTokens ?? 0)}) reached — raise it in reeval.config.ts, or ${finish}`)] : [],
|
|
675
|
-
...failed[0] ? [c.yellow(`insights failed for ${plural(failed.length, "eval")} (${clip(failed[0].error, 160)}), so they print their facts: ${finish}`)] : [],
|
|
676
|
-
...stopped ? [c.yellow(`insights stopped: ${plural(stopped.skipped.length, "eval")} skipped (${stopped.signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"}): ${finish}`)] : []
|
|
677
|
-
];
|
|
678
|
-
if (!verbose) return budget;
|
|
679
|
-
const { dropped, regenerated } = insights.validation;
|
|
680
|
-
return [...budget, c.dim([
|
|
681
|
-
`insights by ${insights.model} (${insights.promptVersion})`,
|
|
682
|
-
`${plural(dropped, "sentence")} removed`,
|
|
683
|
-
...regenerated ? [`${regenerated} asked again`] : [],
|
|
684
|
-
`${formatTokens(insights.usage.inputTokens)} in, ${formatTokens(insights.usage.outputTokens)} out${insights.usage.reasoningTokens ? ` (${formatTokens(insights.usage.reasoningTokens)} reasoning)` : ""}`
|
|
685
|
-
].join(" · "))];
|
|
686
|
-
}
|
|
687
539
|
/**
|
|
688
|
-
* A run's result: its errors outside any eval, its blocks
|
|
689
|
-
*
|
|
690
|
-
* other facts are left out with a note; remarks come from the record and history, not a model,
|
|
691
|
-
* so they print either way.
|
|
540
|
+
* A run's result: its errors outside any eval, its blocks, then the tally and the remarks, which
|
|
541
|
+
* come from the record and the history.
|
|
692
542
|
*/
|
|
693
|
-
function render(suite,
|
|
543
|
+
function render(suite, remarks, opts) {
|
|
694
544
|
const c = colors(opts.color);
|
|
695
|
-
const
|
|
696
|
-
const blocks = renderBlocks(suite, opts, fresh);
|
|
545
|
+
const blocks = renderBlocks(suite, opts);
|
|
697
546
|
return [
|
|
698
547
|
...suite.errors.flatMap((error) => [c.red(`error: ${error}`), ""]),
|
|
699
548
|
...blocks,
|
|
700
549
|
...blocks.length ? [""] : [],
|
|
701
|
-
...fresh?.headline ? [oneLine(fresh.headline.text)] : [],
|
|
702
|
-
...insightNotes(c, suite, insights, fresh, opts.verbose),
|
|
703
550
|
...opts.notes ?? [],
|
|
704
551
|
tallyLine(c, suite.tally),
|
|
705
|
-
...
|
|
706
|
-
...remarkLines(c, suite, insights, opts.verbose)
|
|
552
|
+
...remarkLines(c, suite, remarks, opts.verbose)
|
|
707
553
|
];
|
|
708
554
|
}
|
|
709
555
|
//#endregion
|
|
@@ -742,7 +588,7 @@ function stateRows(run) {
|
|
|
742
588
|
return rows;
|
|
743
589
|
}
|
|
744
590
|
/** Markdown for people and agents landing on a run cold. Links are relative to the artifact dir. */
|
|
745
|
-
function renderSummary(run, opts) {
|
|
591
|
+
function renderSummary(run, opts = {}) {
|
|
746
592
|
const lines = [
|
|
747
593
|
`# reeval run ${run.runId}`,
|
|
748
594
|
"",
|
|
@@ -753,10 +599,9 @@ function renderSummary(run, opts) {
|
|
|
753
599
|
...run.plan.sandboxWarnings.map((w) => `> **Warning:** ${w.message}\n`),
|
|
754
600
|
"## Results",
|
|
755
601
|
"",
|
|
756
|
-
...fenced(render(suiteResult(run), opts.
|
|
602
|
+
...fenced(render(suiteResult(run), opts.remarks ?? null, {
|
|
757
603
|
color: false,
|
|
758
|
-
verbose: false
|
|
759
|
-
hint: opts.hint
|
|
604
|
+
verbose: false
|
|
760
605
|
}), "text"),
|
|
761
606
|
"",
|
|
762
607
|
"## State",
|
|
@@ -773,7 +618,8 @@ function renderSummary(run, opts) {
|
|
|
773
618
|
const rank = (o) => o === "fail" || o === "infra_error" ? 0 : o === "pass" ? 1 : 2;
|
|
774
619
|
for (const e of [...run.evals].sort((x, y) => rank(x.outcome) - rank(y.outcome))) {
|
|
775
620
|
if (e.outcome === "skipped") continue;
|
|
776
|
-
|
|
621
|
+
const mark = isNotGatedPass(e) ? "○" : OUTCOME_MARK[evalMarkOutcome(e)];
|
|
622
|
+
lines.push("", `### ${mark} ${e.id}${isNotGatedPass(e) ? " · not gated" : ""}`, "");
|
|
777
623
|
const multi = e.attempts.length > 1;
|
|
778
624
|
const inputs = (e.answers ?? []).filter((a) => a.input);
|
|
779
625
|
for (const a of inputs) lines.push(`- Question: ${cell(`"${clip(a.question, 80)}"`)} · [what the model saw](${a.input?.path})`);
|
|
@@ -808,14 +654,19 @@ function attemptDetail(lines, a) {
|
|
|
808
654
|
const act = a.activity;
|
|
809
655
|
const web = (n, what) => n === null ? `${what} not reported` : `${n} ${what}`;
|
|
810
656
|
lines.push(`- Agent: ${act.fileEdits} file edits · ${act.tools} other tool calls · ${web(act.webSearches, "web searches")} · ${web(act.webFetches, "fetches")} · ${formatDuration(act.reasoningMs)} reasoning${act.stalledTools ? ` · ${act.stalledTools} stalled tool${act.stalledTools === 1 ? "" : "s"} (≥2m)${act.maxToolMs ? `, longest ${formatDuration(act.maxToolMs)}` : ""}` : ""}`);
|
|
811
|
-
for (const {
|
|
657
|
+
for (const { judgment: j, label, labelled: labeled } of labeledJudgments(a)) {
|
|
812
658
|
const retried = j.retries?.length ? `, after ${j.retries.length} ${j.retries.length === 1 ? "retry" : "retries"}` : "";
|
|
813
|
-
lines.push(`- Judge${
|
|
659
|
+
lines.push(`- Judge${labeled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
|
|
660
|
+
}
|
|
661
|
+
for (const q of a.answers ?? []) {
|
|
662
|
+
const said = q.error ? `failed: ${clip(cell(q.error), 200)}` : `${q.choice !== void 0 ? `**${cell(q.choice)}** ` : ""}${clip(cell(q.text), 300)}`;
|
|
663
|
+
lines.push(`- Question ${cell(`"${clip(q.question, 80)}"`)}: ${said}${q.input ? ` ([what the model saw](${q.input.path}))` : ""}`);
|
|
814
664
|
}
|
|
815
665
|
const labels = checkLabels(a);
|
|
816
666
|
a.checks.forEach((c, i) => {
|
|
817
667
|
const label = labels[i];
|
|
818
|
-
|
|
668
|
+
const tries = checkTries(c);
|
|
669
|
+
lines.push(`- Check${label ? ` ${cell(label)}` : ""} exit ${c.exitCode}${tries ? ` (${tries})` : ""}: \`${cell(commandLine(c.command))}\`${c.log ? ` ([log](${c.log.path}))` : ""}`);
|
|
819
670
|
});
|
|
820
671
|
if (a.changedFiles.length) lines.push(`- Changed: ${a.changedFiles.map((f) => f.count ? `\`${f.path}\` (${f.count} files)` : `${f.status} \`${f.path}\``).join(", ")}`);
|
|
821
672
|
const last = a.turns.at(-1)?.text;
|
|
@@ -832,6 +683,6 @@ function attemptDetail(lines, a) {
|
|
|
832
683
|
if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
|
|
833
684
|
}
|
|
834
685
|
//#endregion
|
|
835
|
-
export {
|
|
686
|
+
export { assertionOf as a, errorKind as c, labeler as d, render as f, table as g, t as h, stateRows as i, errorText as l, show as m, renderSummary as n, checkCount as o, renderBlocks as p, runTally as r, checkLabel as s, OUTCOME_MARK as t, evalBlock as u };
|
|
836
687
|
|
|
837
|
-
//# sourceMappingURL=summary-
|
|
688
|
+
//# sourceMappingURL=summary-DLFHVoux.js.map
|