reeval 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/dist/{adapter-IKKo4sZj.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-ClE-g9VL.js → agent-m-_t-IuW.js} +53 -25
- package/dist/{agent-ClE-g9VL.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-B0nOfAzA.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-B0nOfAzA.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-BO93VAk1.js → assemble-BDEpelpi.js} +92 -35
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{capture-dn2hjSLj.js → capture-CVRwRTnb.js} +572 -36
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-BCE1xlm4.js → cli-C_4gaJ50.js} +132 -140
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C6vwmjS7.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C6vwmjS7.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CANun0m0.js → command-BXLUfw4h.js} +12 -12
- package/dist/{command-CANun0m0.js.map → command-BXLUfw4h.js.map} +1 -1
- package/dist/{command-CBaHukyl.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CBaHukyl.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-D_kEnYcX.js → command-Bxdy62dW.js} +14 -13
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-Bh3yhN_b.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-7tH3EsVu.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BKGzBMgb.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-B4nOhn0Z.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-DJUuFoWQ.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-Dp_AJgYN.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-Dp_AJgYN.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-DuHcQidD.js → docker-BnRlTpZA.js} +2 -2
- package/dist/{docker-DuHcQidD.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CQxucRPe.js → docker-D01slDF3.js} +26 -10
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-Dac6Zalz.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-De_QlOTM.js → doctor-DQ2p0euG.js} +135 -36
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-B4IuURK1.js → environment-CO0P0soF.js} +8 -7
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-Djk1YPFL.js → evaluate-BUmsN7yG.js} +66 -27
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-C9-ritA5.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-Cfx1H70y.js → facts-DzOUTreB.js} +12 -4
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-CN1QfenC.js → files-B-CJeeEh.js} +13 -8
- package/dist/{files-CN1QfenC.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-Bs6H3HiG.js → files-C_gQNVB-.js} +3 -3
- package/dist/{files-Bs6H3HiG.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-D27nUk0K.js → files-kUCrswI6.js} +2 -2
- package/dist/{files-D27nUk0K.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-BnrvBAzF.js → generate-DZ7vp69F.js} +18 -17
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-rPfl7AD1.js → git-Bk8DSwHy.js} +2 -2
- package/dist/{git-rPfl7AD1.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-1OkumOAm.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-BVzbRnNv.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-BVzbRnNv.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-m784ueGK.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-m784ueGK.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-hiTkiM4L.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-D_ePcwyL.js → invocations-DzyKSJBa.js} +2 -2
- package/dist/{invocations-D_ePcwyL.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-DpkclE4r.js → model-1ZLr9Dcv.js} +40 -26
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-UgHDMhI1.js → model-CgnOpss4.js} +57 -318
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/{model-B8H3gL69.js → model-Ch8kX87u.js} +1 -1
- package/dist/{node-CGZYh4Fj.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-snfeWd6Z.js → opencode-providers-klkqgzvb.js} +23 -52
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-uMq9rCl_.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-uMq9rCl_.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-CZYgXnZh.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-P-uh54VF.js → plugins-Dkp_H3pB.js} +4 -4
- package/dist/{plugins-P-uh54VF.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-BQlPTHVu.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-D3HvKeAx.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-D3HvKeAx.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DCwM2KKD.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-Dx7dOnJB.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-Dx7dOnJB.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-C_-8y-x3.js → provenance-CfZpFpAh.js} +2 -2
- package/dist/{provenance-C_-8y-x3.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CTBYvMxu.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-CF5UEaP8.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-CF5UEaP8.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-DXthKmq0.js → query-CemclveB.js} +2 -2
- package/dist/{query-DXthKmq0.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-KENpWnfV.js → questions-bAudIc04.js} +308 -45
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{remarks-Ml6bcSdV.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-Ml6bcSdV.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-Dkeghk8k.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-DzdWcIL6.js → resolve-CBoKzQMP.js} +8 -9
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-D-naai_Q.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-D-naai_Q.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-8-nnW8iO.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-8-nnW8iO.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DNuLhaA5.js → root-Dlff-zp8.js} +4 -4
- package/dist/{root-DNuLhaA5.js.map → root-Dlff-zp8.js.map} +1 -1
- package/dist/{rules-CcV3zLDR.js → rules-VhftPOOT.js} +53 -5
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-Bel7Vde4.js → run-CwIEK16q.js} +9 -9
- package/dist/{run-Bel7Vde4.js.map → run-CwIEK16q.js.map} +1 -1
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-YLbk9mO7.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-YLbk9mO7.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-De8oH3bt.js → server-TGW4WyAz.js} +25 -26
- package/dist/{server-De8oH3bt.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-C686eZpY.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-C686eZpY.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-CIpKL45U.js → src-8zoX_qrb.js} +36 -26
- package/dist/{src-CIpKL45U.js.map → src-8zoX_qrb.js.map} +1 -1
- package/dist/{src-CzQrAcF8.js → src-CHIwOy0o.js} +84 -52
- package/dist/src-CHIwOy0o.js.map +1 -0
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-djNNYIdP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-djNNYIdP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/{src-DrJSnoMR.js → src-CyaDNG9r.js} +1 -1
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-C7W6kg81.js → src-DKPVX2Us.js} +34 -16
- package/dist/{src-C7W6kg81.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DeY52Gyf.js → src-DmUgeTlE.js} +4 -4
- package/dist/{src-DeY52Gyf.js.map → src-DmUgeTlE.js.map} +1 -1
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-Xj_BZKbq.js → src-NSdQ9zI6.js} +31 -16
- package/dist/{src-Xj_BZKbq.js.map → src-NSdQ9zI6.js.map} +1 -1
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-C0i_ujQZ.js → state-B1t143wG.js} +18 -18
- package/dist/{state-C0i_ujQZ.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-DmuF_ApO.js → suite-WgT7m6Vs.js} +76 -11
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-BFMHMTz2.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-B-9b1MDV.d.ts → types-DXBbP9Mk.d.ts} +298 -70
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-DWjkshl1.js → validate-BhwqrVsm.js} +44 -12
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +133 -86
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-Dk8MCQKr.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-CqrzGPF4.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-CqrzGPF4.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-CLeOybkr.js → workspace-CDI68wtX.js} +4 -4
- package/dist/{workspace-CLeOybkr.js.map → workspace-CDI68wtX.js.map} +1 -1
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +6 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-IKKo4sZj.js.map +0 -1
- package/dist/assemble-BO93VAk1.js.map +0 -1
- package/dist/assemble-VQxyoDxK.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-dn2hjSLj.js.map +0 -1
- package/dist/capture-nIJ-7kUE.js +0 -1
- package/dist/cli-BCE1xlm4.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-Bh3yhN_b.js.map +0 -1
- package/dist/command-D_kEnYcX.js.map +0 -1
- package/dist/commands-7tH3EsVu.js.map +0 -1
- package/dist/config-D7TFvUpL.js +0 -759
- package/dist/config-D7TFvUpL.js.map +0 -1
- package/dist/contrasts-B4nOhn0Z.js.map +0 -1
- package/dist/db-DJUuFoWQ.js.map +0 -1
- package/dist/db-W9EFlXnn.js +0 -2
- package/dist/docker-CQxucRPe.js.map +0 -1
- package/dist/doctor-De_QlOTM.js.map +0 -1
- package/dist/egress-DEAvq8Fl.js +0 -1258
- package/dist/egress-DEAvq8Fl.js.map +0 -1
- package/dist/environment-B4IuURK1.js.map +0 -1
- package/dist/evaluate-Djk1YPFL.js.map +0 -1
- package/dist/facts-Cfx1H70y.js.map +0 -1
- package/dist/files-CtjEXWXm.js +0 -1
- package/dist/files-D3gJa91B.js +0 -1
- package/dist/generate-BnrvBAzF.js.map +0 -1
- package/dist/harness-1OkumOAm.js.map +0 -1
- package/dist/init-hiTkiM4L.js.map +0 -1
- package/dist/init-kMkTgTZc.js +0 -1
- package/dist/judge-CEDIfRiU.js +0 -186
- package/dist/judge-CEDIfRiU.js.map +0 -1
- package/dist/model-CTiuzkWw.js +0 -46
- package/dist/model-CTiuzkWw.js.map +0 -1
- package/dist/model-DcIbmggK.js +0 -1
- package/dist/model-DpkclE4r.js.map +0 -1
- package/dist/model-UgHDMhI1.js.map +0 -1
- package/dist/node-CGZYh4Fj.js.map +0 -1
- package/dist/opencode-providers-snfeWd6Z.js.map +0 -1
- package/dist/options-CyAyLjgJ.js +0 -1129
- package/dist/options-CyAyLjgJ.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-CZYgXnZh.js.map +0 -1
- package/dist/query-D_1ARx5q.js +0 -1
- package/dist/questions-KENpWnfV.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-Dkeghk8k.js.map +0 -1
- package/dist/resolve-CMmhACr6.js +0 -1
- package/dist/resolve-DzdWcIL6.js.map +0 -1
- package/dist/rules-CcV3zLDR.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-C-ysqkLv.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-DjgypUHW.js +0 -1
- package/dist/src-3mP821Pt.js +0 -2
- package/dist/src-B3KCLZVk.js +0 -3
- package/dist/src-BAyg6CtQ.js +0 -2
- package/dist/src-BezgGX4X.js +0 -2
- package/dist/src-ChBhk06Q.js +0 -2
- package/dist/src-CoqZYspK.js +0 -2
- package/dist/src-CzQrAcF8.js.map +0 -1
- package/dist/src-D9fSTxlc.js +0 -1
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-dhSTRN50.js +0 -2
- package/dist/state-BGWGAwdo.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-Bb_cid9k.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-DmuF_ApO.js.map +0 -1
- package/dist/suite-mn9g4veR.js +0 -2
- package/dist/summary-BFMHMTz2.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-DWjkshl1.js.map +0 -1
- package/dist/views-DE2A1jEX.js +0 -2
- package/dist/views-Dk8MCQKr.js.map +0 -1
- package/dist/workspace-B1eGO46M.js +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"match-Bz0nSdzN.js","names":[],"sources":["../src/attempt/match.ts"],"mappings":";;;;;AAsBA,MAAM,aAAa,UACjB,YAAY,SACZ,OAAO,MAAM,WAAW,YACxB,OAAO,KAAK,KAAK,CAAC,CAAC,OAAO,MAAM,MAAM,YAAY,MAAM,OAAO;AAEjE,MAAM,cACJ,UAEA,OAAO,KAAK,KAAK,CAAC,CAAC,WAAW,KAC9B,eAAe,SACf,OAAO,MAAM,cAAc,YAC3B,aAAa,SACb,OAAO,MAAM,YAAY,YACzB,MAAM,YAAY,QAClB,CAAC,MAAM,QAAQ,MAAM,OAAO;;;;;AAM9B,IAAM,kBAAN,cAA8B,OAAO;CACnC,YAAqB,OAAO,WAAW;EACrC,OAAO;CACT;CAEA;CACA;CACA;CAEA,YAAY,WAAmB,SAAkC,UAAoB;EACnF,MAAM,MAAM;EACZ,KAAK,YAAY;EACjB,KAAK,UAAU;EACf,KAAK,WAAW;CAClB;CAEA,KAAc,OAAwB;EACpC,MAAM,OAAO,OAAO,KAAK;EACzB,IAAI,OAAO,OAAO,KAAK,SAAS,IAAI,GAAG,OAAO,KAAK,QAAQ,UAAU;EACrE,MAAM,UAAU,KAAK,SAAS,IAAI,KAAK,SAAS,qBAAK,IAAI,IAAI;EAC7D,KAAK,SAAS,IAAI,KAAK,WAAW,QAAQ,IAAI,IAAI,CAAC;EACnD,OAAO;CACT;CAEA,KAAc,OAAuC;EACnD,MAAM,OAAO,OAAO,KAAK;EACzB,OAAO,KAAK,KAAK,IAAI,IAChB,OAAO,OAAO,CAAC,IAAI,GAAG;GAAE,OAAO;GAAG,OAAO;EAAK,CAAC,IAChD;CACN;CAEA,IAAa,SAAiB;EAC5B,OAAO,KAAK;CACd;CAEA,WAA4B;EAC1B,OAAO,KAAK;CACd;AACF;;;;;AAMA,SAAgB,SAAS,OAAgB,2BAAqB,IAAI,IAAI,GAAY;CAChF,IAAI,MAAM,QAAQ,KAAK,GAAG,OAAO,MAAM,KAAK,MAAM,SAAS,GAAG,QAAQ,CAAC;CACvE,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM,OAAO;CACxD,IAAI,WAAW,KAAK,GAAG,OAAO,IAAI,gBAAgB,MAAM,WAAW,MAAM,SAAS,QAAQ;CAC1F,IAAI,UAAU,KAAK,GACjB,IAAI;EACF,OAAO,IAAI,OAAO,MAAM,QAAQ,MAAM,KAAK;CAC7C,SAAS,OAAO;EACd,MAAM,IAAI,WACR,GAAG,KAAK,UAAU,KAAK,EAAE,+BAAgC,MAAgB,SAC3E;CACF;CAEF,OAAO,OAAO,YAAY,OAAO,QAAQ,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC;AAC7F;AAEA,SAAS,UAAU,GAAmB,SAAyC;CAC7E,IAAI,UAAU,SAAS;EACrB,MAAM,OAAO,EAAE,MAAM,QAAQ;EAC7B,IAAI,CAAC,MACH,MAAM,IAAI,WACR,4BAA4B,QAAQ,KAAK,wBAAwB,EAAE,MAAM,OAAO,OAAO,EAAE,MAAM,WAAW,IAAI,KAAK,KACrH;EAEF,OAAO;CACT;CACA,IAAI,aAAa,SAAS,OAAO;CACjC,IAAI,WAAW,SAAS,OAAO,QAAQ;CACvC,IAAI,UAAU,SAAS,OAAO,QAAQ,UAAU,EAAE,KAAK,QAAQ,IAAI;CACnE,OAAO,EAAE,SAAS,QAAQ,QAAQ;AACpC;;AAGA,SAAS,YAAY,SAAoD;CACvE,IAAI,UAAU,SAAS,OAAO,QAAQ;CACtC,IAAI,EAAE,WAAW,UAAU,OAAO,KAAA;CAClC,MAAM,QAAQ,QAAQ;CACtB,OAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,aAAa,QAC9D,MAAM,WACP,KAAA;AACN;AAEA,eAAe,KACb,GACA,MACA,MACA,UACkB;CAClB,MAAM,OAAO,KAAK,KAAK,KAAK,QAAQ,SAAS,KAAK,QAAQ,CAAC;CAC3D,IAAI,QAAQ;CACZ,IAAI;EACF,MAAM,SAAS,MAAM,UAAU,GAAG,KAAK,OAAO;EAC9C,MAAM,UAAU,WAAW,MAAM,QAAQ,MAAM,EAAE,KAAK,KAAK,IAAI,CAAC;EAChE,QAAQ,mBAAmB;EAC3B,MAAM,SAAS,MAAM;EACrB,IAAI,SAAS,OAAO,GAAG,OAAO;GAAE,QAAQ;GAAQ;EAAM;EACtD,IAAI,OAAO,SAAS,KAAK,KAAK,OAAO;GAAE,QAAQ;GAAQ;EAAM;EAC7D,MAAM,UAAU,OAAO,QAAQ;EAC/B,MAAM,QAAQ,IAAI,MAAM,KAAK,QAAQ,GAAG,KAAK,MAAM,IAAI,YAAY,OAAO;EAC1E,MAAM,OAAO;EACb,OAAO;GAAE,QAAQ;GAAQ;GAAO;EAAM;CACxC,SAAS,OAAO;EACd,IAAI,CAAC,eAAe,KAAK,GAAG,MAAM;EAClC,OAAO;GAAE,QAAQ;GAAS;GAAO;EAAM;CACzC;AACF;AAEA,MAAM,iBAAiB,cAAuC,EAC5D,UAAU,OAAO,YAAY,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,aAAa,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EACjG;;;;;;;AAQA,eAAsB,MACpB,GACA,MACuC;CACvC,MAAM,OAAO,KAAK;CAClB,IAAI,CAAC,cAAc,IAAI,GACrB,MAAM,IAAI,WACR,GAAG,KAAK,mCAAmC,OAAO,KAAK,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,EAC9E;CAEF,MAAM,YAAY,YAAY,IAAI;CAClC,MAAM,2BAAqB,IAAI,IAAI;CACnC,IAAI,UAAU,MAAM,KAAK,GAAG,MAAM,MAAM,QAAQ;CAChD,IAAI,SAAS,OAAO,GAAG,OAAO,cAAc,QAAQ;CACpD,IAAI,KAAK,MAAM;EACb,MAAM,EAAE,WAAW,IAAI,UAAU,QAAS,KAAK;EAC/C,MAAM,WAAW,KAAK,IAAI,IAAI;EAC9B,OAAO,QAAQ,WAAW,UAAU,KAAK,IAAI,IAAI,WAAW,UAAU;GACpE,MAAM,IAAI,SAAS,YAAY,WAAW,SAAS,QAAQ,CAAC;GAC5D,UAAU,MAAM,KAAK,GAAG,MAAM,MAAM,QAAQ;GAC5C,IAAI,SAAS,OAAO,GAAG,OAAO,cAAc,QAAQ;EACtD;CACF;CACA,MAAM,UAAU,YAAY,KAAK,OAAO;CACxC,OAAO;EACL,SAAS;GAAC,KAAK,QAAQ;GAAQ,KAAK,OAAO;GAAO;EAAI,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,GAAG;EAChF,MAAM,KAAK,KAAK,KAAK,QAAQ,SAAS,GAAG,CAAC;EAC1C,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;EAC3C,GAAI,KAAK,QAAQ,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;EAC1C,MAAM,KAAK;EACX,QAAQ,QAAQ;EAChB,GAAI,QAAQ,WAAW,SAAS,CAAC,IAAI,EAAE,OAAO,QAAQ,MAAM;EAC5D,GAAI,QAAQ,SAAS,KAAK,OACtB,EAAE,YAAY,KAAK,MAAM,YAAY,IAAI,IAAI,SAAS,EAAE,IACxD,CAAC;CACP;AACF;;;;;AAMA,eAAsB,WACpB,GACA,OACA,MACyD;CAEzD,MAAM,OAAqB;EACzB,SAAS;GAAC,KAAK,QAAQ;GAAQ,KAAK,OAAO;GAAO,KAAK;EAAO,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,GAAG;EACxF,MAAM,KAAK,KAAK,KAAK,QAAQ,SAAS,GAAG,CAAC;EAC1C,SAAS,YAAY,KAAK,OAAO;EACjC,GAAI,KAAK,QAAQ,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;EAC1C,MAAM,KAAK;EACX,QAAQ;CACV;CACA,MAAM,KAAK,IAAI;CACf,IAAI;CACJ,IAAI;EACF,SAAS,MAAM,MAAM,GAAG,IAAI;CAC9B,SAAS,OAAO;EACd,MAAM,OAAO,MAAM,QAAQ,IAAI,GAAG,CAAC;EACnC,MAAM;CACR;CACA,IAAI,cAAc,QAAQ;EACxB,MAAM,OAAO,MAAM,QAAQ,IAAI,GAAG,CAAC;EACnC,OAAO;CACT;CACA,OAAO,OAAO,MAAM,MAAM;CAC1B,OAAO,EAAE,WAAW,iBAAiB,KAAK,CAAC,CAAC,MAAM,QAAQ,IAAI,GAAsB;AACtF;;;;;AAMA,SAAgB,WAAW,OAAuB,MAAmC;CACnF,MAAM,UAAU,IAAI,MAAM,KAAK,WAAW,GAAG,KAAK,SAAS,KAAK,WAAW,KAAK,QAAQ,QAAQ;CAChG,QAAQ,OAAO;CACf,MAAM,KAAK;EACT,SAAS,KAAK;EACd,MAAM,CAAC;EACP,GAAI,KAAK,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,KAAK,QAAQ;EAC9D,GAAI,KAAK,UAAU,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,KAAK,MAAM;EACxD,MAAM,KAAK,QAAQ;EACnB,QAAQ,KAAK;EACb,GAAI,KAAK,WAAW,SAAS,CAAC,IAAI,EAAE,OAAO,QAAQ;CACrD,CAAC;CACD,OAAO,iBAAiB,KAAK,CAAC,CAAC,GAAG,EAAE;AACtC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { t as isObjectLike } from "./object-CLf2-tNq.js";
|
|
2
|
-
import { a as
|
|
2
|
+
import { a as calledMcp, c as changed, d as ran, f as searched, l as edited, o as calledSkill, s as calledTool, u as fetched } from "./client-D5hNjpEx.js";
|
|
3
3
|
import { isDeepStrictEqual } from "node:util";
|
|
4
4
|
//#region src/matchers.ts
|
|
5
5
|
const matcher = (run) => run;
|
|
@@ -115,4 +115,4 @@ const evalMatchers = Object.fromEntries(Object.keys(matchers).filter((name) => !
|
|
|
115
115
|
//#endregion
|
|
116
116
|
export { runMatcher as i, evalMatchers as n, isMatcherName as r, MATCHERS as t };
|
|
117
117
|
|
|
118
|
-
//# sourceMappingURL=matchers-
|
|
118
|
+
//# sourceMappingURL=matchers-D4eUqx4j.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"matchers-
|
|
1
|
+
{"version":3,"file":"matchers-D4eUqx4j.js","names":[],"sources":["../src/matchers.ts"],"mappings":";;;;AA+BA,MAAM,WAAoE,QAAmB;;AAG7F,MAAM,QAAQ,GAAW,QAAQ,IAAI,QAAQ,QAAS;CACpD,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI;CAC5D,OAAO,KAAK,UAAU,QAClB,OACA,MAAM,KAAK,SAAS,MAAM,sBAAsB,KAAK,MAAM,CAAC,KAAK;AACvE;AAIA,MAAM,YAAY,UAChB,aAAa,KAAK,KAAK,OAAO,MAAM,aAAa;AAEnD,MAAM,QAAQ,UAA2B;CACvC,IAAI,OAAO,UAAU,UAAU,OAAO,IAAI,MAAM;CAChD,IAAI;EACF,OAAO,KAAK,UAAU,KAAK,KAAK,OAAO,KAAK;CAC9C,QAAQ;EACN,OAAO,OAAO,KAAK;CACrB;AACF;AAEA,MAAa,WAAW;CACtB,MAAM,SAAS,QAAQ,CAAC,cAAyB;EAC/C,MAAM,OAAO,OAAO,GAAG,QAAQ,QAAQ;EACvC,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,aAAa,KAAK,QAAQ,EAAE,0BACrD,YAAY,KAAK,MAAM,EAAE,SAAS,KAAK,QAAQ,EAAE;EACzD;CACF,CAAC;CAED,SAAS,SAAS,QAAQ,CAAC,cAAyB;EAClD,MAAM,OAAO,kBAAkB,QAAQ,QAAQ;EAC/C,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,uBAAuB,KAAK,QAAQ,MAC7D,YAAY,KAAK,MAAM,EAAE,mBAAmB,KAAK,QAAQ;EACjE;CACF,CAAC;CAED,UAAU,SAAS,QAAQ,OAAW;EACpC,MAAM,WAAW;EACjB,eACE,WAAW,OACP,YAAY,KAAK,MAAM,EAAE,mBACzB,YAAY,KAAK,MAAM,EAAE;CACjC,EAAE;CAEF,YAAY,SAAS,QAAQ,OAAW;EACtC,MAAM,CAAC,CAAC;EACR,eACE,SAAS,YAAY,KAAK,MAAM,EAAE,gBAAgB,YAAY,KAAK,MAAM,EAAE;CAC/E,EAAE;CAEF,WAAW,SAAS,QAAQ,CAAC,YAAuB;EAClD,MAAM,OACJ,OAAO,WAAW,YAAY,OAAO,WAAW,WAC5C,OAAO,SAAS,MAAM,IACtB,MAAM,QAAQ,MAAM,KAAK,OAAO,SAAS,MAAM;EACrD,OAAO;GACL;GACA,eACE,OACI,YAAY,KAAK,MAAM,EAAE,kBAAkB,KAAK,MAAM,MACtD,YAAY,KAAK,MAAM,EAAE,cAAc,KAAK,MAAM;EAC1D;CACF,CAAC;CAED,YAAY,QACV,OAAO,QAAQ,CAAC,MAAM,aAAwD;EAC5E,IAAI,CAAC,SAAS,MAAM,GAAG,MAAM,IAAI,UAAU,iDAAiD;EAC5F,MAAM,UAAU,MAAM,OAAO,SAAS,IAAI;EAC1C,IAAI,YAAY,MAAM,OAAO;GAAE,MAAM;GAAO,eAAe,iBAAiB,KAAK;EAAW;EAC5F,IAAI,YAAY,KAAA,GAAW,OAAO;GAAE,MAAM;GAAM,eAAe,uBAAuB;EAAO;EAC7F,MAAM,OAAO,OAAO,YAAY,WAAW,QAAQ,SAAS,OAAO,IAAI,QAAQ,KAAK,OAAO;EAC3F,OAAO;GACL;GACA,eACE,YAAY,KAAK,GAAG,OAAO,SAAS,GAAG,WAAW,QAAQ,QAAQ,KAAK,QAAQ,KAAK,OAAO;EAC/F;CACF,CACF;CAEA,WAAW,QAAQ,OAAO,QAAQ,MAAU;EAC1C,MAAM,IAAK,MAAM;EACjB,IAAI,OAAO,GAAG,aAAa,UAAU,MAAM,IAAI,UAAU,mCAAmC;EAC5F,OAAO;GACL,MAAM,EAAE,aAAa;GACrB,eACE,EAAE,aAAa,IACX,6BACA,0CAA0C,EAAE,SAAS,oBAClC,KAAK,EAAE,MAAM,EAAE,oBAAoB,KAAK,EAAE,MAAM;EAC3E;CACF,CAAC;CAED,gBAAgB,SAAS,QAAQ,CAAC,WAAsC,SAAS,QAAQ,KAAK,CAAC;CAE/F,eAAe,SAAS,QAAQ,CAAC,SAAkC,QAAQ,QAAQ,GAAG,CAAC;CAEvF,mBAAmB,SAAS,QAAQ,CAAC,UAA0B,YAAY,QAAQ,IAAI,CAAC;CAExF,iBAAiB,SACd,QAAQ,CAAC,QAAQ,MAAM,WACtB,UAAU,QAAQ,QAAQ,MAAM,KAAK,CACzC;CAEA,kBAAkB,SACf,QAAQ,CAAC,MAAM,WACd,WAAW,QAAQ,MAAM,KAAK,CAClC;CAEA,WAAW,SAAS,QAAQ,CAAC,aAA0C,IAAI,QAAQ,OAAO,CAAC;CAE3F,cAAc,SAAS,QAAQ,CAAC,UAAoC,OAAO,QAAQ,IAAI,CAAC;CAExF,eAAe,SAAS,QAAQ,CAAC,WAAgC,QAAQ,QAAQ,KAAK,CAAC;CAEvF,QAAQ,SAAS,QAAQ,GAAO,EAAE,UAAkB;EAClD,MAAM,UAAU;EAChB,IAAI,SAAS,YAAY,WAAW,OAAO,QAAQ,UAAU,UAE3D,OAAO;GACL,MAAM;GACN,eACE,qBAAqB,QAAQ,SAAS,KAAK,QAAQ,MAAM,KAAK,QAAQ;EAC1E;EAEF,MAAM,IAAI;EACV,IAAI,OAAO,GAAG,UAAU,UAAU,MAAM,IAAI,UAAU,sCAAsC;EAC5F,MAAM,QAAQ,EAAE,MAAM,QAAQ,CAAC;EAC/B,OAAO;GACL,MAAM,EAAE,YAAY;GACpB,eACE,EAAE,YAAY,SACV,iCAAiC,MAAM,KAAK,EAAE,UAAU,QAAQ,EAAE,SAAS,KAC3E,eAAe,MAAM,YAAY,EAAE,UAAU,QAAQ,EAAE,SAAS,KAAK,EAAE,MAAM;EACrF;CACF,CAAC;AACH;;AAKA,MAAM,mCAAmB,IAAI,IAAiB;CAC5C;CACA;CACA;CACA;CACA;AACF,CAAC;AAKD,MAAa,iBAAiB,SAAsC,OAAO,OAAO,UAAU,IAAI;;AAGhG,MAAa,cACX,MACA,QACA,MACA,UAEC,SAAS,KAAK,CAA8C,QAAQ,MAAM,KAAK;;AASlF,MAAa,WAAW,OAAO,YAC5B,OAAO,KAAK,QAAQ,CAAC,CAAmB,KAAK,SAAS,CACrD,MACA,SAAiD,QAAiB,GAAG,MAAiB;CACpF,OAAO,WAAW,MAAM,QAAQ,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC;AAC9D,CACF,CAAC,CACH;;AAGA,MAAa,eAAe,OAAO,YAChC,OAAO,KAAK,QAAQ,CAAC,CACnB,QAAQ,SAAS,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC,CAC7C,KAAK,SAAS,CAAC,MAAM,SAAS,KAAK,CAAC,CACzC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { n as analysisModelOf, v as gatewayScope, y as resolveModelGateway } from "./config-BrdHA9bJ.js";
|
|
3
|
+
import { a as checkModelString, l as judgeRoute, o as isFakeJudge, p as unusedGatewayKey, s as judgeCredentials, u as modelKeyOptions } from "./judge-yLvjC5A3.js";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { Output, generateText } from "ai";
|
|
6
6
|
//#region src/analysis/schema.ts
|
|
@@ -73,7 +73,7 @@ Rules:
|
|
|
73
73
|
- When the digest's frame is study, every cell is ungated (gate: 'none'): the run is a study of how its arms differ, not a failing CI gate. The TL;DR says how the arms differ (the criteria rows, and contrast facts for each pair of arms), not that attempts failed. A label in a criteria fact's neverMet or alwaysMet is the same in every arm: name those once (as never met, or met, in any arm), and never make one a why or the reason for an eval change; never suggest rewriting the task prompt to name criteria the eval deliberately leaves out.
|
|
74
74
|
- Take numbers from the facts. Don't compute new statistics beyond simple ratios of cited numbers.
|
|
75
75
|
- A cited judge score must be the recorded score on that attempt. Do not invent a different number.
|
|
76
|
-
- The verdicts in the facts are final: if a lift or effort verdict says within noise, insufficient or not comparable, say so; never call it an effect. insufficient is not measured: never put it on a results line as an effect. The one exception is effort whose separated is true on a metric at n of at least 3 vs 3: every attempt on one side beat every attempt on the other, so put it under results as directional, with the ratio, the n and "no overlap, not significant". insufficient on effort is statistical only (too few attempts); a missing token metric is omitted, not insufficient — do not treat unreportedUsage as a result. When sources or the eval fingerprint differ across the compared runs, the first line says the eval changed; that pass@1 difference is not an effect, and mixed counts such as 6/10 vs 8/9 are not a results line. A lift is an effect only when sources match. Name each factor claim's tier (controlled, pooled, observed). Contrast facts (kind contrast) set two arms side by side (a against b, variant against variant too): cite them when the question is how two arms differ. Check facts (kind checks) list each expect label on a variant and its control: a both-pass or both-fail check is no evidence for the variant; cite discriminates rows for which requirement separated them. Compare-question facts (kind compare-question) are annotation: win rates, ties, errors and vote reasons never change pass, fail or the exit code. Do not put them on a results line as a gate. A vote's reason is the model's wording: do not treat it as a check.
|
|
76
|
+
- The verdicts in the facts are final: if a lift or effort verdict says within noise, insufficient or not comparable, say so; never call it an effect. insufficient is not measured: never put it on a results line as an effect. The one exception is effort whose separated is true on a metric at n of at least 3 vs 3: every attempt on one side beat every attempt on the other, so put it under results as directional, with the ratio, the n and "no overlap, not significant". insufficient on effort is statistical only (too few attempts); a missing token metric is omitted, not insufficient — do not treat unreportedUsage as a result. When sources or the eval fingerprint differ across the compared runs, the first line says the eval changed; that pass@1 difference is not an effect, and mixed counts such as 6/10 vs 8/9 are not a results line. A lift is an effect only when sources match. Name each factor claim's tier (controlled, pooled, observed). Contrast facts (kind contrast) set two arms side by side (a against b, variant against variant too): cite them when the question is how two arms differ. Check facts (kind checks) list each expect label on a variant and its control: a both-pass or both-fail check is no evidence for the variant; cite discriminates rows for which requirement separated them. Compare-question facts (kind compare-question) are annotation: win rates, ties, errors and vote reasons never change pass, fail or the exit code. Do not put them on a results line as a gate. A vote's reason is the model's wording: do not treat it as a check. Attempt-question facts (kind attempt-question) are annotation too: a model's answer about one attempt, and how many attempts picked each choice; they never gate, and an answer is the model's reading, not a check.
|
|
77
77
|
- Say what can't be attributed (factors that are not_measured).
|
|
78
78
|
- Separate causes: the thing under test (skill, docs), the eval itself (task, checks, judge question), and reeval or infrastructure (bugs, fingerprints, harness).
|
|
79
79
|
- Explain a failed judge verdict by comparing the cited output with the judge question, and say it's your reading: judges record no reason.
|
|
@@ -86,41 +86,47 @@ Rules:
|
|
|
86
86
|
//#region src/analysis/model.ts
|
|
87
87
|
const isFake = (m) => typeof m === "object" && m !== null && m.kind === "fake-analysis";
|
|
88
88
|
const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
|
|
89
|
-
/**
|
|
90
|
-
|
|
89
|
+
/**
|
|
90
|
+
* The model reeval falls back to for analysis and compare questions: `analysis.model`, else
|
|
91
|
+
* `judge` when it's a language model. `judge` is the run's judge model (a profile's over the
|
|
92
|
+
* config's); the config's when none is given.
|
|
93
|
+
*/
|
|
94
|
+
function fallbackModel(config, judge = config.judge.model) {
|
|
91
95
|
const model = analysisModelOf(config);
|
|
92
96
|
if (model !== void 0) return model;
|
|
93
|
-
const judge = config.judge.model;
|
|
94
97
|
return typeof judge === "string" && !isEvaluationModel(judge) ? judge : void 0;
|
|
95
98
|
}
|
|
96
99
|
/** ` (the judge, typesafe-ai/jev-1.14.0, is an evaluation model: it scores yes/no questions and can't <task>)`, when it is one. */
|
|
97
|
-
function judgeAside(
|
|
98
|
-
const judge = config.judge.model;
|
|
100
|
+
function judgeAside(judge, task) {
|
|
99
101
|
if (typeof judge !== "string" && !isFakeJudge(judge)) return "";
|
|
100
102
|
return ` (the judge, ${isFakeJudge(judge) ? "a fake judge" : judge}, is an evaluation model: it scores yes/no questions and can't ${task})`;
|
|
101
103
|
}
|
|
102
104
|
/** Which model analyses: `analysis.model`, else the judge's when it's a language model. */
|
|
103
105
|
function analysisModelName(config) {
|
|
104
|
-
const model =
|
|
106
|
+
const model = fallbackModel(config);
|
|
105
107
|
if (model !== void 0) return model;
|
|
106
|
-
throw new UsageError(`reeval analyze needs a language model. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(config, "write an analysis")}, or run reeval analyze --context to hand the digest to your own coding agent.`);
|
|
108
|
+
throw new UsageError(`reeval analyze needs a language model. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(config.judge.model, "write an analysis")}, or run reeval analyze --context to hand the digest to your own coding agent.`);
|
|
107
109
|
}
|
|
108
|
-
/**
|
|
109
|
-
|
|
110
|
-
|
|
110
|
+
/**
|
|
111
|
+
* The plan's name for the model a question without its own is answered by; null when none.
|
|
112
|
+
* `judge` is the run's judge model.
|
|
113
|
+
*/
|
|
114
|
+
function questionModelLabel(config, judge = config.judge.model) {
|
|
115
|
+
const model = fallbackModel(config, judge);
|
|
111
116
|
if (model === void 0) return null;
|
|
112
117
|
return isFake(model) ? "fake" : model;
|
|
113
118
|
}
|
|
114
119
|
/**
|
|
115
|
-
* The model that answers a
|
|
116
|
-
* UsageError (exit 2) when that's missing or an evaluation model.
|
|
120
|
+
* The model that answers a question: its own, else the fallback, with `judge` the run's judge
|
|
121
|
+
* model. Throws a UsageError (exit 2) when that's missing or an evaluation model.
|
|
117
122
|
*/
|
|
118
|
-
function questionModel(config, question, where) {
|
|
123
|
+
function questionModel(config, question, where, judge = config.judge.model) {
|
|
119
124
|
if (question.model !== void 0) return question.model;
|
|
120
|
-
const
|
|
125
|
+
const task = question.across !== void 0 ? "pick between two attempts" : "answer in words";
|
|
126
|
+
const model = fallbackModel(config, judge);
|
|
121
127
|
const fix = `give the question its own, { question: '…', model: 'google/gemini-3.8-flash' }, or set question: { model } on its group or eval`;
|
|
122
|
-
if (model === void 0) throw new UsageError(`
|
|
123
|
-
if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`
|
|
128
|
+
if (model === void 0) throw new UsageError(`questions "${question.question}" (${where}) needs a language model to answer it. Set analysis: { model: 'google/gemini-3.8-flash' } in ${config.file}${judgeAside(judge, task)}, or ${fix}.`);
|
|
129
|
+
if (typeof model === "string" && isEvaluationModel(model)) throw new UsageError(`questions "${question.question}" (${where}) would be answered by analysis.model in ${config.file}, "${model}", an evaluation model: it scores yes/no questions and can't ${task}. Use a language model, e.g. 'google/gemini-3.8-flash', or ${fix}.`);
|
|
124
130
|
return model;
|
|
125
131
|
}
|
|
126
132
|
/**
|
|
@@ -132,13 +138,20 @@ function questionModel(config, question, where) {
|
|
|
132
138
|
function analysisKeyProblem(config, env = process.env, brief = false) {
|
|
133
139
|
const model = analysisModelOf(config);
|
|
134
140
|
if (typeof model !== "string" || isEvaluationModel(model) || checkModelString(model)) return null;
|
|
135
|
-
|
|
141
|
+
let gateway;
|
|
142
|
+
try {
|
|
143
|
+
gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
|
|
144
|
+
} catch (error) {
|
|
145
|
+
return brief ? "add vercel() to gateway, or analysis: { insights: false }" : `${error.message} Or analysis: { insights: false }.`;
|
|
146
|
+
}
|
|
147
|
+
const keys = judgeCredentials(model, gateway);
|
|
136
148
|
if (!keys.some((k) => env[k])) {
|
|
137
|
-
if (brief) return `set ${keys
|
|
138
|
-
|
|
149
|
+
if (brief) return `set ${keys.slice(0, 2).join(" or ")}, or analysis: { insights: false }`;
|
|
150
|
+
const unused = unusedGatewayKey(env, gateway);
|
|
151
|
+
return `set ${modelKeyOptions(model, gateway)} for analysis.model, "${model}", in .env.local, or analysis: { insights: false }${unused ? `.${unused}` : ""}`;
|
|
139
152
|
}
|
|
140
153
|
try {
|
|
141
|
-
judgeRoute(model, env, "analysis model");
|
|
154
|
+
judgeRoute(model, env, "analysis model", gateway);
|
|
142
155
|
return null;
|
|
143
156
|
} catch (error) {
|
|
144
157
|
return error.message;
|
|
@@ -153,10 +166,11 @@ function resolveAnalysisModel(config) {
|
|
|
153
166
|
fake: model
|
|
154
167
|
};
|
|
155
168
|
if (isEvaluationModel(model)) throw new UsageError(`analysis.model in ${config.file} is "${model}", an evaluation model: it scores yes/no questions and can't write an analysis. Use a language model, e.g. 'google/gemini-3.8-flash'.`);
|
|
169
|
+
const gateway = resolveModelGateway(model, void 0, gatewayScope(config), "analysis.model");
|
|
156
170
|
try {
|
|
157
171
|
return {
|
|
158
172
|
name: model,
|
|
159
|
-
route: judgeRoute(model, process.env, "analysis model").kind
|
|
173
|
+
route: judgeRoute(model, process.env, "analysis model", gateway).kind
|
|
160
174
|
};
|
|
161
175
|
} catch (error) {
|
|
162
176
|
throw new UsageError(`${error.message} Or run reeval analyze --context, which needs no key.`);
|
|
@@ -227,7 +241,7 @@ async function languageModel(model, digest) {
|
|
|
227
241
|
if (model.fake) return fakeModel(model.fake, digest);
|
|
228
242
|
return providerModel(model);
|
|
229
243
|
}
|
|
230
|
-
/** The provider's own model for a direct route,
|
|
244
|
+
/** The provider's own model for a direct route, or the AI Gateway's by name on vercel. */
|
|
231
245
|
async function providerModel(model) {
|
|
232
246
|
if (model.route === "gateway") return model.name;
|
|
233
247
|
const provider = model.name.slice(0, model.name.indexOf("/"));
|
|
@@ -265,4 +279,4 @@ async function callAnalysisModel(model, digest, prompt, maxOutputTokens) {
|
|
|
265
279
|
//#endregion
|
|
266
280
|
export { isFake as a, questionModelLabel as c, cannedModel as i, resolveAnalysisModel as l, analysisModelName as n, providerModel as o, callAnalysisModel as r, questionModel as s, analysisKeyProblem as t, SYSTEM_PROMPT as u };
|
|
267
281
|
|
|
268
|
-
//# sourceMappingURL=model-
|
|
282
|
+
//# sourceMappingURL=model-1ZLr9Dcv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-1ZLr9Dcv.js","names":[],"sources":["../src/analysis/schema.ts","../src/analysis/model.ts"],"mappings":";;;;;;AAGA,MAAM,aAAa,EAAE,KAAK;CAAC;CAAQ;CAAU;AAAK,CAAC;AACnD,MAAM,QAAQ,EACX,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,uFAAuF;AAEnG,MAAM,QAAQ,EAAE,OAAO;CACrB,MAAM,EAAE,OAAO;CACf;CACA;CACA,QAAQ,EACL,MAAM,EAAE,OAAO,CAAC,CAAC,CACjB,SAAS,CAAC,CACV,SAAS,qEAAqE;AACnF,CAAC;AAED,MAAa,iBAAiB,EAAE,OAAO;CACrC,MAAM,EACH,OAAO,CAAC,CACR,SACC,2HACF;CACF,cAAc,EAAE,MAAM,KAAK;CAC3B,SAAS,EACN,MAAM,KAAK,CAAC,CACZ,SACC,uLACF;CACF,KAAK,EACF,MAAM,KAAK,CAAC,CACZ,SAAS,yEAAyE;CACrF,WAAW,EAAE,MACX,EAAE,OAAO;EACP,MAAM,EAAE,OAAO;EACf,MAAM,EAAE,KAAK;GAAC;GAAc;GAAe;GAAS;GAAS;EAAO,CAAC;EACrE;CACF,CAAC,CACH;CACA,SAAS,EACN,MACC,EAAE,OAAO;EACP,OAAO,EAAE,OAAO;EAChB,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD;EAC/E,MAAM,EAAE,KAAK;GAAC;GAAS;GAAQ;GAAQ;GAAS;GAAU;EAAS,CAAC;EACpE,QAAQ,EAAE,OAAO;EACjB,gBAAgB,EAAE,OAAO,CAAC,CAAC,SAAS,+BAA+B;EACnE,QAAQ,EACL,OAAO,CAAC,CACR,SACC,sIACF;EACF,QAAQ,EAAE,KAAK;GAAC;GAAQ;GAAU;EAAK,CAAC;EACxC,QAAQ,EAAE,KAAK;GAAC;GAAO;GAAU;EAAM,CAAC;EACxC;CACF,CAAC,CACH,CAAC,CACA,IAAI,CAAC;CACR,YAAY,EAAE,OAAO;EAAE,SAAS;EAAY,KAAK,EAAE,OAAO;CAAE,CAAC;AAC/D,CAAC;AAID,MAAa,gBAAgB;;;;;;;;;;;;;;;;;;;;;ACvC7B,MAAa,UAAU,MACrB,OAAO,MAAM,YAAY,MAAM,QAAS,EAAwB,SAAS;AAE3E,MAAM,qBAAqB,UAAkB,MAAM,WAAW,cAAc;;;;;;AAO5E,SAAS,cACP,QACA,QAAgC,OAAO,MAAM,OACL;CACxC,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,UAAU,YAAY,CAAC,kBAAkB,KAAK,IAAI,QAAQ,KAAA;AAC1E;;AAGA,SAAS,WAAW,OAA+B,MAAsB;CACvE,IAAI,OAAO,UAAU,YAAY,CAAC,YAAY,KAAK,GAAG,OAAO;CAC7D,OAAO,gBAAgB,YAAY,KAAK,IAAI,iBAAiB,MAAM,iEAAiE,KAAK;AAC3I;;AAGA,SAAgB,kBAAkB,QAAoD;CACpF,MAAM,QAAQ,cAAc,MAAM;CAClC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,MAAM,IAAI,WACR,gGAAgG,OAAO,OAAO,WAAW,OAAO,MAAM,OAAO,mBAAmB,EAAE,+EACpK;AACF;;;;;AAMA,SAAgB,mBACd,QACA,QAAgC,OAAO,MAAM,OAC9B;CACf,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,IAAI,UAAU,KAAA,GAAW,OAAO;CAChC,OAAO,OAAO,KAAK,IAAI,SAAS;AAClC;;;;;AAMA,SAAgB,cACd,QACA,UACA,OACA,QAAgC,OAAO,MAAM,OACjB;CAC5B,IAAI,SAAS,UAAU,KAAA,GAAW,OAAO,SAAS;CAClD,MAAM,OAAO,SAAS,WAAW,KAAA,IAAY,8BAA8B;CAC3E,MAAM,QAAQ,cAAc,QAAQ,KAAK;CACzC,MAAM,MAAM;CACZ,IAAI,UAAU,KAAA,GACZ,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,+FAA+F,OAAO,OAAO,WAAW,OAAO,IAAI,EAAE,OAAO,IAAI,EAC7L;CAEF,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,GACtD,MAAM,IAAI,WACR,cAAc,SAAS,SAAS,KAAK,MAAM,2CAA2C,OAAO,KAAK,KAAK,MAAM,+DAA+D,KAAK,6DAA6D,IAAI,EACpP;CAEF,OAAO;AACT;;;;;;;AAQA,SAAgB,mBACd,QACA,MAAyB,QAAQ,KACjC,QAAQ,OACO;CACf,MAAM,QAAQ,gBAAgB,MAAM;CACpC,IAAI,OAAO,UAAU,YAAY,kBAAkB,KAAK,KAAK,iBAAiB,KAAK,GAAG,OAAO;CAC7F,IAAI;CACJ,IAAI;EACF,UAAU,oBAAoB,OAAO,KAAA,GAAW,aAAa,MAAM,GAAG,gBAAgB;CACxF,SAAS,OAAO;EACd,OAAO,QACH,8DACA,GAAI,MAAgB,QAAQ;CAClC;CACA,MAAM,OAAO,iBAAiB,OAAO,OAAO;CAC5C,IAAI,CAAC,KAAK,MAAM,MAAM,IAAI,EAAE,GAAG;EAC7B,IAAI,OAAO,OAAO,OAAO,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE;EACvD,MAAM,SAAS,iBAAiB,KAAK,OAAO;EAC5C,OAAO,OAAO,gBAAgB,OAAO,OAAO,EAAE,wBAAwB,MAAM,oDAAoD,SAAS,IAAI,WAAW;CAC1J;CACA,IAAI;EACF,WAAW,OAAO,KAAK,kBAAkB,OAAO;EAChD,OAAO;CACT,SAAS,OAAO;EACd,OAAQ,MAAgB;CAC1B;AACF;;AAGA,SAAgB,qBAAqB,QAA+C;CAClF,MAAM,QAAQ,kBAAkB,MAAM;CACtC,IAAI,OAAO,KAAK,GAAG,OAAO;EAAE,MAAM;EAAQ,OAAO;EAAQ,MAAM;CAAM;CACrE,IAAI,kBAAkB,KAAK,GACzB,MAAM,IAAI,WACR,qBAAqB,OAAO,KAAK,OAAO,MAAM,sIAChD;CAEF,MAAM,UAAU,oBAAoB,OAAO,KAAA,GAAW,aAAa,MAAM,GAAG,gBAAgB;CAC5F,IAAI;EACF,OAAO;GACL,MAAM;GACN,OAAO,WAAW,OAAO,QAAQ,KAAK,kBAAkB,OAAO,CAAC,CAAC;EACnE;CACF,SAAS,OAAO;EACd,MAAM,IAAI,WACR,GAAI,MAAgB,QAAQ,sDAC9B;CACF;AACF;;;;;;AAOA,SAAgB,YACd,QACA,cAAc,OACC;CACf,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAY;GAC7B,IAAI,WAAW,QAAQ,MAAM,IAAI,MAAM,OAAO,KAAK;GACnD,MAAM,QAAQ,cAAc,KAAK,KAAK,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC,SAAS,CAAC,IAAI;GACnF,IAAI,SAAS,cAAc,KAAK,KAAK,OAAO,KAAK,SAAS,CAAC,IAAI;GAC/D,IAAI,OAAO,OAAO;GAElB,MAAM,MACJ,eAAe,QAAQ,oBAAoB,KAAA,KAAa,SAAS,QAAQ;GAC3E,IAAI,KAAK;IACP,SAAS,QAAQ;IACjB,OAAO,KAAK,MAAM,GAAG,SAAS,CAAC;GACjC;GACA,OAAO;IACL,SAAS,CAAC;KAAE,MAAM;KAAQ;IAAK,CAAC;IAChC,cAAc,MACV;KAAE,SAAS;KAAU,KAAK;IAAa,IACvC;KAAE,SAAS;KAAQ,KAAK;IAAO;IACnC,OAAO;KACL,aAAa;MAAE,OAAO;MAAO,SAAS;MAAO,WAAW;MAAG,YAAY;KAAE;KACzE,cAAc;MAAE,OAAO;MAAQ,MAAM;MAAQ,WAAW;KAAE;IAC5D;IACA,UAAU,CAAC;IACX,UAAU,EAAE,SAAS,gBAAgB;GACvC;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF;;AAGA,SAAS,UAAU,MAAyB,QAA+B;CACzE,IAAI,KAAK,WAAW,KAAA,GAClB,MAAM,IAAI,MAAM,+DAA+D;CAEjF,MAAM,SAAS,OAAO,KAAK,WAAW,aAAa,KAAK,OAAO,MAAM,IAAI,KAAK;CAC9E,OAAO,YAAY,EAAE,MAAM,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM,EAAE,CAAC;AAC3F;AAEA,eAAe,cAAc,OAA8B,QAAwC;CACjG,IAAI,MAAM,MAAM,OAAO,UAAU,MAAM,MAAM,MAAM;CACnD,OAAO,cAAc,KAAK;AAC5B;;AAGA,eAAsB,cAAc,OAAsD;CACxF,IAAI,MAAM,UAAU,WAAW,OAAO,MAAM;CAC5C,MAAM,WAAW,MAAM,KAAK,MAAM,GAAG,MAAM,KAAK,QAAQ,GAAG,CAAC;CAC5D,MAAM,KAAK,MAAM,KAAK,MAAM,SAAS,SAAS,CAAC;CAC/C,QAAQ,UAAR;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,EAAE;EACzD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,EAAE;EACnD,SACE,OAAO,MAAM;CACjB;AACF;AASA,eAAsB,kBACpB,OACA,QACA,QACA,iBACsB;CACtB,MAAM,UAAU,KAAK,IAAI;CACzB,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,cAAc,OAAO,MAAM;EACxC,QAAQ;EACR;EACA,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAgB,MAAM;EAAW,CAAC;EAClE;EACA,YAAY;CACd,CAAC;CACD,OAAO;EACL,QAAQ,OAAO;EACf,eAAe,OAAO,UAAU,WAAW;EAC3C,OAAO;GACL,aAAa,OAAO,OAAO,eAAe;GAC1C,cAAc,OAAO,OAAO,gBAAgB;EAC9C;EACA,WAAW,KAAK,IAAI,IAAI;CAC1B;AACF"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { i as RECOMMENDED_JUDGE_MODEL, l as judgeRoute, n as DIRECT_PROVIDERS } from "./judge-yLvjC5A3.js";
|
|
2
|
+
import { i as debugTrace } from "./attempt-jI0OWxlz.js";
|
|
3
|
+
import { experimental_decide } from "ai";
|
|
4
|
+
//#region src/judge/model.ts
|
|
5
|
+
/** See `judgeRoute`: the provider's own package on direct, the AI Gateway on vercel. */
|
|
6
|
+
async function resolveModel(model, gateway) {
|
|
7
|
+
if (typeof model !== "string") return model;
|
|
8
|
+
const route = judgeRoute(model, process.env, "judge model", gateway);
|
|
9
|
+
if (route.kind === "gateway") return model;
|
|
10
|
+
const id = model.slice(route.provider.length + 1);
|
|
11
|
+
switch (route.provider) {
|
|
12
|
+
case "anthropic": return (await import("@ai-sdk/anthropic")).anthropic.decisionModel(id);
|
|
13
|
+
case "google": return (await import("@ai-sdk/google")).google.decisionModel(id);
|
|
14
|
+
case "openai": return (await import("@ai-sdk/openai")).openai.decisionModel(id);
|
|
15
|
+
default: return (await import("@ai-sdk/typesafe-ai")).typeSafeAi.decisionModel(id);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/** The decision model reeval's `openai/*` judges call on OpenAI's Decisions API. */
|
|
19
|
+
const OPENAI_DECISION_MODEL = "gpt-6-luna";
|
|
20
|
+
/**
|
|
21
|
+
* reeval's sentence when the judge's route answers only decision models and `model` isn't one:
|
|
22
|
+
* Vercel AI Gateway refuses a language model as a judge, and OpenAI's Decisions API answers only
|
|
23
|
+
* its decision model. Undefined for any other error.
|
|
24
|
+
*/
|
|
25
|
+
function judgeModelRefusal(model, gateway, error) {
|
|
26
|
+
const e = error;
|
|
27
|
+
const provider = model.slice(0, model.indexOf("/"));
|
|
28
|
+
if (gateway && /is a language model, not a decision model/.test(e?.message ?? "")) {
|
|
29
|
+
const direct = DIRECT_PROVIDERS[provider];
|
|
30
|
+
const instead = provider === "openai" ? `, or 'openai/${OPENAI_DECISION_MODEL}' pinned to direct() with judge: { gateway: 'direct' } on ${direct?.key}` : direct ? `, or pin this one to direct() with judge: { gateway: 'direct' } to call it on ${direct.key}` : "";
|
|
31
|
+
return `the judge model "${model}" is a language model, and Vercel AI Gateway judges only with decision models. Use a decision model on vercel(), such as '${RECOMMENDED_JUDGE_MODEL}' or 'openai/${OPENAI_DECISION_MODEL}-decisions'${instead}.`;
|
|
32
|
+
}
|
|
33
|
+
const id = model.slice(provider.length + 1);
|
|
34
|
+
if (!gateway && provider === "openai" && id !== OPENAI_DECISION_MODEL && e?.data?.error?.code === "model_not_found") return `the judge model "${model}" isn't a decision model: reeval's openai/* judges call OpenAI's Decisions API, which answers with '${OPENAI_DECISION_MODEL}' (OpenAI said: ${e.message}). Use 'openai/${OPENAI_DECISION_MODEL}'.`;
|
|
35
|
+
}
|
|
36
|
+
async function scoreWithModel(model, question, state, providerOptions, signal, gateway) {
|
|
37
|
+
const resolved = await resolveModel(model, gateway);
|
|
38
|
+
const result = await experimental_decide({
|
|
39
|
+
model: resolved,
|
|
40
|
+
state,
|
|
41
|
+
questions: { answer: {
|
|
42
|
+
type: "boolean",
|
|
43
|
+
instructions: question
|
|
44
|
+
} },
|
|
45
|
+
providerOptions,
|
|
46
|
+
abortSignal: signal,
|
|
47
|
+
maxRetries: 0
|
|
48
|
+
}).catch((error) => {
|
|
49
|
+
const refusal = typeof model === "string" && judgeModelRefusal(model, typeof resolved === "string", error);
|
|
50
|
+
throw refusal ? new Error(refusal, { cause: error }) : error;
|
|
51
|
+
});
|
|
52
|
+
const { answers, usage, warnings, response, providerMetadata } = result;
|
|
53
|
+
debugTrace("judge", "provider-response", {
|
|
54
|
+
answers,
|
|
55
|
+
usage,
|
|
56
|
+
warnings,
|
|
57
|
+
response,
|
|
58
|
+
providerMetadata
|
|
59
|
+
}, "judge provider response");
|
|
60
|
+
return {
|
|
61
|
+
score: result.answers.answer.probability,
|
|
62
|
+
resolvedModel: result.response?.modelId
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
//#endregion
|
|
66
|
+
export { scoreWithModel };
|
|
67
|
+
|
|
68
|
+
//# sourceMappingURL=model-B45lDUV1.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-B45lDUV1.js","names":["decide"],"sources":["../src/judge/model.ts"],"mappings":";;;;;AAQA,eAAe,aACb,OACA,SACwB;CACxB,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,MAAM,QAAQ,WAAW,OAAO,QAAQ,KAAK,eAAe,OAAO;CACnE,IAAI,MAAM,SAAS,WAAW,OAAO;CACrC,MAAM,KAAK,MAAM,MAAM,MAAM,SAAS,SAAS,CAAC;CAChD,QAAQ,MAAM,UAAd;EACE,KAAK,aACH,QAAQ,MAAM,OAAO,qBAAA,CAAsB,UAAU,cAAc,EAAE;EACvE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,KAAK,UACH,QAAQ,MAAM,OAAO,kBAAA,CAAmB,OAAO,cAAc,EAAE;EACjE,SACE,QAAQ,MAAM,OAAO,uBAAA,CAAwB,WAAW,cAAc,EAAE;CAC5E;AACF;;AAGA,MAAM,wBAAwB;;;;;;AAS9B,SAAS,kBAAkB,OAAe,SAAkB,OAAoC;CAC9F,MAAM,IAAI;CACV,MAAM,WAAW,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;CAClD,IAAI,WAAW,4CAA4C,KAAK,GAAG,WAAW,EAAE,GAAG;EACjF,MAAM,SAAS,iBAAiB;EAChC,MAAM,UACJ,aAAa,WACT,gBAAgB,sBAAsB,4DAA4D,QAAQ,QAC1G,SACE,iFAAiF,OAAO,QACxF;EACR,OAAO,oBAAoB,MAAM,4HAA4H,wBAAwB,eAAe,sBAAsB,aAAa,QAAQ;CACjP;CACA,MAAM,KAAK,MAAM,MAAM,SAAS,SAAS,CAAC;CAC1C,IACE,CAAC,WACD,aAAa,YACb,OAAO,yBACP,GAAG,MAAM,OAAO,SAAS,mBAEzB,OAAO,oBAAoB,MAAM,sGAAsG,sBAAsB,kBAAkB,EAAE,QAAQ,iBAAiB,sBAAsB;AAGpO;AAEA,eAAsB,eACpB,OACA,UACA,OACA,iBACA,QACA,SAC+D;CAC/D,MAAM,WAAW,MAAM,aAAa,OAAO,OAAO;CAClD,MAAM,SAAS,MAAMA,oBAAO;EAC1B,OAAO;EACP;EACA,WAAW,EAAE,QAAQ;GAAE,MAAM;GAAW,cAAc;EAAS,EAAE;EACjE;EACA,aAAa;EAEb,YAAY;CACd,CAAC,CAAC,CAAC,OAAO,UAAmB;EAC3B,MAAM,UACJ,OAAO,UAAU,YAAY,kBAAkB,OAAO,OAAO,aAAa,UAAU,KAAK;EAC3F,MAAM,UAAU,IAAI,MAAM,SAAS,EAAE,OAAO,MAAM,CAAC,IAAI;CACzD,CAAC;CACD,MAAM,EAAE,SAAS,OAAO,UAAU,UAAU,qBAAqB;CACjE,WACE,SACA,qBACA;EAAE;EAAS;EAAO;EAAU;EAAU;CAAiB,GACvD,yBACF;CACA,OAAO;EAAE,OAAO,OAAO,QAAQ,OAAO;EAAa,eAAe,OAAO,UAAU;CAAQ;AAC7F"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { D as runOutcome } from "./model-CgnOpss4.js";
|