reeval 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -259
- package/THIRD_PARTY_NOTICES.md +121 -121
- package/assets/package.json +1 -1
- package/assets/pnpm-lock.yaml +41 -30
- package/dist/{adapter-DigeVFWC.js → adapter-BloGVZ_a.js} +20 -8
- package/dist/adapter-BloGVZ_a.js.map +1 -0
- package/dist/{agent-CatUEo5h.js → agent-m-_t-IuW.js} +54 -26
- package/dist/{agent-CatUEo5h.js.map → agent-m-_t-IuW.js.map} +1 -1
- package/dist/{analysis-Bxub9LMT.js → analysis-C2-aEbr7.js} +4 -4
- package/dist/{analysis-Bxub9LMT.js.map → analysis-C2-aEbr7.js.map} +1 -1
- package/dist/app-files-C-jwhRDt.js +11 -0
- package/dist/app-files-C-jwhRDt.js.map +1 -0
- package/dist/{assemble-CF063ms6.js → assemble-BDEpelpi.js} +95 -38
- package/dist/assemble-BDEpelpi.js.map +1 -0
- package/dist/assemble-C5bfncHU.js +1 -0
- package/dist/{attempt-CGPV3BZY.js → attempt-jI0OWxlz.js} +2 -2
- package/dist/{attempt-CGPV3BZY.js.map → attempt-jI0OWxlz.js.map} +1 -1
- package/dist/{capture-BiD7lVaM.js → capture-CVRwRTnb.js} +574 -38
- package/dist/capture-CVRwRTnb.js.map +1 -0
- package/dist/capture-CherbjzB.js +1 -0
- package/dist/{cli-B5zQCR2r.js → cli-C_4gaJ50.js} +145 -142
- package/dist/cli-C_4gaJ50.js.map +1 -0
- package/dist/{cli-spec-BXIBUlaK.js → cli-spec-CpnbATtI.js} +2 -2
- package/dist/{cli-spec-BXIBUlaK.js.map → cli-spec-CpnbATtI.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/{assert-EUIoRdIt.js → client-D5hNjpEx.js} +105 -2
- package/dist/client-D5hNjpEx.js.map +1 -0
- package/dist/{command-C94CQNh8.js → command-BWAuwQNk.js} +12 -12
- package/dist/{command-C94CQNh8.js.map → command-BWAuwQNk.js.map} +1 -1
- package/dist/{command-CaZgCTIw.js → command-BXLUfw4h.js} +14 -14
- package/dist/command-BXLUfw4h.js.map +1 -0
- package/dist/{command-CVtgyqlM.js → command-Be6BSE2L.js} +12 -12
- package/dist/{command-CVtgyqlM.js.map → command-Be6BSE2L.js.map} +1 -1
- package/dist/{command-BMHXHqMT.js → command-Bxdy62dW.js} +15 -14
- package/dist/command-Bxdy62dW.js.map +1 -0
- package/dist/{command-DTgBZR0k.js → command-I1euLLio.js} +35 -15
- package/dist/command-I1euLLio.js.map +1 -0
- package/dist/{commands-CjpGBUWr.js → commands-jzigbNFb.js} +23 -14
- package/dist/commands-jzigbNFb.js.map +1 -0
- package/dist/{commands-BXzIDB3X.js → commands-p1S4nw_H.js} +1 -1
- package/dist/config-BrdHA9bJ.js +3549 -0
- package/dist/config-BrdHA9bJ.js.map +1 -0
- package/dist/{contrasts-CcoMLC_y.js → contrasts-DRxMQQTC.js} +3 -2
- package/dist/contrasts-DRxMQQTC.js.map +1 -0
- package/dist/cost-D4elx933.js +359 -0
- package/dist/cost-D4elx933.js.map +1 -0
- package/dist/cost-DmnAod9x.js +1 -0
- package/dist/db-CAfksinX.js +2 -0
- package/dist/{db-BeqRiWLl.js → db-DO0p7xgP.js} +25 -8
- package/dist/db-DO0p7xgP.js.map +1 -0
- package/dist/{dist-CMjp3qqF.js → dist-CYrBwyHA.js} +2 -2
- package/dist/{dist-CMjp3qqF.js.map → dist-CYrBwyHA.js.map} +1 -1
- package/dist/{docker-BbTlMmYi.js → docker-BnRlTpZA.js} +3 -3
- package/dist/{docker-BbTlMmYi.js.map → docker-BnRlTpZA.js.map} +1 -1
- package/dist/{docker-CCXdbREk.js → docker-D01slDF3.js} +29 -28
- package/dist/docker-D01slDF3.js.map +1 -0
- package/dist/{docker-DC69dKxB.js → docker-DivfZl2u.js} +1 -1
- package/dist/{doctor-CqttrZcs.js → doctor-DQ2p0euG.js} +136 -38
- package/dist/doctor-DQ2p0euG.js.map +1 -0
- package/dist/{environment-BBz1Yes9.js → environment-CO0P0soF.js} +31 -11
- package/dist/environment-CO0P0soF.js.map +1 -0
- package/dist/{evaluate-CXQ1o-pd.js → evaluate-BUmsN7yG.js} +68 -29
- package/dist/evaluate-BUmsN7yG.js.map +1 -0
- package/dist/{evaluate-yjZKUexW.d.ts → evaluate-C_sgRA-V.d.ts} +45 -3
- package/dist/{facts-DtENlf9s.js → facts-DzOUTreB.js} +13 -5
- package/dist/facts-DzOUTreB.js.map +1 -0
- package/dist/{files-B-eUpCZ1.js → files-B-CJeeEh.js} +14 -9
- package/dist/{files-B-eUpCZ1.js.map → files-B-CJeeEh.js.map} +1 -1
- package/dist/files-Bp5j5BqI.js +1 -0
- package/dist/files-C2sjmXhk.js +1 -0
- package/dist/{files-BsQbYz2i.js → files-C_gQNVB-.js} +4 -4
- package/dist/{files-BsQbYz2i.js.map → files-C_gQNVB-.js.map} +1 -1
- package/dist/{files-COHQLEHE.js → files-kUCrswI6.js} +3 -3
- package/dist/{files-COHQLEHE.js.map → files-kUCrswI6.js.map} +1 -1
- package/dist/gateway.d.ts +17 -0
- package/dist/gateway.js +17 -0
- package/dist/gateway.js.map +1 -0
- package/dist/{generate-Cs2TMH_k.js → generate-DZ7vp69F.js} +18 -18
- package/dist/generate-DZ7vp69F.js.map +1 -0
- package/dist/{git-DhykwgKF.js → git-Bk8DSwHy.js} +3 -3
- package/dist/{git-DhykwgKF.js.map → git-Bk8DSwHy.js.map} +1 -1
- package/dist/{harness-BAjw1fcr.js → harness-COVXSwJ6.js} +46 -15
- package/dist/harness-COVXSwJ6.js.map +1 -0
- package/dist/harness.d.ts +26 -18
- package/dist/harness.js +1 -1
- package/dist/{header-kSmWatcO.js → header-BLPP6aCU.js} +2 -2
- package/dist/{header-kSmWatcO.js.map → header-BLPP6aCU.js.map} +1 -1
- package/dist/{images-BmJRuTpm.js → images-DzLeFjLx.js} +8 -8
- package/dist/{images-BmJRuTpm.js.map → images-DzLeFjLx.js.map} +1 -1
- package/dist/index.d.ts +3 -45
- package/dist/index.js +5 -6
- package/dist/init-RgCQXZhb.js +1 -0
- package/dist/{init-Bwc9byQr.js → init-SRalM5xF.js} +6 -8
- package/dist/init-SRalM5xF.js.map +1 -0
- package/dist/{insights-B2Xmhz9o.js → insights-TyAvz9aV.js} +5 -1
- package/dist/{insights-B2Xmhz9o.js.map → insights-TyAvz9aV.js.map} +1 -1
- package/dist/{invocations-CQTbP092.js → invocations-DzyKSJBa.js} +3 -3
- package/dist/{invocations-CQTbP092.js.map → invocations-DzyKSJBa.js.map} +1 -1
- package/dist/judge-yLvjC5A3.js +385 -0
- package/dist/judge-yLvjC5A3.js.map +1 -0
- package/dist/judge.d.ts +7 -6
- package/dist/judge.js +7 -5
- package/dist/judge.js.map +1 -1
- package/dist/match-Bz0nSdzN.js +204 -0
- package/dist/match-Bz0nSdzN.js.map +1 -0
- package/dist/{matchers-Ddmc2z7H.js → matchers-D4eUqx4j.js} +2 -2
- package/dist/{matchers-Ddmc2z7H.js.map → matchers-D4eUqx4j.js.map} +1 -1
- package/dist/{model-B-mel-Gv.js → model-1ZLr9Dcv.js} +59 -35
- package/dist/model-1ZLr9Dcv.js.map +1 -0
- package/dist/model-B45lDUV1.js +68 -0
- package/dist/model-B45lDUV1.js.map +1 -0
- package/dist/model-BTZq8HIZ.js +1 -0
- package/dist/{model-CWlAqmbU.js → model-CgnOpss4.js} +58 -316
- package/dist/model-CgnOpss4.js.map +1 -0
- package/dist/model-Ch8kX87u.js +1 -0
- package/dist/{node-D0mtjRAO.js → node-BSq5qHnQ.js} +12 -5
- package/dist/node-BSq5qHnQ.js.map +1 -0
- package/dist/{opencode-providers-J9colP0-.js → opencode-providers-klkqgzvb.js} +72 -62
- package/dist/opencode-providers-klkqgzvb.js.map +1 -0
- package/dist/{pairs-DP6XOJby.js → pairs-CHfZNRT6.js} +8 -7
- package/dist/pairs-CHfZNRT6.js.map +1 -0
- package/dist/{patch-D4oluWzH.js → patch-DWf-I_Q_.js} +2 -2
- package/dist/{patch-D4oluWzH.js.map → patch-DWf-I_Q_.js.map} +1 -1
- package/dist/{plan-OkwUcJQ9.js → plan-BIwUzlB8.js} +63 -26
- package/dist/plan-BIwUzlB8.js.map +1 -0
- package/dist/{plugins-CUWMnSNb.js → plugins-Dkp_H3pB.js} +5 -5
- package/dist/{plugins-CUWMnSNb.js.map → plugins-Dkp_H3pB.js.map} +1 -1
- package/dist/{preflight-B6pY89bF.js → preflight-D9y-HvZQ.js} +1 -1
- package/dist/{preflight-Bjm7_ylE.js → preflight-DJCD_BDn.js} +3 -3
- package/dist/{preflight-Bjm7_ylE.js.map → preflight-DJCD_BDn.js.map} +1 -1
- package/dist/{prepare-DbdgtZKe.js → prepare-BJG_pD54.js} +2 -2
- package/dist/{prepare-CQmK3SBn.js → prepare-CIxCGG0M.js} +8 -8
- package/dist/{prepare-CQmK3SBn.js.map → prepare-CIxCGG0M.js.map} +1 -1
- package/dist/prices-BcLEXvdY.js +250 -0
- package/dist/prices-BcLEXvdY.js.map +1 -0
- package/dist/{provenance-B3Y17oCC.js → provenance-CfZpFpAh.js} +3 -3
- package/dist/{provenance-B3Y17oCC.js.map → provenance-CfZpFpAh.js.map} +1 -1
- package/dist/{providers-CXN2RFz-.js → providers-D1Zxp2he.js} +1 -1
- package/dist/{providers-eoniqK6w.js → providers-DFk_6W4K.js} +5 -5
- package/dist/{providers-eoniqK6w.js.map → providers-DFk_6W4K.js.map} +1 -1
- package/dist/{query-CLRZmWBT.js → query-CemclveB.js} +2 -2
- package/dist/{query-CLRZmWBT.js.map → query-CemclveB.js.map} +1 -1
- package/dist/query-Cvbwl804.js +1 -0
- package/dist/{questions-1XOulevV.js → questions-bAudIc04.js} +309 -47
- package/dist/questions-bAudIc04.js.map +1 -0
- package/dist/{redact-DJ9JCShu.js → redact-CRGdfV2s.js} +4 -2
- package/dist/redact-CRGdfV2s.js.map +1 -0
- package/dist/{remarks-DlJCqWRs.js → remarks-CXwSnSjR.js} +9 -9
- package/dist/{remarks-DlJCqWRs.js.map → remarks-CXwSnSjR.js.map} +1 -1
- package/dist/remarks-CyAu2DiB.js +1 -0
- package/dist/{remarks-1521zwfW.js → remarks-mujJv1Ey.js} +5 -3
- package/dist/remarks-mujJv1Ey.js.map +1 -0
- package/dist/{remote-cE2VWfkJ.js → remote-CKcEKja4.js} +129 -174
- package/dist/remote-CKcEKja4.js.map +1 -0
- package/dist/resolve-B5-uqmJ6.js +1 -0
- package/dist/{resolve-C4zeccCH.js → resolve-CBoKzQMP.js} +9 -10
- package/dist/resolve-CBoKzQMP.js.map +1 -0
- package/dist/{results-Drtt0uFl.js → results-CMgZEANy.js} +8 -8
- package/dist/{results-Drtt0uFl.js.map → results-CMgZEANy.js.map} +1 -1
- package/dist/{resume-BEFazUfy.js → resume-B9SOSnK9.js} +3 -3
- package/dist/{resume-BEFazUfy.js.map → resume-B9SOSnK9.js.map} +1 -1
- package/dist/{root-DICDHCtp.js → root-Dlff-zp8.js} +5 -5
- package/dist/root-Dlff-zp8.js.map +1 -0
- package/dist/{rules-BFzQqPIm.js → rules-VhftPOOT.js} +53 -4
- package/dist/rules-VhftPOOT.js.map +1 -0
- package/dist/{run-BeMy92Us.js → run-CwIEK16q.js} +12 -12
- package/dist/run-CwIEK16q.js.map +1 -0
- package/dist/{runtime-PX_oXiCS.js → runtime-BgQRyCLl.js} +14 -2
- package/dist/runtime-BgQRyCLl.js.map +1 -0
- package/dist/{sandbox-CQaJj7n9.js → sandbox-BOYZ0hlo.js} +2 -2
- package/dist/{sandbox-CQaJj7n9.js.map → sandbox-BOYZ0hlo.js.map} +1 -1
- package/dist/sandbox.d.ts +2 -1
- package/dist/sandbox.js +1 -1
- package/dist/server-CHRSgKRp.js +1 -0
- package/dist/{server-xK0les4f.js → server-TGW4WyAz.js} +26 -27
- package/dist/{server-xK0les4f.js.map → server-TGW4WyAz.js.map} +1 -1
- package/dist/{shared-CA61BuPJ.js → shared-BDUzStad.js} +2 -2
- package/dist/shared-BDUzStad.js.map +1 -0
- package/dist/shared-CHvO7eLi.d.ts +1 -0
- package/dist/{source-Q1hEKbCs.js → source-B1WvBIQt.js} +2 -2
- package/dist/{source-Q1hEKbCs.js.map → source-B1WvBIQt.js.map} +1 -1
- package/dist/source-DAzztZwU.js +1 -0
- package/dist/src-1jHbFaGW.js +1 -0
- package/dist/{src-LekQrr_S.js → src-8zoX_qrb.js} +614 -171
- package/dist/src-8zoX_qrb.js.map +1 -0
- package/dist/{src-C0CeKiLU.js → src-BYLM3hEz.js} +4 -3
- package/dist/src-BYLM3hEz.js.map +1 -0
- package/dist/{src-CFp86GtV.js → src-CHIwOy0o.js} +87 -55
- package/dist/{src-CFp86GtV.js.map → src-CHIwOy0o.js.map} +1 -1
- package/dist/src-C_9wgowC.js +2 -0
- package/dist/{src-C7_1SqDP.js → src-ChhtI6BP.js} +2 -2
- package/dist/{src-C7_1SqDP.js.map → src-ChhtI6BP.js.map} +1 -1
- package/dist/src-CyaDNG9r.js +2 -0
- package/dist/src-D005Cbqz.js +2 -0
- package/dist/src-DIfRbp5a.js +3 -0
- package/dist/{src-BhmaorOU.js → src-DKPVX2Us.js} +43 -17
- package/dist/{src-BhmaorOU.js.map → src-DKPVX2Us.js.map} +1 -1
- package/dist/src-Dg8121Xy.js +2 -0
- package/dist/{src-cbCb-MPy.js → src-DjF_M8_f.js} +44 -20
- package/dist/{src-cbCb-MPy.js.map → src-DjF_M8_f.js.map} +1 -1
- package/dist/{src-DIL0jST0.js → src-DjRI-IeE.js} +42 -14
- package/dist/src-DjRI-IeE.js.map +1 -0
- package/dist/{src-DiLMkIr9.js → src-DmUgeTlE.js} +47 -17
- package/dist/src-DmUgeTlE.js.map +1 -0
- package/dist/src-ElBqEN-S.js +2 -0
- package/dist/{src-qGoFkuJF.js → src-NSdQ9zI6.js} +88 -23
- package/dist/src-NSdQ9zI6.js.map +1 -0
- package/dist/src-YHGaUrGE.js +2 -0
- package/dist/src-mJVxw1Jm.js +2 -0
- package/dist/{state-BmZ8QVZ8.js → state-B1t143wG.js} +22 -20
- package/dist/{state-BmZ8QVZ8.js.map → state-B1t143wG.js.map} +1 -1
- package/dist/state-vznMh2wl.js +1 -0
- package/dist/stats-DSy5HfOv.js +315 -0
- package/dist/stats-DSy5HfOv.js.map +1 -0
- package/dist/suite-BsWGHzlG.js +2 -0
- package/dist/{suite-C8sw2pox.js → suite-WgT7m6Vs.js} +76 -13
- package/dist/suite-WgT7m6Vs.js.map +1 -0
- package/dist/{summary-0oQNGskv.js → summary-DIGEVr08.js} +58 -17
- package/dist/summary-DIGEVr08.js.map +1 -0
- package/dist/testing.d.ts +174 -2
- package/dist/testing.js +14 -3
- package/dist/testing.js.map +1 -0
- package/dist/types-C5vI3g1C.js.map +1 -1
- package/dist/{types-GraZtfHN.d.ts → types-DXBbP9Mk.d.ts} +302 -72
- package/dist/{usage-CleNKkly.js → usage-BNY1ABrU.js} +3 -3
- package/dist/{usage-CleNKkly.js.map → usage-BNY1ABrU.js.map} +1 -1
- package/dist/usage-DF3_p5pE.js +1 -0
- package/dist/{validate-BdoXUuL_.js → validate-BhwqrVsm.js} +48 -16
- package/dist/validate-BhwqrVsm.js.map +1 -0
- package/dist/view/index.d.ts +143 -91
- package/dist/view/index.js +7 -6
- package/dist/views-BW_C5eVH.js +2 -0
- package/dist/{views-BoUoX4_A.js → views-sXpGE6a2.js} +10 -8
- package/dist/views-sXpGE6a2.js.map +1 -0
- package/dist/{worker-entry-D0-GErmU.js → worker-entry-Duo9oAeT.js} +8 -8
- package/dist/{worker-entry-D0-GErmU.js.map → worker-entry-Duo9oAeT.js.map} +1 -1
- package/dist/{workspace-tjZFkjxC.js → workspace-CDI68wtX.js} +45 -11
- package/dist/workspace-CDI68wtX.js.map +1 -0
- package/dist/workspace-QZCacP8A.js +1 -0
- package/package.json +8 -2
- package/skills/eval-rubric/SKILL.md +9 -0
- package/dist/adapter-DigeVFWC.js.map +0 -1
- package/dist/assemble-CF063ms6.js.map +0 -1
- package/dist/assemble-CJoD_Rzr.js +0 -1
- package/dist/assert-EUIoRdIt.js.map +0 -1
- package/dist/capture-BiD7lVaM.js.map +0 -1
- package/dist/capture-Dp3q7K7Y.js +0 -1
- package/dist/cli-B5zQCR2r.js.map +0 -1
- package/dist/client-fcwgwOAD.js +0 -106
- package/dist/client-fcwgwOAD.js.map +0 -1
- package/dist/command-BMHXHqMT.js.map +0 -1
- package/dist/command-CaZgCTIw.js.map +0 -1
- package/dist/command-DTgBZR0k.js.map +0 -1
- package/dist/commands-CjpGBUWr.js.map +0 -1
- package/dist/config-Dv5MG7Rp.js +0 -753
- package/dist/config-Dv5MG7Rp.js.map +0 -1
- package/dist/contrasts-CcoMLC_y.js.map +0 -1
- package/dist/db-BeqRiWLl.js.map +0 -1
- package/dist/db-CJDRVQal.js +0 -2
- package/dist/docker-CCXdbREk.js.map +0 -1
- package/dist/doctor-CqttrZcs.js.map +0 -1
- package/dist/egress-BfK_O9ce.js +0 -1241
- package/dist/egress-BfK_O9ce.js.map +0 -1
- package/dist/environment-BBz1Yes9.js.map +0 -1
- package/dist/evaluate-CXQ1o-pd.js.map +0 -1
- package/dist/facts-DtENlf9s.js.map +0 -1
- package/dist/files-DqCBW7i8.js +0 -1
- package/dist/files-Ha-jsQXf.js +0 -1
- package/dist/generate-Cs2TMH_k.js.map +0 -1
- package/dist/harness-BAjw1fcr.js.map +0 -1
- package/dist/init-Bwc9byQr.js.map +0 -1
- package/dist/init-D7Y1ny1z.js +0 -1
- package/dist/judge-Dt6KW6_O.js +0 -153
- package/dist/judge-Dt6KW6_O.js.map +0 -1
- package/dist/model-B-mel-Gv.js.map +0 -1
- package/dist/model-BGKYVpoj.js +0 -1
- package/dist/model-C7tSwT7f.js +0 -44
- package/dist/model-C7tSwT7f.js.map +0 -1
- package/dist/model-CWlAqmbU.js.map +0 -1
- package/dist/model-DHY-dfoO.js +0 -1
- package/dist/node-D0mtjRAO.js.map +0 -1
- package/dist/opencode-providers-J9colP0-.js.map +0 -1
- package/dist/options-CPUW97_7.js +0 -1116
- package/dist/options-CPUW97_7.js.map +0 -1
- package/dist/pairs-DP6XOJby.js.map +0 -1
- package/dist/plan-OkwUcJQ9.js.map +0 -1
- package/dist/query-CmBDYt0_.js +0 -1
- package/dist/questions-1XOulevV.js.map +0 -1
- package/dist/redact-DJ9JCShu.js.map +0 -1
- package/dist/remarks-1521zwfW.js.map +0 -1
- package/dist/remarks-BhAV1H4V.js +0 -1
- package/dist/remote-cE2VWfkJ.js.map +0 -1
- package/dist/resolve-C4zeccCH.js.map +0 -1
- package/dist/resolve-D30yZYmB.js +0 -1
- package/dist/root-DICDHCtp.js.map +0 -1
- package/dist/rules-BFzQqPIm.js.map +0 -1
- package/dist/run-BeMy92Us.js.map +0 -1
- package/dist/runtime-PX_oXiCS.js.map +0 -1
- package/dist/server-V4-2w0l8.js +0 -1
- package/dist/shared-CA61BuPJ.js.map +0 -1
- package/dist/source-BDow_oW8.js +0 -1
- package/dist/src-B9L9yGKI.js +0 -1
- package/dist/src-Bcah6ZkT.js +0 -3
- package/dist/src-C0CeKiLU.js.map +0 -1
- package/dist/src-C0xowZ3C.js +0 -2
- package/dist/src-CHIfIKUu.js +0 -2
- package/dist/src-DIL0jST0.js.map +0 -1
- package/dist/src-DLtyz4w5.js +0 -2
- package/dist/src-DPIjmReZ.js +0 -2
- package/dist/src-DXNh5mTr.js +0 -2
- package/dist/src-DiLMkIr9.js.map +0 -1
- package/dist/src-LekQrr_S.js.map +0 -1
- package/dist/src-bCnsA6Vp.js +0 -2
- package/dist/src-qGoFkuJF.js.map +0 -1
- package/dist/src-ynBVosfX.js +0 -2
- package/dist/state-BQR_BCm4.js +0 -1
- package/dist/state-CeR6oz0k.js +0 -66
- package/dist/state-CeR6oz0k.js.map +0 -1
- package/dist/studio-app/assets/index-CnYJijyI.js +0 -58
- package/dist/studio-app/assets/index-CwQN3f_h.css +0 -2
- package/dist/studio-app/favicon.svg +0 -1
- package/dist/studio-app/index.html +0 -15
- package/dist/suite-BaHPmAGy.js +0 -1
- package/dist/suite-C8sw2pox.js.map +0 -1
- package/dist/summary-0oQNGskv.js.map +0 -1
- package/dist/usage-BDo2q_yu.js +0 -1
- package/dist/validate-BdoXUuL_.js.map +0 -1
- package/dist/views-BoUoX4_A.js.map +0 -1
- package/dist/views-noVQNCUP.js +0 -2
- package/dist/workspace-B90YDU_a.js +0 -1
- package/dist/workspace-tjZFkjxC.js.map +0 -1
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
import { a as providerServedModel, i as providerModelResponses, n as providerEndpoints, r as providerHosts, s as resolveOpenCodeProvider } from "./opencode-providers-klkqgzvb.js";
|
|
2
|
+
import { r as badCharacterAt } from "./redact-CRGdfV2s.js";
|
|
3
|
+
import { setTimeout } from "node:timers/promises";
|
|
4
|
+
//#region src/gateway/vercel.ts
|
|
5
|
+
const VERCEL_HOST = "ai-gateway.vercel.sh";
|
|
6
|
+
/** What the adapters take Vercel's key as, whichever host variable it came from. */
|
|
7
|
+
const VERCEL_KEY = "AI_GATEWAY_API_KEY";
|
|
8
|
+
/** A Vercel project's short-lived token (`vercel env pull`), used when there's no key. */
|
|
9
|
+
const VERCEL_OIDC = "VERCEL_OIDC_TOKEN";
|
|
10
|
+
/** The host variable the key comes from: `AI_GATEWAY_API_KEY`, else `VERCEL_OIDC_TOKEN`. */
|
|
11
|
+
const vercelCredential = (env = process.env) => !env["AI_GATEWAY_API_KEY"] && env["VERCEL_OIDC_TOKEN"] ? VERCEL_OIDC : VERCEL_KEY;
|
|
12
|
+
/** When a JWT's `exp` says it expires, or undefined when it isn't one or doesn't say. */
|
|
13
|
+
function jwtExpiry(token) {
|
|
14
|
+
const [, payload] = token.split(".");
|
|
15
|
+
if (!payload) return void 0;
|
|
16
|
+
try {
|
|
17
|
+
const { exp } = JSON.parse(Buffer.from(payload, "base64url").toString("utf8"));
|
|
18
|
+
return typeof exp === "number" ? /* @__PURE__ */ new Date(exp * 1e3) : void 0;
|
|
19
|
+
} catch {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Why the run can't use Vercel's key, or undefined when it can: `VERCEL_OIDC_TOKEN` lasts about
|
|
25
|
+
* 12 hours, and a run that starts on an expired one fails every attempt.
|
|
26
|
+
*/
|
|
27
|
+
function vercelCredentialProblem(env = process.env, now = /* @__PURE__ */ new Date()) {
|
|
28
|
+
if (vercelCredential(env) !== "VERCEL_OIDC_TOKEN") return void 0;
|
|
29
|
+
const expiry = jwtExpiry(env["VERCEL_OIDC_TOKEN"] ?? "");
|
|
30
|
+
if (!expiry || expiry > now) return void 0;
|
|
31
|
+
return `${VERCEL_OIDC} expired at ${expiry.toISOString()}, so vercel() can't authenticate. Run vercel env pull for a new one, or set ${VERCEL_KEY}.`;
|
|
32
|
+
}
|
|
33
|
+
const onVercel = (path, method) => ({
|
|
34
|
+
host: VERCEL_HOST,
|
|
35
|
+
path: { exact: path },
|
|
36
|
+
method: [method]
|
|
37
|
+
});
|
|
38
|
+
/**
|
|
39
|
+
* The response names the model that served it on Vercel's host, in Vercel's form
|
|
40
|
+
* (`openai/gpt-5.6-luna`), with the reasoning effort when it says one.
|
|
41
|
+
*/
|
|
42
|
+
const servedOnVercel = ({ host, responseModel, responseEffort }) => host === "ai-gateway.vercel.sh" && responseModel ? `${responseModel}${responseEffort ? `[effort=${responseEffort}]` : ""}` : void 0;
|
|
43
|
+
/**
|
|
44
|
+
* Claude Code with `ANTHROPIC_BASE_URL` at Vercel's host: its adapter switches to it when given
|
|
45
|
+
* the key as `AI_GATEWAY_API_KEY`, and the CLI sends it in `x-api-key` on Anthropic's message
|
|
46
|
+
* paths. On another base URL it makes none of its `/api/claude_code/*` calls.
|
|
47
|
+
*/
|
|
48
|
+
function claudeCode(env) {
|
|
49
|
+
return {
|
|
50
|
+
credential: vercelCredential(env),
|
|
51
|
+
authAs: VERCEL_KEY,
|
|
52
|
+
endpoints: [onVercel("/v1/messages", "POST"), onVercel("/v1/messages/count_tokens", "POST")],
|
|
53
|
+
hosts: [VERCEL_HOST]
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Codex on a Responses API provider at Vercel, whose ids name the provider (`openai/…`). */
|
|
57
|
+
function codex(env) {
|
|
58
|
+
const endpoints = [onVercel("/v1/responses", "POST")];
|
|
59
|
+
return {
|
|
60
|
+
credential: vercelCredential(env),
|
|
61
|
+
authAs: VERCEL_KEY,
|
|
62
|
+
endpoints,
|
|
63
|
+
modelResponses: endpoints,
|
|
64
|
+
servedModel: servedOnVercel,
|
|
65
|
+
hosts: [VERCEL_HOST],
|
|
66
|
+
modelId: (model) => model.includes("/") ? model : `openai/${model}`
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** fx's language-model calls on Vercel, whose responses name the model that served them. */
|
|
70
|
+
const FX_VERCEL_CALL = {
|
|
71
|
+
host: VERCEL_HOST,
|
|
72
|
+
path: { regex: "^/v\\d+/ai/language-model$" },
|
|
73
|
+
method: ["POST"]
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* The model an fx call asked the gateway for, in `ai-language-model-id`: what reeval records when
|
|
77
|
+
* a provider's stream (Gemini's) names no served model. Only a call that succeeded asked anything.
|
|
78
|
+
*/
|
|
79
|
+
const fxAskedModel = (request) => {
|
|
80
|
+
if (request.host !== "ai-gateway.vercel.sh" || !new RegExp(FX_VERCEL_CALL.path.regex).test(request.path)) return void 0;
|
|
81
|
+
if (request.status === void 0 || request.status < 200 || request.status > 299) return void 0;
|
|
82
|
+
const asked = request.headers?.["ai-language-model-id"];
|
|
83
|
+
return (typeof asked === "string" ? asked : asked?.[0]) || void 0;
|
|
84
|
+
};
|
|
85
|
+
/** The model list fx fetches before a turn, and its language-model calls. */
|
|
86
|
+
const FX_VERCEL_ENDPOINTS = [{
|
|
87
|
+
host: VERCEL_HOST,
|
|
88
|
+
path: { regex: "^/coding-agent/v1/" },
|
|
89
|
+
method: ["GET"]
|
|
90
|
+
}, FX_VERCEL_CALL];
|
|
91
|
+
/** fx on its own AI Gateway provider, which takes a gateway id; a bare one is OpenAI's. */
|
|
92
|
+
function fx(env) {
|
|
93
|
+
return {
|
|
94
|
+
credential: vercelCredential(env),
|
|
95
|
+
authAs: VERCEL_KEY,
|
|
96
|
+
endpoints: FX_VERCEL_ENDPOINTS,
|
|
97
|
+
modelResponses: [FX_VERCEL_CALL],
|
|
98
|
+
servedModel: servedOnVercel,
|
|
99
|
+
askedModel: fxAskedModel,
|
|
100
|
+
hosts: [VERCEL_HOST],
|
|
101
|
+
modelId: (model) => model.includes("/") ? model : `openai/${model}`
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** A Gemini id as Vercel names it (`google/gemini-3-flash`), unless the id names a provider. */
|
|
105
|
+
const googleId = (model) => model.includes("/") ? model : `google/${model}`;
|
|
106
|
+
/**
|
|
107
|
+
* Goose on its `vercel_ai_gateway` provider, an OpenAI-compatible one: Chat Completions, the
|
|
108
|
+
* Responses API for OpenAI's reasoning models, and the model list it fetches each turn.
|
|
109
|
+
*/
|
|
110
|
+
function goose(env) {
|
|
111
|
+
const calls = [onVercel("/v1/chat/completions", "POST"), onVercel("/v1/responses", "POST")];
|
|
112
|
+
return {
|
|
113
|
+
credential: vercelCredential(env),
|
|
114
|
+
authAs: VERCEL_KEY,
|
|
115
|
+
endpoints: [...calls, onVercel("/v1/models", "GET")],
|
|
116
|
+
modelResponses: calls,
|
|
117
|
+
servedModel: servedOnVercel,
|
|
118
|
+
hosts: [VERCEL_HOST],
|
|
119
|
+
modelId: googleId
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* OpenCode on its own `vercel` provider, with the model as `vercel/<id>`: the requests the pinned
|
|
124
|
+
* CLI sends that provider's key on (harness-opencode's provider table), plus Exa's host for web
|
|
125
|
+
* search.
|
|
126
|
+
*/
|
|
127
|
+
function opencode(env) {
|
|
128
|
+
const resolved = resolveOpenCodeProvider("vercel/model");
|
|
129
|
+
if ("error" in resolved) throw new Error(resolved.error);
|
|
130
|
+
const { provider } = resolved;
|
|
131
|
+
return {
|
|
132
|
+
credential: vercelCredential(env),
|
|
133
|
+
authAs: VERCEL_KEY,
|
|
134
|
+
endpoints: providerEndpoints(provider),
|
|
135
|
+
modelResponses: providerModelResponses(provider),
|
|
136
|
+
servedModel: (request) => providerServedModel(provider, request),
|
|
137
|
+
hosts: [...providerHosts(provider), "mcp.exa.ai"],
|
|
138
|
+
modelId: (model) => `vercel/${model}`
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Grok Build with the model as a `[model.<id>]` table on Vercel's Chat Completions API, keyed by
|
|
143
|
+
* `env_key`; it then calls nothing on xAI's API.
|
|
144
|
+
*/
|
|
145
|
+
function grokBuild(env) {
|
|
146
|
+
const endpoints = [onVercel("/v1/chat/completions", "POST")];
|
|
147
|
+
return {
|
|
148
|
+
credential: vercelCredential(env),
|
|
149
|
+
authAs: VERCEL_KEY,
|
|
150
|
+
endpoints,
|
|
151
|
+
modelResponses: endpoints,
|
|
152
|
+
servedModel: servedOnVercel,
|
|
153
|
+
hosts: [VERCEL_HOST],
|
|
154
|
+
expectedBlocked: {
|
|
155
|
+
hosts: ["vercel.com"],
|
|
156
|
+
why: "Grok Build's probe of the gateway's root, which Vercel redirects to its site"
|
|
157
|
+
},
|
|
158
|
+
modelId: (model) => model.includes("/") ? model : `xai/${model}`
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/** The harnesses reeval runs on Vercel, and what each needs there. */
|
|
162
|
+
const VERCEL_ROUTES = {
|
|
163
|
+
"claude-code": claudeCode,
|
|
164
|
+
codex,
|
|
165
|
+
goose,
|
|
166
|
+
opencode,
|
|
167
|
+
fx,
|
|
168
|
+
"grok-build": grokBuild
|
|
169
|
+
};
|
|
170
|
+
/** Why Vercel can't serve an agent reeval doesn't route there, for the plan's messages. */
|
|
171
|
+
const VERCEL_REFUSALS = {
|
|
172
|
+
cursor: "Cursor's agent only calls Cursor's own API",
|
|
173
|
+
antigravity: "Antigravity's CLI only calls Google's API, with a Gemini key",
|
|
174
|
+
pi: "pi reaches a Gemini model on Vercel AI Gateway only through Anthropic's Messages API, with Anthropic's thinking budget for Gemini's own thinking level, and doesn't behave as it does on Gemini's API; reeval runs pi on its Gemini key"
|
|
175
|
+
};
|
|
176
|
+
/**
|
|
177
|
+
* The harnesses on Vercel whose `effort` reaches the model by a path no real run has checked: the
|
|
178
|
+
* plan refuses `effort` with them on `vercel()`. Claude Code's goes as on Anthropic's API, and
|
|
179
|
+
* Codex's response names the effort it ran at.
|
|
180
|
+
*/
|
|
181
|
+
const VERCEL_UNPROVEN_EFFORT = [
|
|
182
|
+
"goose",
|
|
183
|
+
"opencode",
|
|
184
|
+
"grok-build"
|
|
185
|
+
];
|
|
186
|
+
//#endregion
|
|
187
|
+
//#region src/judge/retry.ts
|
|
188
|
+
/** Up to six tries over about a minute: long enough to ride out a provider's "high demand". */
|
|
189
|
+
const JUDGE_RETRY = {
|
|
190
|
+
attempts: 6,
|
|
191
|
+
baseMs: 2e3,
|
|
192
|
+
maxMs: 3e4
|
|
193
|
+
};
|
|
194
|
+
const TRANSIENT_STATUS = /* @__PURE__ */ new Set([
|
|
195
|
+
408,
|
|
196
|
+
409,
|
|
197
|
+
425,
|
|
198
|
+
429
|
|
199
|
+
]);
|
|
200
|
+
const TRANSIENT_MESSAGE = /high demand|overloaded|unavailable|rate.?limit|resource.?exhausted|try again|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed/i;
|
|
201
|
+
/** The AI SDK wraps the provider's error once it has retried itself; look through that. */
|
|
202
|
+
const innermost = (error) => {
|
|
203
|
+
const e = error;
|
|
204
|
+
return e?.name === "AI_RetryError" && e.lastError ? innermost(e.lastError) : error;
|
|
205
|
+
};
|
|
206
|
+
/** An error from the provider's API (an AI SDK `APICallError`), not one reeval raised itself. */
|
|
207
|
+
const isApiError = (error) => typeof innermost(error)?.statusCode === "number";
|
|
208
|
+
/** Rate limits, 5xx, overload and network blips. Never a bad request, bad key, abort or timeout. */
|
|
209
|
+
function isTransient(error) {
|
|
210
|
+
const e = innermost(error);
|
|
211
|
+
if (!e || e.name === "AbortError" || e.name === "TimeoutError") return false;
|
|
212
|
+
if (typeof e.statusCode === "number") return e.statusCode >= 500 || TRANSIENT_STATUS.has(e.statusCode) || e.isRetryable === true;
|
|
213
|
+
return e.isRetryable === true || TRANSIENT_MESSAGE.test(e.message ?? String(e));
|
|
214
|
+
}
|
|
215
|
+
function retryAfterMs(error) {
|
|
216
|
+
const header = innermost(error)?.responseHeaders?.["retry-after"];
|
|
217
|
+
const seconds = Number(header);
|
|
218
|
+
return Number.isFinite(seconds) && seconds > 0 ? seconds * 1e3 : 0;
|
|
219
|
+
}
|
|
220
|
+
/** Exponential backoff with equal jitter, at least any `retry-after`, never above `maxMs`. */
|
|
221
|
+
function backoffMs(policy, retry, error) {
|
|
222
|
+
const cap = Math.min(policy.maxMs, policy.baseMs * 2 ** retry);
|
|
223
|
+
const jittered = cap / 2 + Math.random() * (cap / 2);
|
|
224
|
+
return Math.round(Math.min(policy.maxMs, Math.max(jittered, retryAfterMs(error))));
|
|
225
|
+
}
|
|
226
|
+
/** Runs `call` until it succeeds, fails for good, or runs out of tries. */
|
|
227
|
+
async function withRetries(call, policy, signal) {
|
|
228
|
+
const retries = [];
|
|
229
|
+
for (;;) try {
|
|
230
|
+
return {
|
|
231
|
+
value: await call(),
|
|
232
|
+
retries
|
|
233
|
+
};
|
|
234
|
+
} catch (error) {
|
|
235
|
+
if (signal.aborted || retries.length + 1 >= policy.attempts || !isTransient(error)) {
|
|
236
|
+
const failed = error instanceof Error ? error : new Error(String(error));
|
|
237
|
+
failed.retries = retries;
|
|
238
|
+
throw failed;
|
|
239
|
+
}
|
|
240
|
+
const delayMs = backoffMs(policy, retries.length, error);
|
|
241
|
+
const e = innermost(error);
|
|
242
|
+
retries.push({
|
|
243
|
+
error: e?.message ?? String(e),
|
|
244
|
+
delayMs
|
|
245
|
+
});
|
|
246
|
+
await setTimeout(delayMs, void 0, { signal });
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
//#endregion
|
|
250
|
+
//#region src/judge/index.ts
|
|
251
|
+
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
252
|
+
const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
|
|
253
|
+
/** The language model `reeval init` sets as `analysis.model` and the docs recommend for insights. */
|
|
254
|
+
const RECOMMENDED_ANALYSIS_MODEL = "google/gemini-3.8-flash";
|
|
255
|
+
const DEFAULT_THRESHOLD = .8;
|
|
256
|
+
const GATEWAY_KEYS = [VERCEL_KEY, VERCEL_OIDC];
|
|
257
|
+
/**
|
|
258
|
+
* Providers `direct()` calls, each through its own AI SDK package and key. Any other prefix needs
|
|
259
|
+
* `vercel()` in the config's gateway list.
|
|
260
|
+
*/
|
|
261
|
+
const DIRECT_PROVIDERS = {
|
|
262
|
+
anthropic: {
|
|
263
|
+
key: "ANTHROPIC_API_KEY",
|
|
264
|
+
pkg: "@ai-sdk/anthropic",
|
|
265
|
+
name: "Anthropic"
|
|
266
|
+
},
|
|
267
|
+
google: {
|
|
268
|
+
key: "GOOGLE_GENERATIVE_AI_API_KEY",
|
|
269
|
+
pkg: "@ai-sdk/google",
|
|
270
|
+
name: "Google"
|
|
271
|
+
},
|
|
272
|
+
openai: {
|
|
273
|
+
key: "OPENAI_API_KEY",
|
|
274
|
+
pkg: "@ai-sdk/openai",
|
|
275
|
+
name: "OpenAI"
|
|
276
|
+
},
|
|
277
|
+
"typesafe-ai": {
|
|
278
|
+
key: "TYPESAFE_AI_API_KEY",
|
|
279
|
+
pkg: "@ai-sdk/typesafe-ai",
|
|
280
|
+
name: "TypeSafe AI"
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
const isFakeJudge = (m) => typeof m === "object" && m !== null && m.kind === "fake-judge";
|
|
284
|
+
function judgeModelName(model) {
|
|
285
|
+
if (typeof model === "string") return model;
|
|
286
|
+
if (isFakeJudge(model)) return "fake";
|
|
287
|
+
const m = model;
|
|
288
|
+
return m.provider && m.modelId ? `${m.provider}/${m.modelId}` : "custom";
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Checks a model string's shape; returns an error message naming it as `what` (with where it was
|
|
292
|
+
* set), or undefined when it's valid.
|
|
293
|
+
*/
|
|
294
|
+
function checkModelString(model, what = "The judge model", examples = [RECOMMENDED_JUDGE_MODEL, `google/${model}`]) {
|
|
295
|
+
if (model.includes("/")) return void 0;
|
|
296
|
+
return `${what} needs a provider prefix, e.g. ${examples.map((e) => `"${e}"`).join(" or ")} (got "${model}")`;
|
|
297
|
+
}
|
|
298
|
+
const providerOf = (model) => model.slice(0, model.indexOf("/"));
|
|
299
|
+
/**
|
|
300
|
+
* Environment variables a model needs on `gateway`, in the order reeval prefers them; any one is
|
|
301
|
+
* enough. On direct, its provider's own key (none for a prefix reeval can't call directly); on
|
|
302
|
+
* vercel, the Gateway's.
|
|
303
|
+
*/
|
|
304
|
+
function judgeCredentials(model, gateway = "direct") {
|
|
305
|
+
if (isFakeJudge(model)) return [];
|
|
306
|
+
if (gateway === "vercel") return GATEWAY_KEYS;
|
|
307
|
+
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
308
|
+
return direct ? [direct.key] : [];
|
|
309
|
+
}
|
|
310
|
+
/** The keys that would work for a model on `on`: `ANTHROPIC_API_KEY (for direct())`. */
|
|
311
|
+
function modelKeyOptions(model, on = "direct") {
|
|
312
|
+
if (on === "vercel") return `${GATEWAY_KEYS.join(" or ")} (for vercel())`;
|
|
313
|
+
const direct = DIRECT_PROVIDERS[providerOf(model)];
|
|
314
|
+
return direct ? `${direct.key} (for direct())` : `vercel() in gateway (direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models)`;
|
|
315
|
+
}
|
|
316
|
+
/** The Vercel credential set in `env`, if any. */
|
|
317
|
+
const setGatewayKey = (env) => GATEWAY_KEYS.find((k) => env[k]);
|
|
318
|
+
/**
|
|
319
|
+
* ` AI_GATEWAY_API_KEY is set, but …` when a Vercel credential is set and the model goes direct, so
|
|
320
|
+
* a key that worked before reeval stopped falling back to the Gateway says how to use it.
|
|
321
|
+
*/
|
|
322
|
+
function unusedGatewayKey(env, gateway = "direct") {
|
|
323
|
+
if (gateway !== "direct") return "";
|
|
324
|
+
const set = setGatewayKey(env);
|
|
325
|
+
return set ? ` ${set} is set, but no gateway in the config uses it: add vercel() to gateway to call the model through Vercel AI Gateway.` : "";
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* The provider's own package on direct, or the Vercel AI Gateway on vercel. Throws, naming the
|
|
329
|
+
* model, its prefix and the keys that would work, when none is set, when the key it would send
|
|
330
|
+
* has whitespace or a line break in it, or when Vercel's project token has expired.
|
|
331
|
+
*/
|
|
332
|
+
function judgeRoute(model, env = process.env, what = "judge model", gateway = "direct") {
|
|
333
|
+
const provider = providerOf(model);
|
|
334
|
+
const direct = DIRECT_PROVIDERS[provider];
|
|
335
|
+
const keys = judgeCredentials(model, gateway);
|
|
336
|
+
if (!keys.length) throw new Error(`${what} "${model}" needs vercel() in the config's gateway list: direct() calls only ${Object.keys(DIRECT_PROVIDERS).map((p) => `${p}/*`).join(", ")} models.`);
|
|
337
|
+
const key = keys.find((k) => env[k]);
|
|
338
|
+
if (key === void 0) throw new Error(`No credentials for ${what} "${model}" (provider "${provider}"). Set ${modelKeyOptions(model, gateway)}, in the environment or .env.local.${unusedGatewayKey(env, gateway)}`);
|
|
339
|
+
if (key === "VERCEL_OIDC_TOKEN") {
|
|
340
|
+
const expired = vercelCredentialProblem(env);
|
|
341
|
+
if (expired) throw new Error(`${what} "${model}": ${expired}`);
|
|
342
|
+
}
|
|
343
|
+
const at = badCharacterAt(env[key]);
|
|
344
|
+
if (at) throw new Error(`${key} for ${what} "${model}" has whitespace or a line break in it at character ${at}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
|
|
345
|
+
return direct && key === direct.key ? {
|
|
346
|
+
kind: "direct",
|
|
347
|
+
provider,
|
|
348
|
+
pkg: direct.pkg
|
|
349
|
+
} : { kind: "gateway" };
|
|
350
|
+
}
|
|
351
|
+
/** Transient provider errors (429, 5xx, "high demand") are retried with backoff and recorded. */
|
|
352
|
+
async function runJudge(question, state, options, signal, retry = JUDGE_RETRY) {
|
|
353
|
+
const { model, threshold } = options;
|
|
354
|
+
const gateway = typeof options.gateway === "object" ? options.gateway.kind : options.gateway;
|
|
355
|
+
let score;
|
|
356
|
+
let resolvedModel;
|
|
357
|
+
let retries = [];
|
|
358
|
+
if (isFakeJudge(model)) score = typeof model.score === "function" ? model.score(state) : model.score;
|
|
359
|
+
else {
|
|
360
|
+
if (typeof model === "string") {
|
|
361
|
+
const problem = checkModelString(model);
|
|
362
|
+
if (problem) throw new Error(problem);
|
|
363
|
+
}
|
|
364
|
+
const { scoreWithModel } = await import("./model-B45lDUV1.js");
|
|
365
|
+
const scored = await withRetries(() => scoreWithModel(model, question, state, options.providerOptions, signal, gateway), retry, signal);
|
|
366
|
+
({score, resolvedModel} = scored.value);
|
|
367
|
+
retries = scored.retries;
|
|
368
|
+
}
|
|
369
|
+
return {
|
|
370
|
+
verdict: {
|
|
371
|
+
verdict: score >= threshold ? "pass" : "fail",
|
|
372
|
+
score,
|
|
373
|
+
threshold,
|
|
374
|
+
question,
|
|
375
|
+
model: judgeModelName(model),
|
|
376
|
+
...resolvedModel ? { resolvedModel } : {},
|
|
377
|
+
...gateway && typeof model === "string" ? { gateway } : {}
|
|
378
|
+
},
|
|
379
|
+
retries
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
//#endregion
|
|
383
|
+
export { servedOnVercel as C, fxAskedModel as S, VERCEL_HOST as _, checkModelString as a, VERCEL_ROUTES as b, judgeModelName as c, runJudge as d, setGatewayKey as f, FX_VERCEL_ENDPOINTS as g, FX_VERCEL_CALL as h, RECOMMENDED_JUDGE_MODEL as i, judgeRoute as l, isApiError as m, DIRECT_PROVIDERS as n, isFakeJudge as o, unusedGatewayKey as p, RECOMMENDED_ANALYSIS_MODEL as r, judgeCredentials as s, DEFAULT_THRESHOLD as t, modelKeyOptions as u, VERCEL_KEY as v, vercelCredentialProblem as w, VERCEL_UNPROVEN_EFFORT as x, VERCEL_REFUSALS as y };
|
|
384
|
+
|
|
385
|
+
//# sourceMappingURL=judge-yLvjC5A3.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-yLvjC5A3.js","names":["sleep"],"sources":["../src/gateway/vercel.ts","../src/judge/retry.ts","../src/judge/index.ts"],"mappings":";;;;AAYA,MAAa,cAAc;;AAG3B,MAAa,aAAa;;AAE1B,MAAa,cAAc;;AAa3B,MAAa,oBAAoB,MAAyB,QAAQ,QAChE,CAAC,IAAA,yBAAmB,IAAA,uBAAmB,cAAc;;AAGvD,SAAgB,UAAU,OAAiC;CACzD,MAAM,GAAG,WAAW,MAAM,MAAM,GAAG;CACnC,IAAI,CAAC,SAAS,OAAO,KAAA;CACrB,IAAI;EACF,MAAM,EAAE,QAAQ,KAAK,MAAM,OAAO,KAAK,SAAS,WAAW,CAAC,CAAC,SAAS,MAAM,CAAC;EAG7E,OAAO,OAAO,QAAQ,2BAAW,IAAI,KAAK,MAAM,GAAI,IAAI,KAAA;CAC1D,QAAQ;EACN;CACF;AACF;;;;;AAMA,SAAgB,wBACd,MAAyB,QAAQ,KACjC,sBAAM,IAAI,KAAK,GACK;CACpB,IAAI,iBAAiB,GAAG,MAAA,qBAAmB,OAAO,KAAA;CAClD,MAAM,SAAS,UAAU,IAAA,wBAAoB,EAAE;CAC/C,IAAI,CAAC,UAAU,SAAS,KAAK,OAAO,KAAA;CACpC,OAAO,GAAG,YAAY,cAAc,OAAO,YAAY,EAAE,8EAA8E,WAAW;AACpJ;AAEA,MAAM,YAAY,MAAc,YAC7B;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,KAAK;CAAG,QAAQ,CAAC,MAAM;AAAE;;;;;AAMhE,MAAa,kBAAgD,EAC3D,MACA,eACA,qBAEA,SAAA,0BAAwB,gBACpB,GAAG,gBAAgB,iBAAiB,WAAW,eAAe,KAAK,OACnE,KAAA;;;;;;AAON,SAAS,WAAW,KAAqC;CACvD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,SAAS,gBAAgB,MAAM,GAAG,SAAS,6BAA6B,MAAM,CAAC;EAC3F,OAAO,CAAC,WAAW;CACrB;AACF;;AAGA,SAAS,MAAM,KAAqC;CAClD,MAAM,YAAY,CAAC,SAAS,iBAAiB,MAAM,CAAC;CACpD,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAa,iBAAiB;CAC5B,MAAM;CACN,MAAM,EAAE,OAAO,6BAA6B;CAC5C,QAAQ,CAAC,MAAM;AACjB;;;;;AAMA,MAAa,gBAA8C,YAAY;CACrE,IAAI,QAAQ,SAAA,0BAAwB,CAAC,IAAI,OAAO,eAAe,KAAK,KAAK,CAAC,CAAC,KAAK,QAAQ,IAAI,GAC1F,OAAO,KAAA;CACT,IAAI,QAAQ,WAAW,KAAA,KAAa,QAAQ,SAAS,OAAO,QAAQ,SAAS,KAAK,OAAO,KAAA;CACzF,MAAM,QAAQ,QAAQ,UAAU;CAChC,QAAQ,OAAO,UAAU,WAAW,QAAQ,QAAQ,OAAO,KAAA;AAC7D;;AAGA,MAAa,sBAAsB,CACjC;CAAE,MAAM;CAAa,MAAM,EAAE,OAAO,qBAAqB;CAAG,QAAQ,CAAC,KAAK;AAAE,GAC5E,cACF;;AAGA,SAAS,GAAG,KAAqC;CAC/C,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW;EACX,gBAAgB,CAAC,cAAc;EAC/B,aAAa;EACb,YAAY;EACZ,OAAO,CAAC,WAAW;EACnB,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;CAC/D;AACF;;AAGA,MAAM,YAAY,UAAmB,MAAM,SAAS,GAAG,IAAI,QAAQ,UAAU;;;;;AAM7E,SAAS,MAAM,KAAqC;CAClD,MAAM,QAAQ,CAAC,SAAS,wBAAwB,MAAM,GAAG,SAAS,iBAAiB,MAAM,CAAC;CAC1F,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,CAAC,GAAG,OAAO,SAAS,cAAc,KAAK,CAAC;EACnD,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EACnB,SAAS;CACX;AACF;;;;;;AAOA,SAAS,SAAS,KAAqC;CACrD,MAAM,WAAW,wBAAwB,cAAc;CACvD,IAAI,WAAW,UAAU,MAAM,IAAI,MAAM,SAAS,KAAK;CACvD,MAAM,EAAE,aAAa;CACrB,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR,WAAW,kBAAkB,QAAQ;EACrC,gBAAgB,uBAAuB,QAAQ;EAC/C,cAAc,YAAY,oBAAoB,UAAU,OAAO;EAC/D,OAAO,CAAC,GAAG,cAAc,QAAQ,GAAG,YAAY;EAChD,UAAU,UAAU,UAAU;CAChC;AACF;;;;;AAMA,SAAS,UAAU,KAAqC;CACtD,MAAM,YAAY,CAAC,SAAS,wBAAwB,MAAM,CAAC;CAC3D,OAAO;EACL,YAAY,iBAAiB,GAAG;EAChC,QAAQ;EACR;EACA,gBAAgB;EAChB,aAAa;EACb,OAAO,CAAC,WAAW;EAEnB,iBAAiB;GACf,OAAO,CAAC,YAAY;GACpB,KAAK;EACP;EACA,UAAU,UAAW,MAAM,SAAS,GAAG,IAAI,QAAQ,OAAO;CAC5D;AACF;;AAGA,MAAa,gBAAqF;CAChG,eAAe;CACf;CACA;CACA;CACA;CACA,cAAc;AAChB;;AAGA,MAAa,kBAAsD;CACjE,QAAQ;CACR,aAAa;CACb,IAAI;AACN;;;;;;AAOA,MAAa,yBAA+C;CAAC;CAAS;CAAY;AAAY;;;;AC7N9F,MAAa,cAA2B;CAAE,UAAU;CAAG,QAAQ;CAAO,OAAO;AAAO;AAEpF,MAAM,mCAAmB,IAAI,IAAI;CAAC;CAAK;CAAK;CAAK;AAAG,CAAC;AACrD,MAAM,oBACJ;;AAUF,MAAM,aAAa,UAA4B;CAC7C,MAAM,IAAI;CACV,OAAO,GAAG,SAAS,mBAAmB,EAAE,YAAY,UAAU,EAAE,SAAS,IAAI;AAC/E;;AAGA,MAAa,cAAc,UACzB,OAAQ,UAAU,KAAK,CAAC,EAAoB,eAAe;;AAG7D,SAAgB,YAAY,OAAyB;CACnD,MAAM,IAAI,UAAU,KAAK;CACzB,IAAI,CAAC,KAAK,EAAE,SAAS,gBAAgB,EAAE,SAAS,gBAAgB,OAAO;CACvE,IAAI,OAAO,EAAE,eAAe,UAC1B,OAAO,EAAE,cAAc,OAAO,iBAAiB,IAAI,EAAE,UAAU,KAAK,EAAE,gBAAgB;CAExF,OAAO,EAAE,gBAAgB,QAAQ,kBAAkB,KAAK,EAAE,WAAW,OAAO,CAAC,CAAC;AAChF;AAEA,SAAS,aAAa,OAAwB;CAC5C,MAAM,SAAU,UAAU,KAAK,CAAC,EAAoB,kBAAkB;CACtE,MAAM,UAAU,OAAO,MAAM;CAC7B,OAAO,OAAO,SAAS,OAAO,KAAK,UAAU,IAAI,UAAU,MAAO;AACpE;;AAGA,SAAgB,UAAU,QAAqB,OAAe,OAAwB;CACpF,MAAM,MAAM,KAAK,IAAI,OAAO,OAAO,OAAO,SAAS,KAAK,KAAK;CAC7D,MAAM,WAAW,MAAM,IAAI,KAAK,OAAO,KAAK,MAAM;CAClD,OAAO,KAAK,MAAM,KAAK,IAAI,OAAO,OAAO,KAAK,IAAI,UAAU,aAAa,KAAK,CAAC,CAAC,CAAC;AACnF;;AAKA,eAAsB,YACpB,MACA,QACA,QAC8C;CAC9C,MAAM,UAAwB,CAAC;CAC/B,SACE,IAAI;EACF,OAAO;GAAE,OAAO,MAAM,KAAK;GAAG;EAAQ;CACxC,SAAS,OAAO;EACd,IAAI,OAAO,WAAW,QAAQ,SAAS,KAAK,OAAO,YAAY,CAAC,YAAY,KAAK,GAAG;GAClF,MAAM,SAAU,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GACxE,OAAO,UAAU;GACjB,MAAM;EACR;EACA,MAAM,UAAU,UAAU,QAAQ,QAAQ,QAAQ,KAAK;EACvD,MAAM,IAAI,UAAU,KAAK;EACzB,QAAQ,KAAK;GAAE,OAAO,GAAG,WAAW,OAAO,CAAC;GAAG;EAAQ,CAAC;EACxD,MAAMA,WAAM,SAAS,KAAA,GAAW,EAAE,OAAO,CAAC;CAC5C;AAEJ;;;;AC/DA,MAAa,0BAA0B;;AAEvC,MAAa,6BAA6B;AAC1C,MAAa,oBAAoB;AAEjC,MAAM,eAAe,CAAC,YAAY,WAAW;;;;;AAM7C,MAAa,mBAA+E;CAC1F,WAAW;EAAE,KAAK;EAAqB,KAAK;EAAqB,MAAM;CAAY;CACnF,QAAQ;EAAE,KAAK;EAAgC,KAAK;EAAkB,MAAM;CAAS;CACrF,QAAQ;EAAE,KAAK;EAAkB,KAAK;EAAkB,MAAM;CAAS;CACvE,eAAe;EAAE,KAAK;EAAuB,KAAK;EAAuB,MAAM;CAAc;AAC/F;AAMA,MAAa,eAAe,MAC1B,OAAO,MAAM,YAAY,MAAM,QAAS,EAAqB,SAAS;AAExE,SAAgB,eAAe,OAA2B;CACxD,IAAI,OAAO,UAAU,UAAU,OAAO;CACtC,IAAI,YAAY,KAAK,GAAG,OAAO;CAC/B,MAAM,IAAI;CACV,OAAO,EAAE,YAAY,EAAE,UAAU,GAAG,EAAE,SAAS,GAAG,EAAE,YAAY;AAClE;;;;;AAMA,SAAgB,iBACd,OACA,OAAO,mBACP,WAAW,CAAC,yBAAyB,UAAU,OAAO,GAClC;CACpB,IAAI,MAAM,SAAS,GAAG,GAAG,OAAO,KAAA;CAChC,OAAO,GAAG,KAAK,iCAAiC,SAAS,KAAK,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,MAAM;AAC5G;AAEA,MAAM,cAAc,UAAkB,MAAM,MAAM,GAAG,MAAM,QAAQ,GAAG,CAAC;;;;;;AAOvE,SAAgB,iBACd,OACA,UAAuB,UACb;CACV,IAAI,YAAY,KAAK,GAAG,OAAO,CAAC;CAChC,IAAI,YAAY,UAAU,OAAO;CACjC,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;AAClC;;AAGA,SAAgB,gBAAgB,OAAe,KAAkB,UAAkB;CACjF,IAAI,OAAO,UAAU,OAAO,GAAG,aAAa,KAAK,MAAM,EAAE;CACzD,MAAM,SAAS,iBAAiB,WAAW,KAAK;CAChD,OAAO,SACH,GAAG,OAAO,IAAI,mBACd,4CAA4C,OAAO,KAAK,gBAAgB,CAAC,CACtE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE;AACpB;;AAGA,MAAa,iBAAiB,QAA2B,aAAa,MAAM,MAAM,IAAI,EAAE;;;;;AAMxF,SAAgB,iBAAiB,KAAwB,UAAuB,UAAkB;CAChG,IAAI,YAAY,UAAU,OAAO;CACjC,MAAM,MAAM,cAAc,GAAG;CAC7B,OAAO,MACH,IAAI,IAAI,uHACR;AACN;;;;;;AAOA,SAAgB,WACd,OACA,MAAyB,QAAQ,KACjC,OAAO,eACP,UAAuB,UACX;CACZ,MAAM,WAAW,WAAW,KAAK;CACjC,MAAM,SAAS,iBAAiB;CAChC,MAAM,OAAO,iBAAiB,OAAO,OAAO;CAC5C,IAAI,CAAC,KAAK,QACR,MAAM,IAAI,MACR,GAAG,KAAK,IAAI,MAAM,qEAAqE,OAAO,KAC5F,gBACF,CAAC,CACE,KAAK,MAAM,GAAG,EAAE,GAAG,CAAC,CACpB,KAAK,IAAI,EAAE,SAChB;CAEF,MAAM,MAAM,KAAK,MAAM,MAAM,IAAI,EAAE;CACnC,IAAI,QAAQ,KAAA,GACV,MAAM,IAAI,MACR,sBAAsB,KAAK,IAAI,MAAM,eAAe,SAAS,UAAU,gBAAgB,OAAO,OAAO,EAAE,qCAAqC,iBAAiB,KAAK,OAAO,GAC3K;CAEF,IAAI,QAAA,qBAAqB;EACvB,MAAM,UAAU,wBAAwB,GAAG;EAC3C,IAAI,SAAS,MAAM,IAAI,MAAM,GAAG,KAAK,IAAI,MAAM,KAAK,SAAS;CAC/D;CACA,MAAM,KAAK,eAAe,IAAI,IAAc;CAC5C,IAAI,IACF,MAAM,IAAI,MACR,GAAG,IAAI,OAAO,KAAK,IAAI,MAAM,sDAAsD,GAAG,sFACxF;CAEF,OAAO,UAAU,QAAQ,OAAO,MAC5B;EAAE,MAAM;EAAU;EAAU,KAAK,OAAO;CAAI,IAC5C,EAAE,MAAM,UAAU;AACxB;;AAMA,eAAsB,SACpB,UACA,OACA,SACA,QACA,QAAqB,aACC;CACtB,MAAM,EAAE,OAAO,cAAc;CAC7B,MAAM,UAAU,OAAO,QAAQ,YAAY,WAAW,QAAQ,QAAQ,OAAO,QAAQ;CACrF,IAAI;CACJ,IAAI;CACJ,IAAI,UAAwB,CAAC;CAC7B,IAAI,YAAY,KAAK,GACnB,QAAQ,OAAO,MAAM,UAAU,aAAa,MAAM,MAAM,KAAK,IAAI,MAAM;MAClE;EACL,IAAI,OAAO,UAAU,UAAU;GAC7B,MAAM,UAAU,iBAAiB,KAAK;GACtC,IAAI,SAAS,MAAM,IAAI,MAAM,OAAO;EACtC;EACA,MAAM,EAAE,mBAAmB,MAAM,OAAO;EACxC,MAAM,SAAS,MAAM,kBACb,eAAe,OAAO,UAAU,OAAO,QAAQ,iBAAiB,QAAQ,OAAO,GACrF,OACA,MACF;EACC,CAAC,CAAE,OAAO,iBAAkB,OAAO;EACpC,UAAU,OAAO;CACnB;CAUA,OAAO;EAAE,SAAA;GARP,SAAS,SAAS,YAAY,SAAS;GACvC;GACA;GACA;GACA,OAAO,eAAe,KAAK;GAC3B,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;GACzC,GAAI,WAAW,OAAO,UAAU,WAAW,EAAE,QAAQ,IAAI,CAAC;EAE7C;EAAG;CAAQ;AAC5B"}
|
package/dist/judge.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { $ as FakeJudgeModel, $t as Verdict, J as FakeAnalysisModel, Q as FakeInsightsInput, Y as FakeAnswer, Z as FakeInsights, _t as JudgeCallOptions, bt as JudgeOptions, c as BlindAttempt, gt as JudgeArtifactRef, k as ConfigJudgeOptions, l as BlindPair, n as AnalysisConfig, nt as FakeVote, vt as JudgeFailure, xt as JudgeState, yt as JudgeModel } from "./types-DXBbP9Mk.js";
|
|
2
2
|
//#region src/judge/index.d.ts
|
|
3
3
|
/** The judge the docs recommend, pinned so a model update never silently moves scores. */
|
|
4
4
|
export declare const RECOMMENDED_JUDGE_MODEL = "typesafe-ai/jev-1.14.0";
|
|
@@ -14,14 +14,15 @@ export declare const fakeJudge: ({ score }?: {
|
|
|
14
14
|
}) => FakeJudgeModel;
|
|
15
15
|
/**
|
|
16
16
|
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
17
|
-
* model's structured answer,
|
|
18
|
-
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`),
|
|
19
|
-
*
|
|
17
|
+
* model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
18
|
+
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
|
|
19
|
+
* question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
|
|
20
|
+
* blinded attempt, or `{ error }`), through the same parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
|
|
20
21
|
* get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
|
|
21
22
|
* then once for the headline), through the same validator: an uncited sentence is dropped, and one
|
|
22
23
|
* with a banned word is asked for again, then dropped.
|
|
23
24
|
*/
|
|
24
|
-
export declare const fakeAnalysis: ({ output, vote, insights }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'insights'>) => FakeAnalysisModel;
|
|
25
|
+
export declare const fakeAnalysis: ({ output, vote, answer, insights }: Pick<FakeAnalysisModel, 'output' | 'vote' | 'answer' | 'insights'>) => FakeAnalysisModel;
|
|
25
26
|
//#endregion
|
|
26
|
-
export type { AnalysisConfig, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeInsights, FakeInsightsInput, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
|
|
27
|
+
export type { AnalysisConfig, BlindAttempt, BlindPair, ConfigJudgeOptions, FakeAnalysisModel, FakeAnswer, FakeInsights, FakeInsightsInput, FakeJudgeModel, FakeVote, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeModel, JudgeOptions, JudgeState, Verdict };
|
|
27
28
|
//# sourceMappingURL=judge.d.ts.map
|
package/dist/judge.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { i as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-
|
|
1
|
+
import { i as RECOMMENDED_JUDGE_MODEL, t as DEFAULT_THRESHOLD } from "./judge-yLvjC5A3.js";
|
|
2
2
|
//#region src/judge.ts
|
|
3
3
|
/**
|
|
4
4
|
* A judge that returns `score` without calling a model, for tests and dry runs. `score` can be a
|
|
@@ -10,17 +10,19 @@ const fakeJudge = ({ score = 1 } = {}) => ({
|
|
|
10
10
|
});
|
|
11
11
|
/**
|
|
12
12
|
* An analysis model for tests: `reeval analyze` gets `output` (or `output(digest)`) back as the
|
|
13
|
-
* model's structured answer,
|
|
14
|
-
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`),
|
|
15
|
-
*
|
|
13
|
+
* model's structured answer, each pairwise question's pair gets `vote` (a fixed `'A'`, `'B'` or
|
|
14
|
+
* `'tie'`, `{ winner, reason? }`, a function of the blinded pair, or `{ error }`), and each
|
|
15
|
+
* question asked of one attempt gets `answer` (its text, `{ choice?, text? }`, a function of the
|
|
16
|
+
* blinded attempt, or `{ error }`), through the same parsing and validation as a real one. Insights (in `reeval run` and `reeval analyze --insights`)
|
|
16
17
|
* get `insights` (or `insights(input)`, called per eval for its classification and its sentences,
|
|
17
18
|
* then once for the headline), through the same validator: an uncited sentence is dropped, and one
|
|
18
19
|
* with a banned word is asked for again, then dropped.
|
|
19
20
|
*/
|
|
20
|
-
const fakeAnalysis = ({ output, vote, insights }) => ({
|
|
21
|
+
const fakeAnalysis = ({ output, vote, answer, insights }) => ({
|
|
21
22
|
kind: "fake-analysis",
|
|
22
23
|
...output !== void 0 ? { output } : {},
|
|
23
24
|
...vote !== void 0 ? { vote } : {},
|
|
25
|
+
...answer !== void 0 ? { answer } : {},
|
|
24
26
|
...insights !== void 0 ? { insights } : {}
|
|
25
27
|
});
|
|
26
28
|
//#endregion
|
package/dist/judge.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF
|
|
1
|
+
{"version":3,"file":"judge.js","names":[],"sources":["../src/judge.ts"],"mappings":";;;;;;AAQA,MAAa,aAAa,EACxB,QAAQ,MAGN,CAAC,OAAuB;CAC1B,MAAM;CACN;AACF;;;;;;;;;;;AAYA,MAAa,gBAAgB,EAC3B,QACA,MACA,QACA,gBAC4F;CAC5F,MAAM;CACN,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,SAAS,KAAA,IAAY,EAAE,KAAK,IAAI,CAAC;CACrC,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CACzC,GAAI,aAAa,KAAA,IAAY,EAAE,SAAS,IAAI,CAAC;AAC/C"}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { n as UsageError } from "./errors-B6BSMs03.js";
|
|
2
|
+
import { f as isUnanswerable } from "./runtime-BgQRyCLl.js";
|
|
3
|
+
import { i as assertionRecords } from "./evaluate-BUmsN7yG.js";
|
|
4
|
+
import { i as runMatcher, r as isMatcherName, t as MATCHERS } from "./matchers-D4eUqx4j.js";
|
|
5
|
+
//#region src/attempt/match.ts
|
|
6
|
+
const isPattern = (value) => "regexp" in value && typeof value.regexp === "string" && Object.keys(value).every((k) => k === "regexp" || k === "flags");
|
|
7
|
+
const isAnswered = (value) => Object.keys(value).length === 2 && "matchedBy" in value && typeof value.matchedBy === "string" && "answers" in value && typeof value.answers === "object" && value.answers !== null && !Array.isArray(value.answers);
|
|
8
|
+
/**
|
|
9
|
+
* A pattern the runner matches in its own language, as the `RegExp` a matcher takes: it answers
|
|
10
|
+
* each string from the call's `answers`, and notes the strings it has none for, which it fails.
|
|
11
|
+
*/
|
|
12
|
+
var AnsweredPattern = class extends RegExp {
|
|
13
|
+
static get [Symbol.species]() {
|
|
14
|
+
return RegExp;
|
|
15
|
+
}
|
|
16
|
+
matchedBy;
|
|
17
|
+
answers;
|
|
18
|
+
untested;
|
|
19
|
+
constructor(matchedBy, answers, untested) {
|
|
20
|
+
super("(?:)");
|
|
21
|
+
this.matchedBy = matchedBy;
|
|
22
|
+
this.answers = answers;
|
|
23
|
+
this.untested = untested;
|
|
24
|
+
}
|
|
25
|
+
test(value) {
|
|
26
|
+
const text = String(value);
|
|
27
|
+
if (Object.hasOwn(this.answers, text)) return this.answers[text] === true;
|
|
28
|
+
const strings = this.untested.get(this.matchedBy) ?? /* @__PURE__ */ new Set();
|
|
29
|
+
this.untested.set(this.matchedBy, strings.add(text));
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
exec(value) {
|
|
33
|
+
const text = String(value);
|
|
34
|
+
return this.test(text) ? Object.assign([text], {
|
|
35
|
+
index: 0,
|
|
36
|
+
input: text
|
|
37
|
+
}) : null;
|
|
38
|
+
}
|
|
39
|
+
get source() {
|
|
40
|
+
return this.matchedBy;
|
|
41
|
+
}
|
|
42
|
+
toString() {
|
|
43
|
+
return this.matchedBy;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* A wire argument as `expect` takes it: every `{ regexp, flags? }` in it a `RegExp`, and every
|
|
48
|
+
* `{ matchedBy, answers }` one that notes in `untested` the strings it can't answer.
|
|
49
|
+
*/
|
|
50
|
+
function fromWire(value, untested = /* @__PURE__ */ new Map()) {
|
|
51
|
+
if (Array.isArray(value)) return value.map((v) => fromWire(v, untested));
|
|
52
|
+
if (typeof value !== "object" || value === null) return value;
|
|
53
|
+
if (isAnswered(value)) return new AnsweredPattern(value.matchedBy, value.answers, untested);
|
|
54
|
+
if (isPattern(value)) try {
|
|
55
|
+
return new RegExp(value.regexp, value.flags);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
throw new UsageError(`${JSON.stringify(value)} isn't a regular expression: ${error.message}`);
|
|
58
|
+
}
|
|
59
|
+
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, fromWire(v, untested)]));
|
|
60
|
+
}
|
|
61
|
+
function subjectOf(c, subject) {
|
|
62
|
+
if ("turn" in subject) {
|
|
63
|
+
const turn = c.turns[subject.turn];
|
|
64
|
+
if (!turn) throw new UsageError(`expect's subject is turn ${subject.turn}, but the attempt has ${c.turns.length} turn${c.turns.length === 1 ? "" : "s"}`);
|
|
65
|
+
return turn;
|
|
66
|
+
}
|
|
67
|
+
if ("attempt" in subject) return c;
|
|
68
|
+
if ("value" in subject) return subject.value;
|
|
69
|
+
if ("exec" in subject) return subject.result ?? c.exec(subject.exec);
|
|
70
|
+
return c.readFile(subject.readFile);
|
|
71
|
+
}
|
|
72
|
+
/** What the subject names itself by, as `c.exec`'s and `c.judge`'s results do in TypeScript. */
|
|
73
|
+
function subjectName(subject) {
|
|
74
|
+
if ("exec" in subject) return subject.exec;
|
|
75
|
+
if (!("value" in subject)) return void 0;
|
|
76
|
+
const value = subject.value;
|
|
77
|
+
return typeof value === "object" && value !== null && "verdict" in value ? value.question : void 0;
|
|
78
|
+
}
|
|
79
|
+
async function once(c, name, call, untested) {
|
|
80
|
+
const args = call.args.map((arg) => fromWire(arg, untested));
|
|
81
|
+
let async = false;
|
|
82
|
+
try {
|
|
83
|
+
const actual = await subjectOf(c, call.subject);
|
|
84
|
+
const running = runMatcher(name, actual, args, { not: call.not });
|
|
85
|
+
async = running instanceof Promise;
|
|
86
|
+
const result = await running;
|
|
87
|
+
if (untested.size > 0) return {
|
|
88
|
+
status: "pass",
|
|
89
|
+
async
|
|
90
|
+
};
|
|
91
|
+
if (result.pass !== call.not) return {
|
|
92
|
+
status: "pass",
|
|
93
|
+
async
|
|
94
|
+
};
|
|
95
|
+
const message = result.message();
|
|
96
|
+
const error = new Error(call.label ? `${call.label}: ${message}` : message);
|
|
97
|
+
error.name = "AssertionError";
|
|
98
|
+
return {
|
|
99
|
+
status: "fail",
|
|
100
|
+
error,
|
|
101
|
+
async
|
|
102
|
+
};
|
|
103
|
+
} catch (error) {
|
|
104
|
+
if (!isUnanswerable(error)) throw error;
|
|
105
|
+
return {
|
|
106
|
+
status: "error",
|
|
107
|
+
error,
|
|
108
|
+
async
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const untestedReply = (untested) => ({ untested: Object.fromEntries([...untested].map(([pattern, strings]) => [pattern, [...strings]])) });
|
|
113
|
+
/**
|
|
114
|
+
* Runs one `expect` call on the attempt, as the matcher call it records. A polled call re-reads
|
|
115
|
+
* its subject and retries until the matcher passes or its timeout leaves no room for another
|
|
116
|
+
* try, then answers the last try. A try that tested a runner's pattern on a string it has no
|
|
117
|
+
* answer for ends the call with those strings, and records nothing.
|
|
118
|
+
*/
|
|
119
|
+
async function match(c, call) {
|
|
120
|
+
const name = call.matcher;
|
|
121
|
+
if (!isMatcherName(name)) throw new UsageError(`${name} isn't one of reeval's matchers (${Object.keys(MATCHERS).join(", ")})`);
|
|
122
|
+
const startedAt = performance.now();
|
|
123
|
+
const untested = /* @__PURE__ */ new Map();
|
|
124
|
+
let outcome = await once(c, name, call, untested);
|
|
125
|
+
if (untested.size > 0) return untestedReply(untested);
|
|
126
|
+
if (call.poll) {
|
|
127
|
+
const { interval = 50, timeout = 1e3 } = call.poll;
|
|
128
|
+
const deadline = Date.now() + timeout;
|
|
129
|
+
while (outcome.status !== "pass" && Date.now() + interval < deadline) {
|
|
130
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
131
|
+
outcome = await once(c, name, call, untested);
|
|
132
|
+
if (untested.size > 0) return untestedReply(untested);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const subject = subjectName(call.subject);
|
|
136
|
+
return {
|
|
137
|
+
matcher: [
|
|
138
|
+
call.poll && "poll",
|
|
139
|
+
call.not && "not",
|
|
140
|
+
name
|
|
141
|
+
].filter(Boolean).join("."),
|
|
142
|
+
args: call.args.map((arg) => fromWire(arg)),
|
|
143
|
+
...subject === void 0 ? {} : { subject },
|
|
144
|
+
...call.label ? { label: call.label } : {},
|
|
145
|
+
soft: call.soft,
|
|
146
|
+
status: outcome.status,
|
|
147
|
+
...outcome.status === "pass" ? {} : { error: outcome.error },
|
|
148
|
+
...outcome.async || call.poll ? { durationMs: Math.round(performance.now() - startedAt) } : {}
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Runs the call, adds it to the attempt's matcher calls and replies with its record, or with the
|
|
153
|
+
* strings a runner's pattern must answer first.
|
|
154
|
+
*/
|
|
155
|
+
async function expectCall(c, calls, call) {
|
|
156
|
+
const slot = {
|
|
157
|
+
matcher: [
|
|
158
|
+
call.poll && "poll",
|
|
159
|
+
call.not && "not",
|
|
160
|
+
call.matcher
|
|
161
|
+
].filter(Boolean).join("."),
|
|
162
|
+
args: call.args.map((arg) => fromWire(arg)),
|
|
163
|
+
subject: subjectName(call.subject),
|
|
164
|
+
...call.label ? { label: call.label } : {},
|
|
165
|
+
soft: call.soft,
|
|
166
|
+
status: "pass"
|
|
167
|
+
};
|
|
168
|
+
calls.push(slot);
|
|
169
|
+
let result;
|
|
170
|
+
try {
|
|
171
|
+
result = await match(c, call);
|
|
172
|
+
} catch (error) {
|
|
173
|
+
calls.splice(calls.indexOf(slot), 1);
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
176
|
+
if ("untested" in result) {
|
|
177
|
+
calls.splice(calls.indexOf(slot), 1);
|
|
178
|
+
return result;
|
|
179
|
+
}
|
|
180
|
+
Object.assign(slot, result);
|
|
181
|
+
return { assertion: assertionRecords(calls)[calls.indexOf(slot)] };
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Adds a check the eval's own language made (`record`) to the attempt's matcher calls and replies
|
|
185
|
+
* with its record, named and numbered as a matcher call is.
|
|
186
|
+
*/
|
|
187
|
+
function recordCall(calls, call) {
|
|
188
|
+
const failure = new Error(call.message ?? `${call.label ?? call.subject ?? call.matcher} failed`);
|
|
189
|
+
failure.name = "AssertionError";
|
|
190
|
+
calls.push({
|
|
191
|
+
matcher: call.matcher,
|
|
192
|
+
args: [],
|
|
193
|
+
...call.subject === void 0 ? {} : { subject: call.subject },
|
|
194
|
+
...call.label === void 0 ? {} : { label: call.label },
|
|
195
|
+
soft: call.soft ?? false,
|
|
196
|
+
status: call.status,
|
|
197
|
+
...call.status === "pass" ? {} : { error: failure }
|
|
198
|
+
});
|
|
199
|
+
return assertionRecords(calls).at(-1);
|
|
200
|
+
}
|
|
201
|
+
//#endregion
|
|
202
|
+
export { recordCall as n, expectCall as t };
|
|
203
|
+
|
|
204
|
+
//# sourceMappingURL=match-Bz0nSdzN.js.map
|