promptfoo 0.120.24 → 0.120.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (304) hide show
  1. package/README.md +35 -22
  2. package/dist/src/{ListApp-55OAHww3.js → ListApp-D3DG0F8h.js} +1 -1
  3. package/dist/src/{accounts-BwVGYllP.cjs → accounts-BzEY8H3v.cjs} +5 -5
  4. package/dist/src/{accounts-BOlMHrvn.js → accounts-DHHiXsy6.js} +6 -6
  5. package/dist/src/{accounts-DxR9x_En.js → accounts-R3gfCR_g.js} +5 -5
  6. package/dist/src/{agentic-utils-t9-1raty.js → agentic-utils-D6_gzOUF.js} +3 -3
  7. package/dist/src/{agents-DMpQ860j.cjs → agents-C7BiinFI.cjs} +7 -7
  8. package/dist/src/{agents-9Z7pO8yx.js → agents-Cao4i7AX.js} +8 -8
  9. package/dist/src/{agents-DjExVR3v.js → agents-Cnph5GLD.js} +2 -2
  10. package/dist/src/{agents-CI0Ad69d.cjs → agents-CwM7re15.cjs} +5 -5
  11. package/dist/src/{agents-BwI-Tm5m.js → agents-DETIQHqF.js} +7 -7
  12. package/dist/src/{agents-CRAmwjo-.js → agents-DYIT-hQy.js} +6 -6
  13. package/dist/src/{agents-BbIpDhgp.js → agents-GiUianme.js} +5 -5
  14. package/dist/src/{agents-yL5DzIKY.js → agents-v4cW_ZgC.js} +2 -2
  15. package/dist/src/{aimlapi-xHHqwNg8.js → aimlapi-CMJpKK-B.js} +13 -13
  16. package/dist/src/{aimlapi-BM9P45Bp.js → aimlapi-DMF6v_vb.js} +15 -15
  17. package/dist/src/{aimlapi-DKo7IxJg.cjs → aimlapi-DoGLcQW_.cjs} +14 -14
  18. package/dist/src/{aimlapi-ivzDkqbs.js → aimlapi-DtSf1ykJ.js} +2 -2
  19. package/dist/src/app/assets/index-BOgkICuY.css +1 -0
  20. package/dist/src/app/assets/index-CSgqn_Vd.js +428 -0
  21. package/dist/src/app/assets/{vendor-charts-DnwydkkY.js → vendor-charts-CYyo8R8v.js} +1 -1
  22. package/dist/src/app/assets/{vendor-markdown-DjKAU15k.js → vendor-markdown-DSmzq4Jh.js} +1 -1
  23. package/dist/src/app/assets/vendor-react-AtKqiNEf.js +4 -0
  24. package/dist/src/app/assets/{vendor-syntax-Dc86c4Bu.js → vendor-syntax-D06x6TQF.js} +1 -1
  25. package/dist/src/app/index.html +6 -6
  26. package/dist/src/app/tsconfig.app.tsbuildinfo +1 -1
  27. package/dist/src/{audio-B3QGL6Nl.cjs → audio-BWjyvHn9.cjs} +4 -4
  28. package/dist/src/{audio-C-1oBiVZ.js → audio-BrJBFN2b.js} +4 -4
  29. package/dist/src/{audio-DnfDVjhL.js → audio-DUH4q0Xq.js} +5 -5
  30. package/dist/src/{audio-DQWHfAr8.js → audio-U580w8jM.js} +2 -2
  31. package/dist/src/{base-DHam07ip.js → base-BaXmtXYp.js} +3 -3
  32. package/dist/src/{base-BjsgxJp9.js → base-Dtp8b4_N.js} +3 -3
  33. package/dist/src/{base-Bso8ccGu.cjs → base-f71xxWai.cjs} +3 -3
  34. package/dist/src/{blobs-DOdc-E8L.cjs → blobs-673H0jCl.cjs} +3 -3
  35. package/dist/src/{blobs-Dcm272OH.js → blobs-C9J2mVgo.js} +3 -3
  36. package/dist/src/{blobs-MlVQyoSc.js → blobs-kt8v3UyH.js} +3 -3
  37. package/dist/src/{cache-KNsTJLKP.cjs → cache-7xULbvt3.cjs} +35 -8
  38. package/dist/src/{cache-BMl3AIo5.js → cache-BLLayYEN.js} +24 -9
  39. package/dist/src/{cache-B0ptF7sO.js → cache-BUPcq0Ad.js} +3 -3
  40. package/dist/src/cache-CVfRb-HD.cjs +6 -0
  41. package/dist/src/{cache-DPztwHxt.js → cache-O4EuX2JV.js} +4 -4
  42. package/dist/src/{cache-DHm8Pc8l.js → cache-mIszOnuz.js} +5 -22
  43. package/dist/src/{chat-DOQIJx7O.cjs → chat-C4zqjObh.cjs} +10 -10
  44. package/dist/src/{chat-DDGU7Tik.js → chat-C8Ei4f87.js} +9 -9
  45. package/dist/src/{chat-BnzIuoTQ.js → chat-CgyGj2hC.js} +12 -12
  46. package/dist/src/{chat-BRMJOuCv.js → chat-Cpz3O-Xl.js} +14 -14
  47. package/dist/src/{chat-CVOyOBYk.js → chat-DIywASPG.js} +11 -11
  48. package/dist/src/{chat-CrnXMbvh.cjs → chat-Fl6TZJRS.cjs} +13 -13
  49. package/dist/src/{chat-D8GcWK9l.js → chat-XPN9YHhr.js} +2 -2
  50. package/dist/src/{chatkit-IWIuhxFE.js → chatkit-DEls11hE.js} +4 -4
  51. package/dist/src/{chatkit-DlOZ-eNb.js → chatkit-DIrJX8xk.js} +4 -4
  52. package/dist/src/{chatkit-CcktkleS.js → chatkit-Dpxrq4eD.js} +2 -2
  53. package/dist/src/{chatkit-yeVtURUi.cjs → chatkit-e25Ziu17.cjs} +4 -4
  54. package/dist/src/{claude-agent-sdk-BiMgSH9g.js → claude-agent-sdk-6-xTaLwM.js} +48 -19
  55. package/dist/src/{claude-agent-sdk-Yj9Hq0cH.cjs → claude-agent-sdk-BzxF6NIJ.cjs} +48 -19
  56. package/dist/src/{claude-agent-sdk-rVqlUIJl.js → claude-agent-sdk-CmkTnKGH.js} +49 -20
  57. package/dist/src/{claude-agent-sdk-yid1kGsL.js → claude-agent-sdk-rXCBLK_o.js} +40 -11
  58. package/dist/src/{cloud-ByJVRSQI.js → cloud-BMbRVJFw.js} +3 -3
  59. package/dist/src/cloud-CZ4hytdm.js +5 -0
  60. package/dist/src/{cloudflare-ai-DmJZcvVx.cjs → cloudflare-ai-BAQ0u_dg.cjs} +14 -14
  61. package/dist/src/{cloudflare-ai-CWKYPCCF.js → cloudflare-ai-CUg4BTcj.js} +13 -13
  62. package/dist/src/{cloudflare-ai-goDJ5bwr.js → cloudflare-ai-CobxMTR4.js} +15 -15
  63. package/dist/src/{cloudflare-ai-CdKv38f6.js → cloudflare-ai-Z9X219gp.js} +2 -2
  64. package/dist/src/{cloudflare-gateway-BW8hcdB0.cjs → cloudflare-gateway-C0sgfr_z.cjs} +15 -15
  65. package/dist/src/{cloudflare-gateway-CDCOS1Os.js → cloudflare-gateway-D2_yi-Fh.js} +16 -16
  66. package/dist/src/{cloudflare-gateway-Dz_HCMGY.js → cloudflare-gateway-Djf3F3_H.js} +2 -2
  67. package/dist/src/{cloudflare-gateway-gzmTsXcd.js → cloudflare-gateway-_itGuXry.js} +14 -14
  68. package/dist/src/{codex-sdk-BlvhxMr0.js → codex-sdk-BASDNkIl.js} +90 -34
  69. package/dist/src/{codex-sdk-BC5qbCjh.js → codex-sdk-dSnGdgIp.js} +92 -36
  70. package/dist/src/{codex-sdk-CBpm53B8.js → codex-sdk-ibXwdglL.js} +91 -35
  71. package/dist/src/{codex-sdk-Cf5_CQch.cjs → codex-sdk-wTEpMM_X.cjs} +92 -36
  72. package/dist/src/{cometapi-DoaO5SLq.js → cometapi-B01btbfb.js} +16 -16
  73. package/dist/src/{cometapi-BUKGSUCw.js → cometapi-ChAaRjg5.js} +14 -14
  74. package/dist/src/{cometapi-C6BSw9k3.js → cometapi-DHUAH6nK.js} +2 -2
  75. package/dist/src/{cometapi-B3TzpkNI.cjs → cometapi-JbvOJSCO.cjs} +15 -15
  76. package/dist/src/{completion-BLe7SUA4.js → completion-BBJ6zmG3.js} +6 -6
  77. package/dist/src/{completion-1RAiLcwi.js → completion-D9_MDlnd.js} +6 -6
  78. package/dist/src/{completion-D6BTVt6e.cjs → completion-DDyL3Cb2.cjs} +6 -6
  79. package/dist/src/{docker-CkOzlD1W.js → docker-BuButc4D.js} +15 -15
  80. package/dist/src/{docker-BWsSX8Pn.cjs → docker-Ckw-j7Rr.cjs} +14 -14
  81. package/dist/src/{docker-CMf5SGLq.js → docker-JAAubMw3.js} +13 -13
  82. package/dist/src/{docker-C0AzMsuf.js → docker-vnOg96gi.js} +2 -2
  83. package/dist/src/errors-DnGCbnx8.js +36 -0
  84. package/dist/src/{esm-gWVPXn3r.cjs → esm-BQkx5roy.cjs} +2 -2
  85. package/dist/src/{esm-Lh6czAFT.js → esm-CYhseqj4.js} +2 -2
  86. package/dist/src/{esm-D40XsWlR.js → esm-rDtG_2rg.js} +2 -2
  87. package/dist/src/eval-CKHWqG9f.js +16 -0
  88. package/dist/src/{eval-WaSg64w3.js → eval-CYrbG57o.js} +106 -29
  89. package/dist/src/{evalResult-CGrGU_cG.js → evalResult-6JaUIStC.js} +7 -7
  90. package/dist/src/{evalResult-C_UM63nn.js → evalResult-COsVttMA.js} +7 -7
  91. package/dist/src/evalResult-CxTP-LMm.cjs +11 -0
  92. package/dist/src/evalResult-CzLURDcP.js +13 -0
  93. package/dist/src/{evalResult-BU4huold.cjs → evalResult-DlRfu_Rq.cjs} +7 -7
  94. package/dist/src/evalResult-DyttNQ_G.js +11 -0
  95. package/dist/src/evaluator-0PvfeBYh.js +38 -0
  96. package/dist/src/{evaluator-gmF3Vddp.js → evaluator-3EJCMTs0.js} +75 -49
  97. package/dist/src/{extractor-D-RM2m6A.cjs → extractor-DbhlYEeo.cjs} +23 -6
  98. package/dist/src/{extractor-Da8IxCeS.js → extractor-Hs7la_19.js} +23 -6
  99. package/dist/src/{extractor-BA90EtLq.js → extractor-LSYjrhK0.js} +23 -6
  100. package/dist/src/{fetch-BIfcW-Fd.js → fetch-18MuNu9i.js} +21 -7
  101. package/dist/src/{fetch-CW9GabAi.js → fetch-Bi0o-fdp.js} +2 -2
  102. package/dist/src/fetch-CMptBDVg.cjs +4 -0
  103. package/dist/src/fetch-DAZkv3gV.js +6 -0
  104. package/dist/src/{fetch-Mb7SbeM-.js → fetch-SRsE6Ctl.js} +21 -7
  105. package/dist/src/{fetch-MvWxCS5U.cjs → fetch-ZMn_oemb.cjs} +21 -7
  106. package/dist/src/{fileExtensions-CzcPkiKZ.js → fileExtensions-BpuMmaFL.js} +1 -1
  107. package/dist/src/{formatDuration-CCiZ-wBY.js → formatDuration-Doo0xq-z.js} +1 -1
  108. package/dist/src/{genaiTracer-BQ8AmajD.cjs → genaiTracer-CQlpZkrp.cjs} +2 -2
  109. package/dist/src/{genaiTracer-foKtbVa2.js → genaiTracer-CqNnnXrE.js} +1 -1
  110. package/dist/src/graders-BCytzXrb.js +34 -0
  111. package/dist/src/{graders-CwtelA9k.js → graders-BaMCwIKp.js} +212 -91
  112. package/dist/src/graders-CGZQShfJ.cjs +33 -0
  113. package/dist/src/{graders-2dabIkXs.cjs → graders-DzUUnUjC.cjs} +212 -91
  114. package/dist/src/{graders-BscQ6O1Q.js → graders-QsALpIdy.js} +211 -90
  115. package/dist/src/graders-spkuVC-E.js +36 -0
  116. package/dist/src/{image-Dv0o-Ab7.js → image-BXt_7u0v.js} +4 -4
  117. package/dist/src/{image-B6TV9l0v.js → image-BiEVdpdP.js} +7 -7
  118. package/dist/src/{image-DS9hlgSc.cjs → image-C3BjJUAU.cjs} +7 -7
  119. package/dist/src/{image-_jKUeeh9.js → image-COCWy5dX.js} +2 -2
  120. package/dist/src/{image-DqElR73O.cjs → image-D10zEe1f.cjs} +4 -4
  121. package/dist/src/{image-BVEVjrwl.js → image-DB4sHxdJ.js} +5 -5
  122. package/dist/src/{image-C1i3NoZo.js → image-mhAGP07h.js} +7 -7
  123. package/dist/src/index.cjs +305 -112
  124. package/dist/src/index.d.cts +137 -3
  125. package/dist/src/index.d.ts +137 -3
  126. package/dist/src/index.js +304 -111
  127. package/dist/src/{interactiveCheck-CjK2R4rB.js → interactiveCheck-DU-MAhp5.js} +2 -2
  128. package/dist/src/{knowledgeBase-K5fJN2kC.cjs → knowledgeBase-Bnb00xKs.cjs} +8 -8
  129. package/dist/src/{knowledgeBase-D-xthvn0.js → knowledgeBase-CMvMlLZR.js} +9 -9
  130. package/dist/src/{knowledgeBase-DJZHeJqg.js → knowledgeBase-DotRBzUE.js} +2 -2
  131. package/dist/src/{knowledgeBase-BSs2ldSR.js → knowledgeBase-XJQ0Qyez.js} +8 -8
  132. package/dist/src/{litellm-kPhaZkzz.js → litellm-BECdjOTx.js} +3 -3
  133. package/dist/src/{litellm-uRLukY7s.cjs → litellm-BrnZhMcL.cjs} +15 -15
  134. package/dist/src/{litellm-DH_rVIOd.js → litellm-CHrRmPAe.js} +16 -16
  135. package/dist/src/{litellm-DoXGHlpD.js → litellm-CrLJrPIm.js} +14 -14
  136. package/dist/src/{logger-CMp-NS-e.cjs → logger-BdZ-IqBc.cjs} +5 -2
  137. package/dist/src/{logger-Bzi5o47S.js → logger-BotXmWKW.js} +5 -2
  138. package/dist/src/{logger-q5I8CByj.js → logger-w8Ozp0Td.js} +5 -2
  139. package/dist/src/{luma-ray-B3GiVpuq.js → luma-ray-BOeX-h0M.js} +10 -10
  140. package/dist/src/{luma-ray-CPISsLu-.js → luma-ray-C-w6EsJm.js} +2 -2
  141. package/dist/src/{luma-ray-BrWrKIIB.cjs → luma-ray-C0RkI3lt.cjs} +10 -10
  142. package/dist/src/{luma-ray-gyI_GEy2.js → luma-ray-DgKLS0BF.js} +11 -11
  143. package/dist/src/main.js +3260 -2537
  144. package/dist/src/{messages-CJgEjRQF.js → messages-CDZYGNlS.js} +13 -10
  145. package/dist/src/{messages-BMkwrfmt.js → messages-D61tPFQo.js} +14 -11
  146. package/dist/src/{messages-B5HsO_dM.cjs → messages-DXV3Qh8_.cjs} +14 -11
  147. package/dist/src/{meteor-CBciquOS.js → meteor-Dj8cTkU_.js} +1 -1
  148. package/dist/src/{meteor-0cLf9TIn.js → meteor-P2rUE-Uz.js} +1 -1
  149. package/dist/src/{meteor-BQ6Ws9k2.js → meteor-SLNTgmXm.js} +2 -2
  150. package/dist/src/{meteor-DRuEfzuQ.cjs → meteor-odmwVbyG.cjs} +1 -1
  151. package/dist/src/{nova-reel-B5SOonY2.js → nova-reel-Bk5npr2q.js} +11 -11
  152. package/dist/src/{nova-reel-CT9ZuhJ3.js → nova-reel-C2LFfVTf.js} +2 -2
  153. package/dist/src/{nova-reel-ByTx85ed.cjs → nova-reel-D9FXq3Mt.cjs} +10 -10
  154. package/dist/src/{nova-reel-ChM7WaHR.js → nova-reel-DtCjbD5O.js} +10 -10
  155. package/dist/src/{nova-sonic-D4M3kKYu.js → nova-sonic-BXRfQyF-.js} +8 -8
  156. package/dist/src/{nova-sonic-C-H3eDvL.cjs → nova-sonic-BoRSY_U6.cjs} +7 -7
  157. package/dist/src/{nova-sonic-DgifpOKF.js → nova-sonic-CgaWLDM1.js} +7 -7
  158. package/dist/src/{nova-sonic-BqP59oOu.js → nova-sonic-D_qERM-K.js} +2 -2
  159. package/dist/src/{openai-BjpdxIOG.cjs → openai-Bigwjgo1.cjs} +2 -2
  160. package/dist/src/{openai-Cv9pEKxp.js → openai-CT5fwbve.js} +2 -2
  161. package/dist/src/{openai-CitF-gEN.js → openai-Dz3surb_.js} +2 -2
  162. package/dist/src/openclaw-B6XY2kUf.js +526 -0
  163. package/dist/src/openclaw-CpPrXwf6.js +524 -0
  164. package/dist/src/openclaw-DDSfq5fp.cjs +528 -0
  165. package/dist/src/openclaw-dHLcXUWZ.js +511 -0
  166. package/dist/src/{opencode-sdk-Dakn4QMp.js → opencode-sdk-CImWVqy9.js} +7 -7
  167. package/dist/src/{opencode-sdk-BcLvVMV-.js → opencode-sdk-CuCztr4P.js} +6 -6
  168. package/dist/src/{opencode-sdk-z7KKOCdB.js → opencode-sdk-DhcfRbBH.js} +3 -3
  169. package/dist/src/{opencode-sdk-Cqszt4br.cjs → opencode-sdk-mqF-Oj3f.cjs} +6 -6
  170. package/dist/src/{otlpReceiver-BvmMgacx.cjs → otlpReceiver-B6Xo4KZM.cjs} +6 -6
  171. package/dist/src/{otlpReceiver-DRNetlJH.js → otlpReceiver-BO0rbDzh.js} +6 -6
  172. package/dist/src/{otlpReceiver-Bhj_vnzl.js → otlpReceiver-Dg817agV.js} +6 -6
  173. package/dist/src/{otlpReceiver-Cpnk-Hjf.js → otlpReceiver-DmRb0NBj.js} +2 -2
  174. package/dist/src/{providerRegistry-D32Lt9vp.js → providerRegistry-Xf0qdqGQ.js} +2 -2
  175. package/dist/src/{providerRegistry-HGQd2MF6.cjs → providerRegistry-lc7a7utN.cjs} +2 -2
  176. package/dist/src/{providerRegistry-BdKWcUa8.js → providerRegistry-wCWd7sKQ.js} +2 -2
  177. package/dist/src/providers-BMZZmPBJ.cjs +32 -0
  178. package/dist/src/{providers-BWoVY_Wz.cjs → providers-BNKVY53V.cjs} +294 -782
  179. package/dist/src/{providers-Co_FGgH8.js → providers-BiNq_Iyc.js} +281 -769
  180. package/dist/src/{providers-CUCHJCHD.js → providers-BlEhY5mi.js} +286 -779
  181. package/dist/src/providers-CQQrNaJk.js +32 -0
  182. package/dist/src/providers-Ck8HyrC-.js +34 -0
  183. package/dist/src/{pythonUtils-rOCm9w_5.cjs → pythonUtils-DZ6EbdY4.cjs} +3 -3
  184. package/dist/src/{pythonUtils-B9JA-gsC.js → pythonUtils-r1uBuA0n.js} +3 -3
  185. package/dist/src/{pythonUtils-eNq6Wsfr.js → pythonUtils-vMlk9Qp5.js} +3 -3
  186. package/dist/src/{quiverai-BpWtOEQZ.cjs → quiverai-BNfIwKCO.cjs} +13 -13
  187. package/dist/src/{quiverai-Cj-PUa3p.js → quiverai-BQigKdIH.js} +14 -14
  188. package/dist/src/{quiverai-BN8OVvDE.js → quiverai-Bfy2WnE2.js} +12 -12
  189. package/dist/src/{quiverai-D5MSsd2c.js → quiverai-CedIP0PJ.js} +2 -2
  190. package/dist/src/{render-D2710HbA.js → render-CAZvKKkB.js} +4 -4
  191. package/dist/src/responses-DLLjADw5.js +653 -0
  192. package/dist/src/responses-TsdODUpm.js +654 -0
  193. package/dist/src/responses-zOtKtnY_.cjs +671 -0
  194. package/dist/src/{rubyUtils-Dn6MGcsk.js → rubyUtils-BtjlqyXt.js} +3 -3
  195. package/dist/src/{rubyUtils-LBsk3zIm.js → rubyUtils-Cs35SDYa.js} +3 -3
  196. package/dist/src/rubyUtils-D7--T12C.js +6 -0
  197. package/dist/src/{rubyUtils-rnCVDgH-.cjs → rubyUtils-DCVaJ3mc.cjs} +3 -3
  198. package/dist/src/rubyUtils-DRRiMFV2.js +5 -0
  199. package/dist/src/rubyUtils-vb8OYFC-.cjs +5 -0
  200. package/dist/src/{sagemaker-C8MeZIkH.js → sagemaker-BcgLu0U4.js} +18 -18
  201. package/dist/src/{sagemaker-4ukMNSN0.js → sagemaker-CLdUAv5z.js} +17 -17
  202. package/dist/src/{sagemaker-DyVHy2BW.js → sagemaker-Du4LIR97.js} +2 -2
  203. package/dist/src/{sagemaker-DB3Eojau.cjs → sagemaker-DwNnEVYt.cjs} +17 -17
  204. package/dist/src/{scanner-C28XVEq2.js → scanner-Dyw21Wg_.js} +12 -12
  205. package/dist/src/server/index.js +1818 -605
  206. package/dist/src/server-BUbS0Qfh.js +6 -0
  207. package/dist/src/{server-BF3HkMhe.js → server-CbMTRQkg.js} +8 -6
  208. package/dist/src/{server-CSbLW-UI.cjs → server-CgUQ25qW.cjs} +8 -6
  209. package/dist/src/{server-mycZbUH8.js → server-DWmZLfCy.js} +10 -7
  210. package/dist/src/server-XpGXFHkS.cjs +6 -0
  211. package/dist/src/server-gfOx5Zrk.js +8 -0
  212. package/dist/src/{signal-DM_SPNmi.js → signal-Bl32q42d.js} +3 -3
  213. package/dist/src/{slack-Ed1yyt_j.js → slack-BfdBx2tO.js} +2 -2
  214. package/dist/src/{slack-D1F9Y7CH.cjs → slack-BtMkB6xP.cjs} +2 -2
  215. package/dist/src/{slack-DkAF58Tr.js → slack-DPqj42Ts.js} +2 -2
  216. package/dist/src/{slack-BK312SXM.js → slack-OZYxoVON.js} +2 -2
  217. package/dist/src/{store-CT_e5OZX.js → store-2ocbYY9D.js} +3 -3
  218. package/dist/src/store-5u2yriTV.js +7 -0
  219. package/dist/src/{store-BpumNYCl.cjs → store-BqwfFEyF.cjs} +3 -3
  220. package/dist/src/{store-BFLqwuc_.js → store-D4gdn9ih.js} +3 -3
  221. package/dist/src/store-D_lq_8oQ.js +6 -0
  222. package/dist/src/store-m5KT6Ly7.cjs +6 -0
  223. package/dist/src/{tables-Dansasnu.cjs → tables-B9E1kRp-.cjs} +3 -3
  224. package/dist/src/{tables-BjaApSAB.js → tables-C7TT2XVn.js} +3 -3
  225. package/dist/src/{tables-Dwexr5Z6.js → tables-D-NSwNIb.js} +3 -3
  226. package/dist/src/telemetry-5RHFoCJh.js +6 -0
  227. package/dist/src/{telemetry-MVkZQxt9.js → telemetry-BXyVqyAg.js} +5 -4
  228. package/dist/src/{telemetry-DFDFKdnr.cjs → telemetry-D0_yFdtU.cjs} +5 -4
  229. package/dist/src/{telemetry-Ds5Nn81l.js → telemetry-DZ_7PaVq.js} +5 -4
  230. package/dist/src/telemetry-Do8wMnA-.js +8 -0
  231. package/dist/src/telemetry-LojxPoFq.cjs +6 -0
  232. package/dist/src/{text-PYISqVm1.cjs → text-DF2hMKdg.cjs} +1 -1
  233. package/dist/src/{text-DuYSUYPB.js → text-DgMr_tiM.js} +1 -1
  234. package/dist/src/{text-Dx0GJOCN.js → text-Dm78AVGG.js} +1 -1
  235. package/dist/src/{tokenUsageUtils-BtZd3sP7.cjs → tokenUsageUtils-CXhxVj72.cjs} +9 -4
  236. package/dist/src/{tokenUsageUtils-DoinwgKF.js → tokenUsageUtils-DmZSD2eU.js} +9 -4
  237. package/dist/src/{tokenUsageUtils-cFdLMERB.js → tokenUsageUtils-FZd5O_4A.js} +9 -4
  238. package/dist/src/{transcription-Cp19m_Mt.js → transcription-C-M81iDA.js} +2 -2
  239. package/dist/src/{transcription-CLRpAg07.js → transcription-CYuY5sFO.js} +10 -10
  240. package/dist/src/{transcription-D8ifIKOv.js → transcription-Ch7S-LWw.js} +11 -11
  241. package/dist/src/{transcription-CXXFEVM_.cjs → transcription-FNIz3YOe.cjs} +10 -10
  242. package/dist/src/transform-8eGmaH-7.js +7 -0
  243. package/dist/src/transform-BRVvWaG4.cjs +6 -0
  244. package/dist/src/{transform-Bn-lgBE2.js → transform-CoP2bJ7P.js} +44 -5
  245. package/dist/src/{transform-BFPYuBaW.js → transform-CqTFr7KR.js} +5 -5
  246. package/dist/src/{transform-DvQWeBSR.js → transform-D8dILpfZ.js} +4 -4
  247. package/dist/src/{transform-BUrxadlA.js → transform-DMaxQwDx.js} +44 -5
  248. package/dist/src/transform-GybT0X0u.js +8 -0
  249. package/dist/src/{transform-DshYLyBq.cjs → transform-Kd6u-oNm.cjs} +4 -4
  250. package/dist/src/{transform-Bw1IstDE.cjs → transform-ivxEY4f7.cjs} +55 -4
  251. package/dist/src/{transformersAvailability-BaoWHpu1.cjs → transformersAvailability-Bkep3ka7.cjs} +1 -1
  252. package/dist/src/{transformersAvailability-Dhh45n5P.js → transformersAvailability-DEU2naS1.js} +1 -1
  253. package/dist/src/{transformersAvailability-DtpwoeFC.js → transformersAvailability-DkAWaK5B.js} +1 -1
  254. package/dist/src/{transformersAvailability-O2YaCv9Z.js → transformersAvailability-DwmezkVe.js} +1 -1
  255. package/dist/src/{types-B-XUqfNs.cjs → types-BIfttHrT.cjs} +22 -2
  256. package/dist/src/{types-BsU_PxR3.js → types-DMVjYLpx.js} +16 -2
  257. package/dist/src/{types-CgrxBFgm.js → types-t52w-XsS.js} +19 -3
  258. package/dist/src/{util-M2MoNCUR.js → util-BSh4a_Q8.js} +7 -7
  259. package/dist/src/{util-DTJWKLkl.cjs → util-C08Kns6-.cjs} +18 -4
  260. package/dist/src/{util-DHBpsbZE.js → util-CUEt0Vum.js} +18 -4
  261. package/dist/src/{util-Bnw6EyZN.js → util-Cl0zfT3V.js} +18 -4
  262. package/dist/src/{util-C7A-PlKK.cjs → util-DUYOvxAy.cjs} +6 -6
  263. package/dist/src/{util-WYC3rB_p.js → util-DiCePfDu.js} +6 -6
  264. package/dist/src/{util-CV99ps44.cjs → util-DkFTvieG.cjs} +17 -5
  265. package/dist/src/{util-CfepsNVK.js → util-mJ58qbbw.js} +17 -5
  266. package/dist/src/{util-BZaMVBMq.js → util-vjscpUzy.js} +17 -5
  267. package/dist/src/{utils-BryuD3vq.js → utils-CVzb4YiI.js} +3 -3
  268. package/dist/src/{utils-Dezi3MBH.cjs → utils-DFaZa6Rf.cjs} +3 -3
  269. package/dist/src/{utils-GiyI2K4P.js → utils-JaY9veb5.js} +3 -3
  270. package/dist/tsconfig.tsbuildinfo +1 -1
  271. package/package.json +75 -77
  272. package/dist/src/app/assets/index-07dbAcWK.js +0 -427
  273. package/dist/src/app/assets/index-DnEcCQgk.css +0 -1
  274. package/dist/src/app/assets/vendor-react-DHoaNLma.js +0 -4
  275. package/dist/src/cache-BujOdYlc.cjs +0 -6
  276. package/dist/src/cloud-BHYz0XkH.js +0 -5
  277. package/dist/src/eval-0IFf8hON.js +0 -16
  278. package/dist/src/evalResult-BCCv6FXj.js +0 -13
  279. package/dist/src/evalResult-C8iVgVvr.js +0 -11
  280. package/dist/src/evalResult-CqFeWKYx.cjs +0 -11
  281. package/dist/src/evaluator-DwS5NAEj.js +0 -37
  282. package/dist/src/fetch-BH9KSaUC.cjs +0 -4
  283. package/dist/src/fetch-BL6kqunF.js +0 -6
  284. package/dist/src/graders-BK3LuhJ8.cjs +0 -32
  285. package/dist/src/graders-Bpjtip-E.js +0 -35
  286. package/dist/src/graders-KIf6Uej3.js +0 -33
  287. package/dist/src/providers-41mSodR_.js +0 -31
  288. package/dist/src/providers-CiZCpIxz.cjs +0 -31
  289. package/dist/src/providers-G531909f.js +0 -33
  290. package/dist/src/rubyUtils-BLd6EE1u.cjs +0 -5
  291. package/dist/src/rubyUtils-CVELPvUH.js +0 -6
  292. package/dist/src/rubyUtils-_t9Gmf7U.js +0 -5
  293. package/dist/src/server-Bc4T4XDt.cjs +0 -6
  294. package/dist/src/server-CMyxJ7ct.js +0 -6
  295. package/dist/src/server-D9FuxNYE.js +0 -8
  296. package/dist/src/store-CrMinjmN.cjs +0 -6
  297. package/dist/src/store-EaWZOoxz.js +0 -7
  298. package/dist/src/store-KXWnHB15.js +0 -6
  299. package/dist/src/telemetry-BXD2mCyr.cjs +0 -6
  300. package/dist/src/telemetry-CYiBm56v.js +0 -8
  301. package/dist/src/telemetry-lICYKIMB.js +0 -6
  302. package/dist/src/transform-DZI2t8-9.js +0 -8
  303. package/dist/src/transform-DZUV3qpO.js +0 -7
  304. package/dist/src/transform-XcotjzZX.cjs +0 -6
@@ -0,0 +1,671 @@
1
+ const require_logger = require('./logger-BdZ-IqBc.cjs');
2
+ const require_esm = require('./esm-BQkx5roy.cjs');
3
+ const require_fileExtensions = require('./fileExtensions-DkJYkWUy.cjs');
4
+ const require_util = require('./util-DUYOvxAy.cjs');
5
+ const require_fetch = require('./fetch-ZMn_oemb.cjs');
6
+ const require_cache = require('./cache-7xULbvt3.cjs');
7
+ const require_openai = require('./openai-Bigwjgo1.cjs');
8
+ const require_util$1 = require('./util-C08Kns6-.cjs');
9
+ let path = require("path");
10
+ path = require_logger.__toESM(path);
11
+
12
+ //#region src/providers/functionCallbackUtils.ts
13
+ /**
14
+ * Handles function callback execution for AI providers.
15
+ * Provides a unified way to execute function callbacks across different provider formats.
16
+ */
17
+ var FunctionCallbackHandler = class {
18
+ loadedCallbacks = {};
19
+ mcpToolNames = null;
20
+ constructor(mcpClient) {
21
+ this.mcpClient = mcpClient;
22
+ }
23
+ /**
24
+ * Processes a function call by executing its callback or returning the original call
25
+ * @param call The function call to process (can be various formats)
26
+ * @param callbacks Configuration mapping function names to callbacks
27
+ * @param context Optional context to pass to the callback
28
+ * @returns The result of processing
29
+ */
30
+ async processCall(call, callbacks, context) {
31
+ const functionInfo = this.extractFunctionInfo(call);
32
+ if (this.mcpClient && functionInfo) {
33
+ if (this.mcpToolNames === null) {
34
+ const mcpTools = this.mcpClient.getAllTools();
35
+ this.mcpToolNames = new Set(mcpTools.map((tool) => tool.name));
36
+ }
37
+ if (this.mcpToolNames.has(functionInfo.name)) return await this.executeMcpTool(functionInfo.name, functionInfo.arguments);
38
+ }
39
+ if (!functionInfo || !callbacks || !callbacks[functionInfo.name]) return {
40
+ output: typeof call === "string" ? call : JSON.stringify(call),
41
+ isError: false
42
+ };
43
+ try {
44
+ return {
45
+ output: await this.executeCallback(functionInfo.name, functionInfo.arguments || "{}", callbacks, context),
46
+ isError: false
47
+ };
48
+ } catch (error) {
49
+ require_logger.logger_default.debug(`Function callback failed for ${functionInfo.name}: ${error}`);
50
+ return {
51
+ output: typeof call === "string" ? call : JSON.stringify(call),
52
+ isError: true
53
+ };
54
+ }
55
+ }
56
+ /**
57
+ * Processes multiple function calls
58
+ * @param calls Array of calls or a single call
59
+ * @param callbacks Configuration mapping function names to callbacks
60
+ * @param context Optional context to pass to callbacks
61
+ * @param options Processing options
62
+ * @returns Processed output in appropriate format
63
+ */
64
+ async processCalls(calls, callbacks, context, _options) {
65
+ if (!calls) return calls;
66
+ const isArray = Array.isArray(calls);
67
+ const callsArray = isArray ? calls : [calls];
68
+ const results = await Promise.all(callsArray.map((call) => this.processCall(call, callbacks, context)));
69
+ if (results.some((r, index) => !r.isError && r.output !== JSON.stringify(callsArray[index]))) {
70
+ const outputs = results.map((r) => r.output);
71
+ if (!isArray && outputs.length === 1) return outputs[0];
72
+ return outputs.every((o) => typeof o === "string") ? outputs.join("\n") : outputs;
73
+ }
74
+ if (!isArray && results.length === 1) return results[0].output;
75
+ return calls;
76
+ }
77
+ /**
78
+ * Extracts function name and arguments from various call formats
79
+ */
80
+ extractFunctionInfo(call) {
81
+ if (!call || typeof call !== "object") return null;
82
+ if (call.name && typeof call.name === "string") return {
83
+ name: call.name,
84
+ arguments: call.arguments
85
+ };
86
+ if (call.type === "function" && call.function?.name) return {
87
+ name: call.function.name,
88
+ arguments: call.function.arguments
89
+ };
90
+ return null;
91
+ }
92
+ /**
93
+ * Executes a function callback
94
+ */
95
+ async executeCallback(functionName, args, callbacks, context) {
96
+ let callback = this.loadedCallbacks[functionName];
97
+ if (!callback) {
98
+ const callbackConfig = callbacks[functionName];
99
+ if (typeof callbackConfig === "string") if (callbackConfig.startsWith("file://")) callback = await this.loadExternalFunction(callbackConfig);
100
+ else callback = new Function("return " + callbackConfig)();
101
+ else if (typeof callbackConfig === "function") callback = callbackConfig;
102
+ else throw new Error(`Invalid callback configuration for ${functionName}`);
103
+ this.loadedCallbacks[functionName] = callback;
104
+ }
105
+ const result = await callback(args, context);
106
+ return typeof result === "string" ? result : JSON.stringify(result);
107
+ }
108
+ /**
109
+ * Loads a function from an external file
110
+ */
111
+ async loadExternalFunction(fileRef) {
112
+ let filePath = fileRef.slice(7);
113
+ let functionName;
114
+ if (filePath.includes(":")) {
115
+ const splits = filePath.split(":");
116
+ if (splits[0] && require_fileExtensions.isJavascriptFile(splits[0])) [filePath, functionName] = splits;
117
+ }
118
+ try {
119
+ const resolvedPath = path.default.resolve(require_logger.cliState_default.basePath || "", filePath);
120
+ require_logger.logger_default.debug(`Loading function from ${resolvedPath}${functionName ? `:${functionName}` : ""}`);
121
+ const mod = await require_esm.importModule(resolvedPath);
122
+ const func = functionName && mod[functionName] ? mod[functionName] : mod.default || mod;
123
+ if (typeof func !== "function") throw new Error(`Expected ${resolvedPath}${functionName ? `:${functionName}` : ""} to export a function, got ${typeof func}`);
124
+ return func;
125
+ } catch (error) {
126
+ throw new Error(`Failed to load function from ${fileRef}: ${error}`);
127
+ }
128
+ }
129
+ /**
130
+ * Executes an MCP tool
131
+ */
132
+ async executeMcpTool(toolName, args) {
133
+ try {
134
+ if (!this.mcpClient) throw new Error("MCP client not available");
135
+ const parsedArgs = args == null || args === "" ? {} : typeof args === "string" ? JSON.parse(args) : args;
136
+ const result = await this.mcpClient.callTool(toolName, parsedArgs);
137
+ if (result?.error) return {
138
+ output: `MCP Tool Error (${toolName}): ${result.error}`,
139
+ isError: true
140
+ };
141
+ const normalizeContent = (content) => {
142
+ if (content == null) return "";
143
+ if (typeof content === "string") return content;
144
+ if (Array.isArray(content)) return content.map((part) => {
145
+ if (typeof part === "string") return part;
146
+ if (part && typeof part === "object") {
147
+ if ("text" in part && part.text != null) return String(part.text);
148
+ if ("json" in part) return JSON.stringify(part.json);
149
+ if ("data" in part) return JSON.stringify(part.data);
150
+ return JSON.stringify(part);
151
+ }
152
+ return String(part);
153
+ }).join("\n");
154
+ return JSON.stringify(content);
155
+ };
156
+ return {
157
+ output: `MCP Tool Result (${toolName}): ${normalizeContent(result?.content)}`,
158
+ isError: false
159
+ };
160
+ } catch (error) {
161
+ const errorMessage = error instanceof Error ? error.message : String(error);
162
+ require_logger.logger_default.debug(`MCP tool execution failed for ${toolName}: ${errorMessage}`);
163
+ return {
164
+ output: `MCP Tool Error (${toolName}): ${errorMessage}`,
165
+ isError: true
166
+ };
167
+ }
168
+ }
169
+ /**
170
+ * Sets the MCP client, preserving any loaded callbacks
171
+ */
172
+ setMcpClient(client) {
173
+ this.mcpClient = client;
174
+ this.mcpToolNames = null;
175
+ }
176
+ /**
177
+ * Clears the cached callbacks
178
+ */
179
+ clearCache() {
180
+ this.loadedCallbacks = {};
181
+ }
182
+ };
183
+
184
+ //#endregion
185
+ //#region src/providers/responses/processor.ts
186
+ /**
187
+ * Extract user-facing metadata from response data.
188
+ * Only includes fields that are useful for users viewing eval results.
189
+ */
190
+ function extractMetadata(data, processedOutput) {
191
+ const metadata = {};
192
+ if (typeof data.id === "string" && data.id) metadata.responseId = data.id;
193
+ if (typeof data.model === "string" && data.model) metadata.model = data.model;
194
+ if (Array.isArray(processedOutput.annotations) && processedOutput.annotations.length > 0) metadata.annotations = processedOutput.annotations;
195
+ return metadata;
196
+ }
197
+ /**
198
+ * Extract token usage from response data, handling both OpenAI Chat Completions format
199
+ * (prompt_tokens, completion_tokens) and Azure Responses format (input_tokens, output_tokens)
200
+ */
201
+ function getTokenUsage(data, cached) {
202
+ if (data.usage) if (cached) {
203
+ const totalTokens = data.usage.total_tokens || (data.usage.input_tokens || 0) + (data.usage.output_tokens || 0);
204
+ return {
205
+ cached: totalTokens,
206
+ total: totalTokens,
207
+ numRequests: 1
208
+ };
209
+ } else {
210
+ const promptTokens = data.usage.prompt_tokens || data.usage.input_tokens || 0;
211
+ const completionTokens = data.usage.completion_tokens || data.usage.output_tokens || 0;
212
+ return {
213
+ total: data.usage.total_tokens || promptTokens + completionTokens,
214
+ prompt: promptTokens,
215
+ completion: completionTokens,
216
+ numRequests: 1,
217
+ ...data.usage.completion_tokens_details ? { completionDetails: {
218
+ reasoning: data.usage.completion_tokens_details.reasoning_tokens,
219
+ acceptedPrediction: data.usage.completion_tokens_details.accepted_prediction_tokens,
220
+ rejectedPrediction: data.usage.completion_tokens_details.rejected_prediction_tokens
221
+ } } : {}
222
+ };
223
+ }
224
+ return {};
225
+ }
226
+ /**
227
+ * Shared response processor for OpenAI and Azure Responses APIs.
228
+ * Handles all response types with identical logic to ensure feature parity.
229
+ */
230
+ var ResponsesProcessor = class {
231
+ constructor(config) {
232
+ this.config = config;
233
+ }
234
+ async processResponseOutput(data, requestConfig, cached) {
235
+ require_logger.logger_default.debug(`Processing ${this.config.providerType} responses output`, {
236
+ responseId: data.id,
237
+ model: data.model
238
+ });
239
+ if (data.error) return { error: require_util$1.formatOpenAiError(data) };
240
+ try {
241
+ const context = {
242
+ config: requestConfig,
243
+ cached,
244
+ data
245
+ };
246
+ const processedOutput = await this.processOutput(data.output, context);
247
+ if (processedOutput.isRefusal) return {
248
+ output: processedOutput.refusal,
249
+ tokenUsage: getTokenUsage(data, cached),
250
+ isRefusal: true,
251
+ cached,
252
+ cost: this.config.costCalculator(this.config.modelName, data.usage, requestConfig),
253
+ raw: data,
254
+ metadata: extractMetadata(data, processedOutput)
255
+ };
256
+ let finalOutput = processedOutput.result;
257
+ if (requestConfig.response_format?.type === "json_schema" && typeof finalOutput === "string") try {
258
+ finalOutput = JSON.parse(finalOutput);
259
+ } catch (error) {
260
+ require_logger.logger_default.error(`Failed to parse JSON output: ${error}`);
261
+ }
262
+ const result = {
263
+ output: finalOutput,
264
+ tokenUsage: getTokenUsage(data, cached),
265
+ cached,
266
+ cost: this.config.costCalculator(this.config.modelName, data.usage, requestConfig),
267
+ raw: data,
268
+ metadata: extractMetadata(data, processedOutput)
269
+ };
270
+ if (processedOutput.annotations && processedOutput.annotations.length > 0) result.raw = {
271
+ ...data,
272
+ annotations: processedOutput.annotations
273
+ };
274
+ return result;
275
+ } catch (err) {
276
+ return { error: `Error parsing response: ${String(err)}\nResponse: ${JSON.stringify(data)}` };
277
+ }
278
+ }
279
+ async processOutput(output, context) {
280
+ if (this.config.modelName.includes("deep-research")) require_logger.logger_default.debug(`Deep research response structure: ${JSON.stringify(context.data, null, 2)}`);
281
+ if (!output || !Array.isArray(output) || output.length === 0) throw new Error("Invalid response format: Missing output array");
282
+ let result = "";
283
+ let refusal = "";
284
+ let isRefusal = false;
285
+ const annotations = [];
286
+ for (const item of output) {
287
+ if (!item || typeof item !== "object") {
288
+ require_logger.logger_default.warn(`Skipping invalid output item: ${JSON.stringify(item)}`);
289
+ continue;
290
+ }
291
+ const processed = await this.processOutputItem(item, context);
292
+ if (processed.isRefusal) {
293
+ refusal = processed.content || "";
294
+ isRefusal = true;
295
+ } else if (processed.content) if (result) result += "\n" + processed.content;
296
+ else result = processed.content;
297
+ if (processed.annotations) annotations.push(...processed.annotations);
298
+ }
299
+ return {
300
+ result,
301
+ refusal,
302
+ isRefusal,
303
+ annotations: annotations.length > 0 ? annotations : void 0
304
+ };
305
+ }
306
+ async processOutputItem(item, context) {
307
+ switch (item.type) {
308
+ case "function_call": return await this.processFunctionCall(item, context);
309
+ case "message": return await this.processMessage(item, context);
310
+ case "tool_result": return this.processToolResult(item);
311
+ case "reasoning": return this.processReasoning(item);
312
+ case "web_search_call": return this.processWebSearch(item);
313
+ case "code_interpreter_call": return this.processCodeInterpreter(item);
314
+ case "mcp_list_tools": return this.processMcpListTools(item);
315
+ case "mcp_call": return this.processMcpCall(item);
316
+ case "mcp_approval_request": return this.processMcpApprovalRequest(item);
317
+ default:
318
+ require_logger.logger_default.debug(`Unknown output item type: ${item.type}`);
319
+ return {};
320
+ }
321
+ }
322
+ async processFunctionCall(item, context) {
323
+ let functionResult;
324
+ if (item.arguments === "{}" && item.status === "completed") functionResult = JSON.stringify({
325
+ type: "function_call",
326
+ name: item.name,
327
+ status: "no_arguments_provided",
328
+ note: "Function called but no arguments were extracted. Consider using the correct Responses API tool format."
329
+ });
330
+ else functionResult = await this.config.functionCallbackHandler.processCalls(item, context.config.functionToolCallbacks);
331
+ return { content: functionResult };
332
+ }
333
+ async processMessage(item, context) {
334
+ if (item.role !== "assistant") return {};
335
+ let content = "";
336
+ let isRefusal = false;
337
+ let refusal = "";
338
+ const annotations = [];
339
+ if (item.content) for (const contentItem of item.content) {
340
+ if (!contentItem || typeof contentItem !== "object") {
341
+ require_logger.logger_default.warn(`Skipping invalid content item: ${JSON.stringify(contentItem)}`);
342
+ continue;
343
+ }
344
+ if (contentItem.type === "output_text") {
345
+ content += contentItem.text;
346
+ if (Array.isArray(contentItem.annotations) && contentItem.annotations.length > 0) annotations.push(...contentItem.annotations);
347
+ } else if (contentItem.type === "tool_use" || contentItem.type === "function_call") content = await this.config.functionCallbackHandler.processCalls(contentItem, context.config.functionToolCallbacks);
348
+ else if (contentItem.type === "refusal") {
349
+ refusal = contentItem.refusal;
350
+ isRefusal = true;
351
+ }
352
+ }
353
+ else if (item.refusal) {
354
+ refusal = item.refusal;
355
+ isRefusal = true;
356
+ }
357
+ return {
358
+ content: isRefusal ? refusal : content,
359
+ isRefusal,
360
+ annotations: annotations.length > 0 ? annotations : void 0
361
+ };
362
+ }
363
+ processToolResult(item) {
364
+ return Promise.resolve({ content: JSON.stringify(item) });
365
+ }
366
+ processReasoning(item) {
367
+ if (!item.summary || !item.summary.length) return Promise.resolve({});
368
+ const reasoningText = `Reasoning: ${item.summary.map((s) => s.text).join("\n")}`;
369
+ return Promise.resolve({ content: reasoningText });
370
+ }
371
+ processWebSearch(item) {
372
+ let content = "";
373
+ const action = item.action;
374
+ if (action) if (action.type === "search") content = `Web Search: "${action.query}"`;
375
+ else if (action.type === "open_page") content = `Opening page: ${action.url}`;
376
+ else if (action.type === "find_in_page") content = `Finding in page: "${action.query}"`;
377
+ else content = `Web action: ${action.type}`;
378
+ else content = `Web Search Call (status: ${item.status || "unknown"})`;
379
+ if (item.status === "failed" && item.error) content += ` (Error: ${item.error})`;
380
+ return Promise.resolve({ content });
381
+ }
382
+ processCodeInterpreter(item) {
383
+ let content = `Code Interpreter: ${item.code || "Running code..."}`;
384
+ if (item.status === "failed" && item.error) content += ` (Error: ${item.error})`;
385
+ return Promise.resolve({ content });
386
+ }
387
+ processMcpListTools(item) {
388
+ const content = `MCP Tools from ${item.server_label}: ${JSON.stringify(item.tools, null, 2)}`;
389
+ return Promise.resolve({ content });
390
+ }
391
+ processMcpCall(item) {
392
+ let content;
393
+ if (item.error) content = `MCP Tool Error (${item.name}): ${item.error}`;
394
+ else content = `MCP Tool Result (${item.name}): ${item.output}`;
395
+ return Promise.resolve({ content });
396
+ }
397
+ processMcpApprovalRequest(item) {
398
+ const content = `MCP Approval Required for ${item.server_label}.${item.name}: ${item.arguments}`;
399
+ return Promise.resolve({ content });
400
+ }
401
+ };
402
+
403
+ //#endregion
404
+ //#region src/providers/openai/responses.ts
405
+ var OpenAiResponsesProvider = class extends require_openai.OpenAiGenericProvider {
406
+ functionCallbackHandler = new FunctionCallbackHandler();
407
+ processor;
408
+ static OPENAI_RESPONSES_MODEL_NAMES = [
409
+ "gpt-4o",
410
+ "gpt-4o-2024-08-06",
411
+ "gpt-4o-2024-11-20",
412
+ "gpt-4o-2024-05-13",
413
+ "gpt-4o-2024-07-18",
414
+ "gpt-4o-mini",
415
+ "gpt-4o-mini-2024-07-18",
416
+ "gpt-4.1",
417
+ "gpt-4.1-2025-04-14",
418
+ "gpt-4.1-mini",
419
+ "gpt-4.1-mini-2025-04-14",
420
+ "gpt-4.1-nano",
421
+ "gpt-4.1-nano-2025-04-14",
422
+ "gpt-5",
423
+ "gpt-5-2025-08-07",
424
+ "gpt-5-chat",
425
+ "gpt-5-chat-latest",
426
+ "gpt-5-nano",
427
+ "gpt-5-nano-2025-08-07",
428
+ "gpt-5-mini",
429
+ "gpt-5-mini-2025-08-07",
430
+ "gpt-5-pro",
431
+ "gpt-5-pro-2025-10-06",
432
+ "gpt-5.1",
433
+ "gpt-5.1-2025-11-13",
434
+ "gpt-5.1-mini",
435
+ "gpt-5.1-nano",
436
+ "gpt-5.1-codex",
437
+ "gpt-5.1-codex-max",
438
+ "gpt-5.1-chat-latest",
439
+ "gpt-5.2",
440
+ "gpt-5.2-2025-12-11",
441
+ "gpt-5.3-codex",
442
+ "gpt-5.3-codex-spark",
443
+ "gpt-audio",
444
+ "gpt-audio-2025-08-28",
445
+ "gpt-audio-mini",
446
+ "gpt-audio-mini-2025-10-06",
447
+ "computer-use-preview",
448
+ "computer-use-preview-2025-03-11",
449
+ "o1",
450
+ "o1-2024-12-17",
451
+ "o1-preview",
452
+ "o1-preview-2024-09-12",
453
+ "o1-mini",
454
+ "o1-mini-2024-09-12",
455
+ "o1-pro",
456
+ "o1-pro-2025-03-19",
457
+ "o3-pro",
458
+ "o3-pro-2025-06-10",
459
+ "o3",
460
+ "o3-2025-04-16",
461
+ "o4-mini",
462
+ "o4-mini-2025-04-16",
463
+ "o3-mini",
464
+ "o3-mini-2025-01-31",
465
+ "codex-mini-latest",
466
+ "gpt-5-codex",
467
+ "o3-deep-research",
468
+ "o3-deep-research-2025-06-26",
469
+ "o4-mini-deep-research",
470
+ "o4-mini-deep-research-2025-06-26"
471
+ ];
472
+ config;
473
+ constructor(modelName, options = {}) {
474
+ super(modelName, options);
475
+ this.config = options.config || {};
476
+ this.processor = new ResponsesProcessor({
477
+ modelName: this.modelName,
478
+ providerType: "openai",
479
+ functionCallbackHandler: this.functionCallbackHandler,
480
+ costCalculator: (modelName, usage, config) => require_util$1.calculateOpenAICost(modelName, config, usage?.input_tokens, usage?.output_tokens, 0, 0) ?? 0
481
+ });
482
+ }
483
+ isGPT5Model() {
484
+ return this.modelName.startsWith("gpt-5") || this.modelName.includes("/gpt-5");
485
+ }
486
+ isReasoningModel() {
487
+ return this.modelName.startsWith("o1") || this.modelName.startsWith("o3") || this.modelName.startsWith("o4") || this.modelName.includes("/o1") || this.modelName.includes("/o3") || this.modelName.includes("/o4") || this.modelName === "codex-mini-latest" || this.isGPT5Model();
488
+ }
489
+ supportsTemperature() {
490
+ return !this.isReasoningModel();
491
+ }
492
+ async getOpenAiBody(prompt, context, _callApiOptions) {
493
+ const config = {
494
+ ...this.config,
495
+ ...context?.prompt?.config
496
+ };
497
+ let input;
498
+ try {
499
+ const parsedJson = JSON.parse(prompt);
500
+ if (Array.isArray(parsedJson)) input = parsedJson;
501
+ else input = prompt;
502
+ } catch {
503
+ input = prompt;
504
+ }
505
+ const isReasoningModel = this.isReasoningModel();
506
+ const maxOutputTokens = config.max_output_tokens ?? (isReasoningModel ? require_logger.getEnvInt("OPENAI_MAX_COMPLETION_TOKENS") : require_logger.getEnvInt("OPENAI_MAX_TOKENS", 1024));
507
+ const temperature = this.supportsTemperature() ? config.temperature ?? require_logger.getEnvFloat("OPENAI_TEMPERATURE", 0) : void 0;
508
+ const reasoningEffort = isReasoningModel ? require_util.renderVarsInObject(config.reasoning_effort, context?.vars) : void 0;
509
+ const instructions = config.instructions;
510
+ const responseFormat = require_util.maybeLoadResponseFormatFromExternalFile(config.response_format, context?.vars);
511
+ let textFormat;
512
+ if (responseFormat) if (responseFormat.type === "json_object") textFormat = { format: { type: "json_object" } };
513
+ else if (responseFormat.type === "json_schema") {
514
+ const schema = responseFormat.schema || responseFormat.json_schema?.schema;
515
+ textFormat = { format: {
516
+ type: "json_schema",
517
+ name: responseFormat.json_schema?.name || responseFormat.name || "response_schema",
518
+ schema,
519
+ strict: true
520
+ } };
521
+ } else textFormat = { format: { type: "text" } };
522
+ else textFormat = { format: { type: "text" } };
523
+ if (this.isGPT5Model() && config.verbosity) textFormat = {
524
+ ...textFormat,
525
+ verbosity: config.verbosity
526
+ };
527
+ const loadedTools = config.tools ? await require_util.maybeLoadToolsFromExternalFile(config.tools, context?.vars) : void 0;
528
+ const body = {
529
+ model: this.modelName,
530
+ input,
531
+ ...maxOutputTokens !== void 0 ? { max_output_tokens: maxOutputTokens } : {},
532
+ ...reasoningEffort ? { reasoning: { effort: reasoningEffort } } : {},
533
+ ...temperature !== void 0 ? { temperature } : {},
534
+ ...instructions ? { instructions } : {},
535
+ ...config.top_p !== void 0 || require_logger.getEnvString("OPENAI_TOP_P") ? { top_p: config.top_p ?? require_logger.getEnvFloat("OPENAI_TOP_P", 1) } : {},
536
+ ...loadedTools ? { tools: loadedTools } : {},
537
+ ...config.tool_choice ? { tool_choice: config.tool_choice } : {},
538
+ ...config.max_tool_calls ? { max_tool_calls: config.max_tool_calls } : {},
539
+ ...config.previous_response_id ? { previous_response_id: config.previous_response_id } : {},
540
+ text: textFormat,
541
+ ...config.truncation ? { truncation: config.truncation } : {},
542
+ ...config.metadata ? { metadata: config.metadata } : {},
543
+ ..."parallel_tool_calls" in config ? { parallel_tool_calls: Boolean(config.parallel_tool_calls) } : {},
544
+ ...config.stream ? { stream: config.stream } : {},
545
+ ..."store" in config ? { store: Boolean(config.store) } : {},
546
+ ...config.background ? { background: config.background } : {},
547
+ ...config.webhook_url ? { webhook_url: config.webhook_url } : {},
548
+ ...config.user ? { user: config.user } : {},
549
+ ...config.passthrough || {}
550
+ };
551
+ if (config.reasoning && this.isReasoningModel()) body.reasoning = config.reasoning;
552
+ return {
553
+ body,
554
+ config: {
555
+ ...config,
556
+ tools: loadedTools,
557
+ response_format: responseFormat
558
+ }
559
+ };
560
+ }
561
+ async callApi(prompt, context, callApiOptions) {
562
+ if (!this.getApiKey()) throw new Error("OpenAI API key is not set. Set the OPENAI_API_KEY environment variable or add `apiKey` to the provider config.");
563
+ const { body, config } = await this.getOpenAiBody(prompt, context, callApiOptions);
564
+ const isDeepResearchModel = this.modelName.includes("deep-research");
565
+ if (isDeepResearchModel) {
566
+ if (!config.tools?.some((tool) => tool.type === "web_search_preview")) return { error: `Deep research model ${this.modelName} requires the web_search_preview tool to be configured. Add it to your provider config:\ntools:\n - type: web_search_preview` };
567
+ const mcpTools = config.tools?.filter((tool) => tool.type === "mcp") || [];
568
+ for (const mcpTool of mcpTools) if (mcpTool.require_approval !== "never") return { error: `Deep research model ${this.modelName} requires MCP tools to have require_approval: 'never'. Update your MCP tool configuration:\ntools:\n - type: mcp\n require_approval: never` };
569
+ }
570
+ let timeout = require_fetch.REQUEST_TIMEOUT_MS;
571
+ if (isDeepResearchModel || this.modelName.includes("gpt-5-pro")) {
572
+ const evalTimeout = require_logger.getEnvInt("PROMPTFOO_EVAL_TIMEOUT_MS", 0);
573
+ timeout = evalTimeout > 0 ? evalTimeout : require_fetch.LONG_RUNNING_MODEL_TIMEOUT_MS;
574
+ require_logger.logger_default.debug(`Using timeout of ${timeout}ms for long-running model ${this.modelName}`);
575
+ }
576
+ let data;
577
+ let status;
578
+ let statusText;
579
+ let cached = false;
580
+ let deleteFromCache;
581
+ let responseHeaders;
582
+ try {
583
+ ({data, cached, status, statusText, deleteFromCache, headers: responseHeaders} = await require_cache.fetchWithCache(`${this.getApiUrl()}/responses`, {
584
+ method: "POST",
585
+ headers: {
586
+ "Content-Type": "application/json",
587
+ Authorization: `Bearer ${this.getApiKey()}`,
588
+ ...this.getOrganization() ? { "OpenAI-Organization": this.getOrganization() } : {},
589
+ ...config.headers
590
+ },
591
+ body: JSON.stringify(body)
592
+ }, timeout, "json", context?.bustCache ?? context?.debug, this.config.maxRetries));
593
+ if (status < 200 || status >= 300) {
594
+ const errorMessage = `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}`;
595
+ if (typeof data === "object" && data?.error?.code === "invalid_prompt") return {
596
+ output: errorMessage,
597
+ tokenUsage: data?.usage ? require_util$1.getTokenUsage(data, cached) : void 0,
598
+ isRefusal: true,
599
+ metadata: { http: {
600
+ status,
601
+ statusText,
602
+ headers: responseHeaders ?? {}
603
+ } }
604
+ };
605
+ return {
606
+ error: errorMessage,
607
+ metadata: { http: {
608
+ status,
609
+ statusText,
610
+ headers: responseHeaders ?? {}
611
+ } }
612
+ };
613
+ }
614
+ } catch (err) {
615
+ require_logger.logger_default.error(`API call error: ${String(err)}`);
616
+ await deleteFromCache?.();
617
+ return {
618
+ error: `API call error: ${String(err)}`,
619
+ metadata: { http: {
620
+ status: 0,
621
+ statusText: "Error",
622
+ headers: responseHeaders ?? {}
623
+ } }
624
+ };
625
+ }
626
+ if (data.error?.message) {
627
+ await deleteFromCache?.();
628
+ return {
629
+ error: require_util$1.formatOpenAiError(data),
630
+ metadata: { http: {
631
+ status,
632
+ statusText,
633
+ headers: responseHeaders ?? {}
634
+ } }
635
+ };
636
+ }
637
+ const result = await this.processor.processResponseOutput(data, config, cached);
638
+ return {
639
+ ...result,
640
+ metadata: {
641
+ ...result.metadata,
642
+ http: {
643
+ status,
644
+ statusText,
645
+ headers: responseHeaders ?? {}
646
+ }
647
+ }
648
+ };
649
+ }
650
+ };
651
+
652
+ //#endregion
653
+ Object.defineProperty(exports, 'FunctionCallbackHandler', {
654
+ enumerable: true,
655
+ get: function () {
656
+ return FunctionCallbackHandler;
657
+ }
658
+ });
659
+ Object.defineProperty(exports, 'OpenAiResponsesProvider', {
660
+ enumerable: true,
661
+ get: function () {
662
+ return OpenAiResponsesProvider;
663
+ }
664
+ });
665
+ Object.defineProperty(exports, 'ResponsesProcessor', {
666
+ enumerable: true,
667
+ get: function () {
668
+ return ResponsesProcessor;
669
+ }
670
+ });
671
+ //# sourceMappingURL=responses-zOtKtnY_.cjs.map
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env node
2
- import { C as getEnvString, g as safeJsonStringify, o as logger_default } from "./logger-Bzi5o47S.js";
3
- import { n as getWrapperDir } from "./esm-Lh6czAFT.js";
2
+ import { C as getEnvString, g as safeJsonStringify, o as logger_default } from "./logger-BotXmWKW.js";
3
+ import { n as getWrapperDir } from "./esm-CYhseqj4.js";
4
4
  import fs from "fs";
5
5
  import path from "path";
6
6
  import os from "os";
@@ -213,4 +213,4 @@ async function runRuby(scriptPath, method, args, options = {}) {
213
213
 
214
214
  //#endregion
215
215
  export { validateRubyPath as a, tryPath as i, runRuby as n, state as r, getSysExecutable as t };
216
- //# sourceMappingURL=rubyUtils-Dn6MGcsk.js.map
216
+ //# sourceMappingURL=rubyUtils-BtjlqyXt.js.map
@@ -1,5 +1,5 @@
1
- import { b as getEnvString, i as logger_default, m as safeJsonStringify } from "./logger-q5I8CByj.js";
2
- import { n as getWrapperDir } from "./esm-D40XsWlR.js";
1
+ import { b as getEnvString, i as logger_default, m as safeJsonStringify } from "./logger-w8Ozp0Td.js";
2
+ import { n as getWrapperDir } from "./esm-rDtG_2rg.js";
3
3
  import fs from "fs";
4
4
  import path from "path";
5
5
  import os from "os";
@@ -212,4 +212,4 @@ async function runRuby(scriptPath, method, args, options = {}) {
212
212
 
213
213
  //#endregion
214
214
  export { validateRubyPath as a, tryPath as i, runRuby as n, state as r, getSysExecutable as t };
215
- //# sourceMappingURL=rubyUtils-LBsk3zIm.js.map
215
+ //# sourceMappingURL=rubyUtils-Cs35SDYa.js.map