promptfoo 0.125.0 → 0.125.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (869) hide show
  1. package/README.md +3 -3
  2. package/dist/src/accounts-BOxNYxVN.js +229 -0
  3. package/dist/src/accounts-BezXLdAW.cjs +276 -0
  4. package/dist/src/accounts-CPISMNlh.js +227 -0
  5. package/dist/src/accounts-vJBPs89d.js +222 -0
  6. package/dist/src/agentWorkspace-Bom7SpF2.js +592 -0
  7. package/dist/src/agentWorkspace-CEomAcws.cjs +619 -0
  8. package/dist/src/agentWorkspace-DPtMLrlI.js +593 -0
  9. package/dist/src/agentWorkspace-DkAeEYLz.js +592 -0
  10. package/dist/src/agentic-utils-121v7mJo.js +197 -0
  11. package/dist/src/agentic-utils-CBWD_rcd.cjs +237 -0
  12. package/dist/src/agentic-utils-D8wyQwPG.js +198 -0
  13. package/dist/src/agentic-utils-EP1QkQCs.js +197 -0
  14. package/dist/src/agents-Br3ah8iA.js +778 -0
  15. package/dist/src/agents-C5wj_UTv.js +777 -0
  16. package/dist/src/agents-C9Efp4mC.js +266 -0
  17. package/dist/src/agents-CQYwaVCO.js +267 -0
  18. package/dist/src/agents-CjCd4Vo8.js +777 -0
  19. package/dist/src/agents-DRWiCHYz.js +266 -0
  20. package/dist/src/agents-F2rhx0iL.cjs +266 -0
  21. package/dist/src/agents-RTYyy69h.cjs +777 -0
  22. package/dist/src/agents-api-CY72O86e.js +711 -0
  23. package/dist/src/agents-api-CsSsC21b.js +710 -0
  24. package/dist/src/agents-api-DN-ibDc8.js +710 -0
  25. package/dist/src/agents-api-cNDsCCv1.cjs +710 -0
  26. package/dist/src/ai.studio-Bj6Dyi6o.js +2 -0
  27. package/dist/src/ai.studio-C-e0LzyW.js +369 -0
  28. package/dist/src/ai.studio-CR8M6Aqf.cjs +391 -0
  29. package/dist/src/ai.studio-YKk7F0tv.js +374 -0
  30. package/dist/src/aimlapi-BGunsuTu.js +30 -0
  31. package/dist/src/aimlapi-BkzcfBsL.js +29 -0
  32. package/dist/src/aimlapi-DXFcEdP8.js +28 -0
  33. package/dist/src/aimlapi-r-8iCcAw.cjs +29 -0
  34. package/dist/src/anthropicMessages-7-YNRgQR.js +114 -0
  35. package/dist/src/anthropicMessages-C8m9Whrd.cjs +114 -0
  36. package/dist/src/anthropicMessages-DS1vXD0f.js +115 -0
  37. package/dist/src/anthropicMessages-DmvGkz-0.js +114 -0
  38. package/dist/src/app/assets/index-CrdXWTXm.js +392 -0
  39. package/dist/src/app/browser-inventory.json +4 -4
  40. package/dist/src/app/index.html +1 -1
  41. package/dist/src/audio-BLgFgyw-.cjs +104 -0
  42. package/dist/src/audio-C2sibT7Q.js +104 -0
  43. package/dist/src/audio-DXEXb03H.js +104 -0
  44. package/dist/src/audio-TkvojpwO.js +105 -0
  45. package/dist/src/authoritativeMarkupInjection-BAT9B-Wj.js +79 -0
  46. package/dist/src/authoritativeMarkupInjection-B_yHETKT.js +78 -0
  47. package/dist/src/authoritativeMarkupInjection-BfjecQ3Q.cjs +78 -0
  48. package/dist/src/authoritativeMarkupInjection-D-hDKrhD.js +78 -0
  49. package/dist/src/aws-B_9unI_-.js +1149 -0
  50. package/dist/src/aws-D0FY3qkc.js +2433 -0
  51. package/dist/src/aws-Dh1-a90n.js +2433 -0
  52. package/dist/src/aws-qskKGczY.cjs +2454 -0
  53. package/dist/src/base-CaWlBkVz.js +462 -0
  54. package/dist/src/base-CcViNwVs.js +255 -0
  55. package/dist/src/base-CsVbPJkU.cjs +467 -0
  56. package/dist/src/base-DNK9py25.js +255 -0
  57. package/dist/src/base-DlH5tjEZ.js +463 -0
  58. package/dist/src/base-DyLOZ_iX.js +256 -0
  59. package/dist/src/base-RRYXfG7l.cjs +305 -0
  60. package/dist/src/base-oXVgvtVn.js +462 -0
  61. package/dist/src/bedrock-BP17ZXbR.cjs +2 -0
  62. package/dist/src/bedrock-CsjM1dOt.js +1300 -0
  63. package/dist/src/bedrock-CzPwj8GO.js +1 -0
  64. package/dist/src/bedrock-eGXZw8oh.js +1 -0
  65. package/dist/src/bedrock-tSWjmXdj.js +2 -0
  66. package/dist/src/bestOfN-BQ93nUKV.js +160 -0
  67. package/dist/src/bestOfN-ByGGwlcb.cjs +161 -0
  68. package/dist/src/bestOfN-CbP7csHr.js +159 -0
  69. package/dist/src/bestOfN-xn6i2Xs2.js +159 -0
  70. package/dist/src/cache-CHq9hPz9.js +2 -0
  71. package/dist/src/cache-D-9eZN3I.js +754 -0
  72. package/dist/src/cache-DSm6t1MZ.js +741 -0
  73. package/dist/src/cache-DeF5zIEN.js +782 -0
  74. package/dist/src/cache-iG87oYan.cjs +856 -0
  75. package/dist/src/cerebras-CTtM4yMH.js +98 -0
  76. package/dist/src/cerebras-CogcJ3ce.cjs +97 -0
  77. package/dist/src/cerebras-DmkRypTh.js +97 -0
  78. package/dist/src/cerebras-GUp0B8JY.js +97 -0
  79. package/dist/src/chat-B4ihd3fw.js +226 -0
  80. package/dist/src/chat-B9cAQ_rS.js +227 -0
  81. package/dist/src/chat-BU7EB4R-.cjs +76 -0
  82. package/dist/src/chat-C8yxJ2-s.cjs +226 -0
  83. package/dist/src/chat-ClTFM3qq.js +934 -0
  84. package/dist/src/chat-Cnq0Is68.js +123 -0
  85. package/dist/src/chat-CwrurR57.cjs +987 -0
  86. package/dist/src/chat-D0lsojPv.js +124 -0
  87. package/dist/src/chat-D7_8yFIx.js +76 -0
  88. package/dist/src/chat-DCoETlLG.js +226 -0
  89. package/dist/src/chat-DHE7xaTR.js +76 -0
  90. package/dist/src/chat-DmhT6rz2.cjs +123 -0
  91. package/dist/src/chat-DvfMe8KI.js +123 -0
  92. package/dist/src/chat-DwP5kVnl.js +935 -0
  93. package/dist/src/chat-MB7U-QE2.js +77 -0
  94. package/dist/src/chat-zCWuBKAj.js +934 -0
  95. package/dist/src/claude-agent-sdk-23bIVI75.js +999 -0
  96. package/dist/src/claude-agent-sdk-BACpnIJi.cjs +1005 -0
  97. package/dist/src/claude-agent-sdk-BlYlmcEr.js +999 -0
  98. package/dist/src/claude-agent-sdk-Cr-jqJZl.js +1000 -0
  99. package/dist/src/client-BktbiPx9.cjs +357 -0
  100. package/dist/src/client-CjztEBmW.js +351 -0
  101. package/dist/src/client-mRs2UvH9.js +351 -0
  102. package/dist/src/client-uCz0ydjC.js +352 -0
  103. package/dist/src/cloud-BtYF555L.js +2 -0
  104. package/dist/src/cloud-CVSUEmAy.js +287 -0
  105. package/dist/src/cloud-D7Ifb-Fl.js +599 -0
  106. package/dist/src/cloud-DB5oK5NX.cjs +666 -0
  107. package/dist/src/cloud-ecu989Em.js +657 -0
  108. package/dist/src/cloud-ramjGURF.js +599 -0
  109. package/dist/src/cloudflare-ai-BfDkIjO8.js +133 -0
  110. package/dist/src/cloudflare-ai-D9HaXz1M.js +134 -0
  111. package/dist/src/cloudflare-ai-HrD5WUK8.js +132 -0
  112. package/dist/src/cloudflare-ai-Qdj8aKP2.cjs +133 -0
  113. package/dist/src/cloudflare-gateway-C81uKETQ.js +276 -0
  114. package/dist/src/cloudflare-gateway-CX5Z9X0k.js +276 -0
  115. package/dist/src/cloudflare-gateway-DW-og8gb.js +277 -0
  116. package/dist/src/cloudflare-gateway-iRyXo9n2.cjs +276 -0
  117. package/dist/src/codex-app-server-BFIrSgdy.js +2155 -0
  118. package/dist/src/codex-app-server-By1eDVD4.js +2 -0
  119. package/dist/src/codex-app-server-DKsF4qmY.js +2156 -0
  120. package/dist/src/codex-app-server-n7n4uZzC.cjs +2162 -0
  121. package/dist/src/codex-app-server-xAoELVxu.js +2155 -0
  122. package/dist/src/codex-sdk-C2ZLxa6r.js +1552 -0
  123. package/dist/src/codex-sdk-CzG02tQt.cjs +1588 -0
  124. package/dist/src/codex-sdk-DNxAOhiV.js +2 -0
  125. package/dist/src/codex-sdk-DT3aQezK.js +1557 -0
  126. package/dist/src/codex-sdk-pw9DpcJP.js +1557 -0
  127. package/dist/src/codex-security-B0bLmXJF.js +406 -0
  128. package/dist/src/codex-security-BhYs-3iY.js +406 -0
  129. package/dist/src/codex-security-DQcOfxYW.js +407 -0
  130. package/dist/src/codex-security-DsCWr6GZ.cjs +411 -0
  131. package/dist/src/cometapi-B16asPLX.cjs +55 -0
  132. package/dist/src/cometapi-MgTbZA_H.js +53 -0
  133. package/dist/src/cometapi-i33vKzPV.js +55 -0
  134. package/dist/src/cometapi-wK2UWK0X.js +56 -0
  135. package/dist/src/compatible-COj8ZCy9.js +24 -0
  136. package/dist/src/compatible-CRSsIUGb.cjs +24 -0
  137. package/dist/src/compatible-Dy1vCcNF.js +25 -0
  138. package/dist/src/compatible-aUItQrpk.js +24 -0
  139. package/dist/src/completion-Bn7COFMk.js +119 -0
  140. package/dist/src/completion-Bub2N48b.js +118 -0
  141. package/dist/src/completion-Cf2G9OlM.cjs +123 -0
  142. package/dist/src/conversationScoring-BiTYJ9MD.js +374 -0
  143. package/dist/src/conversationScoring-CKL47uIu.js +373 -0
  144. package/dist/src/conversationScoring-DCvQKp6k.cjs +398 -0
  145. package/dist/src/conversationScoring-DLNJtDZI.js +373 -0
  146. package/dist/src/crescendo-5iLk9WiJ.cjs +734 -0
  147. package/dist/src/crescendo-B5ylG7kW.js +733 -0
  148. package/dist/src/crescendo-BD7k6TIx.js +730 -0
  149. package/dist/src/crescendo-DqO2Ch0P.js +732 -0
  150. package/dist/src/custom-8rv_aG2K.js +617 -0
  151. package/dist/src/custom-BOtrIHFN.js +619 -0
  152. package/dist/src/custom-BvN8seCs.cjs +620 -0
  153. package/dist/src/custom-Cld9v_Y4.js +618 -0
  154. package/dist/src/dataUrl-BdHUey2l.cjs +3187 -0
  155. package/dist/src/dataUrl-DBMifX6C.js +3133 -0
  156. package/dist/src/dataUrl-DUSr-ZvI.js +2868 -0
  157. package/dist/src/decisions-CLazADTw.js +516 -0
  158. package/dist/src/decisions-D5X93awq.js +515 -0
  159. package/dist/src/decisions-DmaeArEn.js +515 -0
  160. package/dist/src/decisions-DyQJyHJz.cjs +514 -0
  161. package/dist/src/docker-C9NYQFeu.js +94 -0
  162. package/dist/src/docker-CGq3mMol.cjs +94 -0
  163. package/dist/src/docker-DOm7nmpl.js +95 -0
  164. package/dist/src/docker-skFN-Kfo.js +93 -0
  165. package/dist/src/embedding-0UbzkTk9.js +82 -0
  166. package/dist/src/embedding-DTrSDt7k.cjs +86 -0
  167. package/dist/src/embedding-j2ORNl5H.js +81 -0
  168. package/dist/src/envoy-BvS8XTQO.js +60 -0
  169. package/dist/src/envoy-Cn6Hzw0e.js +60 -0
  170. package/dist/src/envoy-DC3KWcUY.cjs +60 -0
  171. package/dist/src/envoy-DetHjo5B.js +61 -0
  172. package/dist/src/eval-Cer9dPe9.js +1920 -0
  173. package/dist/src/eval-CytYmI2Z.js +2 -0
  174. package/dist/src/evalResult-DzOq5wt9.js +1300 -0
  175. package/dist/src/evaluator-BxCmvGtv.js +10578 -0
  176. package/dist/src/evaluator-Ct7YDELm.js +2 -0
  177. package/dist/src/evaluatorHelpers-B_AwIrsl.js +600 -0
  178. package/dist/src/evaluatorHelpers-D1Tpc7gd.js +601 -0
  179. package/dist/src/evaluatorHelpers-DRR3rLq0.cjs +622 -0
  180. package/dist/src/fetch-B5KxZuKu.js +1559 -0
  181. package/dist/src/fetch-BG_Jl7ty.js +1556 -0
  182. package/dist/src/fetch-BKlgUg08.cjs +1853 -0
  183. package/dist/src/fetch-C9Fsja0R.js +2 -0
  184. package/dist/src/fetch-CM7P8KWA.cjs +14 -0
  185. package/dist/src/fetch-D0FkIB3G.js +9 -0
  186. package/dist/src/fetch-RXHsZF-J.js +1280 -0
  187. package/dist/src/fetch-rSECy8jR.js +10 -0
  188. package/dist/src/functionCallbackUtils-BB9tmf6a.cjs +262 -0
  189. package/dist/src/functionCallbackUtils-BXk2iY01.js +246 -0
  190. package/dist/src/functionCallbackUtils-B_cYp7P5.js +245 -0
  191. package/dist/src/functionCallbackUtils-BfXnPo0o.js +245 -0
  192. package/dist/src/gemini-image-BXLMADDz.js +471 -0
  193. package/dist/src/gemini-image-BYUZyEUx.js +472 -0
  194. package/dist/src/gemini-image-DSt1QubZ.cjs +471 -0
  195. package/dist/src/gemini-image-DdNDMiZ_.js +471 -0
  196. package/dist/src/goat-9b-uU_k6.js +514 -0
  197. package/dist/src/goat-BCJuiDd0.js +515 -0
  198. package/dist/src/goat-CmAHbemm.js +512 -0
  199. package/dist/src/goat-DdOTdyoS.cjs +515 -0
  200. package/dist/src/goblin-Bp5C5lyb.js +22 -0
  201. package/dist/src/goblin-Dmr7GItI.js +23 -0
  202. package/dist/src/goblin-DpM8wxlf.cjs +22 -0
  203. package/dist/src/goblin-cJyH6Pq-.js +22 -0
  204. package/dist/src/google-B6eaPAbE.js +142 -0
  205. package/dist/src/google-D-FStdN6.cjs +142 -0
  206. package/dist/src/google-DAC7BsCW.js +142 -0
  207. package/dist/src/google-Id18H0Va.js +143 -0
  208. package/dist/src/graders-BjNQWkaB.js +2 -0
  209. package/dist/src/graders-CqdIp_vl.js +15470 -0
  210. package/dist/src/hydra-6Ouh74DE.js +610 -0
  211. package/dist/src/hydra-BHjhZnLB.cjs +612 -0
  212. package/dist/src/hydra-BL6k2REV.js +2 -0
  213. package/dist/src/hydra-BWUc0o67.js +613 -0
  214. package/dist/src/hydra-CUkEOLKw.js +612 -0
  215. package/dist/src/image-B6hG_xhb.js +481 -0
  216. package/dist/src/image-BFaq7Krd.cjs +209 -0
  217. package/dist/src/image-Bql5C0N9.cjs +504 -0
  218. package/dist/src/image-BtZfI2pT.js +163 -0
  219. package/dist/src/image-CIM4oN7Z.cjs +163 -0
  220. package/dist/src/image-CbAl7Jpu.js +209 -0
  221. package/dist/src/image-CiYm_yFw.js +209 -0
  222. package/dist/src/image-DCMNvBiV.js +163 -0
  223. package/dist/src/image-DL7pIuUZ.js +210 -0
  224. package/dist/src/image-ey0kxF8o.js +164 -0
  225. package/dist/src/image-smgIk3rE.js +482 -0
  226. package/dist/src/index.cjs +27 -27
  227. package/dist/src/index.d.cts +8 -8
  228. package/dist/src/index.js +27 -27
  229. package/dist/src/indirectWebPwn-B_-UpWY5.js +285 -0
  230. package/dist/src/indirectWebPwn-Cz3SLiEq.js +434 -0
  231. package/dist/src/indirectWebPwn-D2s8zGiM.cjs +463 -0
  232. package/dist/src/indirectWebPwn-DXbj2wGu.cjs +285 -0
  233. package/dist/src/indirectWebPwn-D_4H44E6.js +285 -0
  234. package/dist/src/indirectWebPwn-L044SrpV.js +434 -0
  235. package/dist/src/indirectWebPwn-cnmvdavH.js +435 -0
  236. package/dist/src/indirectWebPwn-zUBzsqTt.js +286 -0
  237. package/dist/src/interactions-Bu3oVQ-j.js +850 -0
  238. package/dist/src/interactions-CND010Lh.cjs +850 -0
  239. package/dist/src/interactions-DAx7tqUz.js +851 -0
  240. package/dist/src/interactions-DPkIumeh.js +850 -0
  241. package/dist/src/iterative-BO4JRnpG.js +545 -0
  242. package/dist/src/iterative-CAK-k9rz.js +548 -0
  243. package/dist/src/iterative-CDgHc0xO.cjs +549 -0
  244. package/dist/src/iterative-CXJlHLl9.js +547 -0
  245. package/dist/src/iterativeImage-BEdGKSmh.cjs +419 -0
  246. package/dist/src/iterativeImage-BfL0QZw5.js +417 -0
  247. package/dist/src/iterativeImage-C9KhnibC.js +417 -0
  248. package/dist/src/iterativeImage-btjkOfWG.js +418 -0
  249. package/dist/src/iterativeMeta-B5Bdw6QA.js +437 -0
  250. package/dist/src/iterativeMeta-C3mI4gpT.js +440 -0
  251. package/dist/src/iterativeMeta-Co2BKyqS.cjs +439 -0
  252. package/dist/src/iterativeMeta-DD8ihRWD.js +439 -0
  253. package/dist/src/iterativeTree-AMi4GV2O.js +878 -0
  254. package/dist/src/iterativeTree-C8FJOEje.cjs +879 -0
  255. package/dist/src/iterativeTree-CvrY1f2Z.js +877 -0
  256. package/dist/src/iterativeTree-bPtESmaD.js +876 -0
  257. package/dist/src/knowledgeBase-Cf5871e3.js +163 -0
  258. package/dist/src/knowledgeBase-D2PQhqLH.js +164 -0
  259. package/dist/src/knowledgeBase-DpoMw1IH.cjs +163 -0
  260. package/dist/src/knowledgeBase-npGFl763.js +163 -0
  261. package/dist/src/litellm-BBB5DQdt.cjs +146 -0
  262. package/dist/src/litellm-Bo43ow7r.js +145 -0
  263. package/dist/src/litellm-CaU8CMLS.js +146 -0
  264. package/dist/src/litellm-wY7Unh8o.js +147 -0
  265. package/dist/src/live-5g2w-M-E.js +891 -0
  266. package/dist/src/live-BfaXsDyD.cjs +891 -0
  267. package/dist/src/live-C1xewnGD.js +2 -0
  268. package/dist/src/live-DaF7nYy5.js +892 -0
  269. package/dist/src/live-lp3urLUw.js +891 -0
  270. package/dist/src/luma-ray-B7zvKGVG.js +309 -0
  271. package/dist/src/luma-ray-DCf5fnrz.cjs +309 -0
  272. package/dist/src/luma-ray-DepBcn6z.js +309 -0
  273. package/dist/src/luma-ray-XQC4M1pt.js +310 -0
  274. package/dist/src/main.js +26 -26
  275. package/dist/src/mantleChat-7xm95vSV.js +154 -0
  276. package/dist/src/mantleChat-CXhAs5RD.js +155 -0
  277. package/dist/src/mantleChat-CcE-zjfj.cjs +154 -0
  278. package/dist/src/mantleChat-Cpefl7Zm.js +154 -0
  279. package/dist/src/mcp-B0V0ikdM.js +84 -0
  280. package/dist/src/mcp-CV2k2gwj.js +85 -0
  281. package/dist/src/memoryPoisoning-BC5FQQWZ.js +111 -0
  282. package/dist/src/memoryPoisoning-BDwrtZKX.js +111 -0
  283. package/dist/src/memoryPoisoning-BFemS_FF.js +112 -0
  284. package/dist/src/memoryPoisoning-DRs2rbVV.cjs +111 -0
  285. package/dist/src/messages-Bc5oZoAJ.js +1128 -0
  286. package/dist/src/messages-BkaH8fRW.js +1127 -0
  287. package/dist/src/messages-CoHlziS5.js +1127 -0
  288. package/dist/src/messages-Cv_m6kYs.cjs +1161 -0
  289. package/dist/src/mischievousUser-BcyFxrqJ.js +51 -0
  290. package/dist/src/mischievousUser-BlhOQSd2.js +52 -0
  291. package/dist/src/mischievousUser-BuQiKE8D.js +51 -0
  292. package/dist/src/mischievousUser-ChIOk6cT.cjs +51 -0
  293. package/dist/src/modelslab-BXIWBTgX.js +171 -0
  294. package/dist/src/modelslab-ChVsUiaJ.js +170 -0
  295. package/dist/src/modelslab-DZ6BFlDv.js +170 -0
  296. package/dist/src/modelslab-DhRr-vyo.cjs +170 -0
  297. package/dist/src/nova-reel-BL8DrN4_.js +272 -0
  298. package/dist/src/nova-reel-BiOMUm5q.cjs +272 -0
  299. package/dist/src/nova-reel-CTsya1hV.js +273 -0
  300. package/dist/src/nova-reel-kgBY4Svd.js +272 -0
  301. package/dist/src/nova-sonic-CaSSA3Jq.cjs +374 -0
  302. package/dist/src/nova-sonic-DLpmoGsR.js +375 -0
  303. package/dist/src/nova-sonic-DwWc4SoN.js +374 -0
  304. package/dist/src/nova-sonic-qnReTaXm.js +374 -0
  305. package/dist/src/novita-B5mAt3Ui.js +114 -0
  306. package/dist/src/novita-D57vCjDD.js +116 -0
  307. package/dist/src/novita-ITRtG8IJ.js +115 -0
  308. package/dist/src/novita-Rh3T13YO.cjs +115 -0
  309. package/dist/src/nscale-8V7iZ_pI.js +210 -0
  310. package/dist/src/nscale-BQnMv4pH.cjs +209 -0
  311. package/dist/src/nscale-BWZXHZVF.js +209 -0
  312. package/dist/src/nscale-BqrHiUqU.js +207 -0
  313. package/dist/src/openaiResponses-BMtGTu8R.js +317 -0
  314. package/dist/src/openaiResponses-CvcI_viE.js +318 -0
  315. package/dist/src/openaiResponses-DYPZEMX1.cjs +318 -0
  316. package/dist/src/openaiResponses-DZTg1-kb.js +319 -0
  317. package/dist/src/openclaw-BeKuig4g.js +1338 -0
  318. package/dist/src/openclaw-COkdQ83y.js +1337 -0
  319. package/dist/src/openclaw-CSoX_HQ5.cjs +1344 -0
  320. package/dist/src/openclaw-DJlhPGCb.js +1336 -0
  321. package/dist/src/opencode-sdk-BEJftEs6.cjs +1018 -0
  322. package/dist/src/opencode-sdk-DMvBB3V7.js +1013 -0
  323. package/dist/src/opencode-sdk-Df7hMjLO.js +1012 -0
  324. package/dist/src/opencode-sdk-bQp5Xf3W.js +1012 -0
  325. package/dist/src/openinterpreter-B3GgVZjK.js +431 -0
  326. package/dist/src/openinterpreter-BiGcU-HN.cjs +435 -0
  327. package/dist/src/openinterpreter-Buj2l3Eh.js +432 -0
  328. package/dist/src/openinterpreter-C3_Cg5b_.js +431 -0
  329. package/dist/src/pricing-0Pampf0q.cjs +999 -0
  330. package/dist/src/pricing-B8VdhEhY.js +976 -0
  331. package/dist/src/pricing-CtLA-rKA.js +976 -0
  332. package/dist/src/promptfoo-4OkHtH9P.cjs +341 -0
  333. package/dist/src/promptfoo-Bhpm7sH6.js +286 -0
  334. package/dist/src/promptfoo-BqV8ogVA.js +285 -0
  335. package/dist/src/promptfoo-S1Q9OHk_.js +286 -0
  336. package/dist/src/providers-B5zs1ZRv.js +24437 -0
  337. package/dist/src/providers-Cd6CJh6m.js +2 -0
  338. package/dist/src/providers-CpJlgHET.js +24039 -0
  339. package/dist/src/providers-CzvLvK6G.cjs +24180 -0
  340. package/dist/src/quiverai-1E_IvLHn.js +325 -0
  341. package/dist/src/quiverai-BXH004Vv.js +325 -0
  342. package/dist/src/quiverai-Ccya2z8Y.cjs +325 -0
  343. package/dist/src/quiverai-Cez57oe-.js +326 -0
  344. package/dist/src/registry-3zX_M9Ik.js +131 -0
  345. package/dist/src/registry-AuCrKTvS.js +132 -0
  346. package/dist/src/registry-CxXWKtQG.js +131 -0
  347. package/dist/src/registry-DG3o66JQ.cjs +131 -0
  348. package/dist/src/remoteGeneration-CAtvamG6.js +370 -0
  349. package/dist/src/remoteGeneration-CF225pDl.cjs +482 -0
  350. package/dist/src/remoteGeneration-DtnqocnL.js +370 -0
  351. package/dist/src/remoteGeneration-g3KvW-sP.js +371 -0
  352. package/dist/src/responses-BMbaUXQI.js +1098 -0
  353. package/dist/src/responses-B_TOSof-.js +1099 -0
  354. package/dist/src/responses-v-yANwTA.cjs +1109 -0
  355. package/dist/src/runtimeTransform-BBZv-oUZ.js +156 -0
  356. package/dist/src/runtimeTransform-DT2v5ovW.js +156 -0
  357. package/dist/src/runtimeTransform-GXXEIXdS.cjs +161 -0
  358. package/dist/src/runtimeTransform-JTwBQc-P.js +157 -0
  359. package/dist/src/sagemaker-B8D0L4rj.cjs +716 -0
  360. package/dist/src/sagemaker-CR2_AGdB.js +713 -0
  361. package/dist/src/sagemaker-Cd9qSQUV.js +714 -0
  362. package/dist/src/sagemaker-mKlltN4p.js +713 -0
  363. package/dist/src/scanner-PG_8lh8r.js +1971 -0
  364. package/dist/src/server/index.js +55 -55
  365. package/dist/src/server-Bmh7sd9F.js +155 -0
  366. package/dist/src/server-C7b8z6aC.js +63 -0
  367. package/dist/src/server-CAcovgtL.js +2 -0
  368. package/dist/src/server-CuuMfKlh.cjs +74 -0
  369. package/dist/src/server-DMPCj9jy.js +189 -0
  370. package/dist/src/session-BksXvRH_.cjs +69 -0
  371. package/dist/src/session-DOxUjqGQ.js +63 -0
  372. package/dist/src/session-Dujsc-b9.js +62 -0
  373. package/dist/src/session-VjtIxREd.js +62 -0
  374. package/dist/src/shared-Bkyz_7e5.cjs +2185 -0
  375. package/dist/src/shared-DDJg8ycC.js +1929 -0
  376. package/dist/src/shared-Dh3yfRBd.js +1927 -0
  377. package/dist/src/shared-tfPT8Tna.js +1928 -0
  378. package/dist/src/simulatedUser-6JdkyL3k.cjs +252 -0
  379. package/dist/src/simulatedUser-CFr2mr3M.js +248 -0
  380. package/dist/src/simulatedUser-DjwwUqx-.js +247 -0
  381. package/dist/src/slack-C_gIYiUs.js +144 -0
  382. package/dist/src/slack-DhNbz9CM.cjs +144 -0
  383. package/dist/src/slack-Dm2iI1zu.js +145 -0
  384. package/dist/src/strategies-BIY-bNul.js +2248 -0
  385. package/dist/src/strategies-C1KQQj9G.cjs +2420 -0
  386. package/dist/src/strategies-DM_3mReG.js +2385 -0
  387. package/dist/src/telemetry-BhMTibZO.js +2 -0
  388. package/dist/src/telemetry-C04d25BM.cjs +216 -0
  389. package/dist/src/telemetry-CIqy4fSC.js +201 -0
  390. package/dist/src/telemetry-CXozp_6C.js +205 -0
  391. package/dist/src/telemetry-g7PkBxmY.js +205 -0
  392. package/dist/src/togetherai-BvCim68-.js +50 -0
  393. package/dist/src/togetherai-CWyDCA9m.cjs +49 -0
  394. package/dist/src/togetherai-CeSE31vK.js +49 -0
  395. package/dist/src/togetherai-_2W0URf8.js +48 -0
  396. package/dist/src/traceContext-Brw3Oe41.js +1016 -0
  397. package/dist/src/traceContext-DpAYv27G.js +1015 -0
  398. package/dist/src/traceContext-KSqhzZI2.cjs +1032 -0
  399. package/dist/src/transcription-BySopazW.js +175 -0
  400. package/dist/src/transcription-DO4UPkRI.cjs +178 -0
  401. package/dist/src/transcription-RJGBa-pC.js +176 -0
  402. package/dist/src/transcription-fZenF3G5.js +175 -0
  403. package/dist/src/transform-BzhRc5z2.js +125 -0
  404. package/dist/src/transform-CPQu05u6.cjs +154 -0
  405. package/dist/src/transform-DAED8imp.js +126 -0
  406. package/dist/src/transform-DIOIQDmM.js +125 -0
  407. package/dist/src/typesafe-ByE0Kxfg.js +253 -0
  408. package/dist/src/typesafe-CFqGFY34.js +253 -0
  409. package/dist/src/typesafe-DCWIw-CP.cjs +253 -0
  410. package/dist/src/typesafe-DyHFV5Qk.js +254 -0
  411. package/dist/src/util-BHkj9joz.js +2049 -0
  412. package/dist/src/util-CICwIpmu.cjs +934 -0
  413. package/dist/src/util-CePiY2xZ.js +286 -0
  414. package/dist/src/util-CgT3uf4p.js +743 -0
  415. package/dist/src/util-Cj9drvOY.js +1770 -0
  416. package/dist/src/util-ClJRjlQA.cjs +357 -0
  417. package/dist/src/util-Cxf8VF2M.js +395 -0
  418. package/dist/src/util-D7RdV7rS.js +395 -0
  419. package/dist/src/util-DMNFhk47.js +351 -0
  420. package/dist/src/util-DNzh3M0z.js +396 -0
  421. package/dist/src/util-Df7rfMGb.cjs +2305 -0
  422. package/dist/src/util-Kf0KnG_6.js +2051 -0
  423. package/dist/src/util-LEbeiYw_.js +743 -0
  424. package/dist/src/util-WH8CgVs4.js +352 -0
  425. package/dist/src/util-cx6Xw8Lw.cjs +478 -0
  426. package/dist/src/util-qzBqirBH.js +2050 -0
  427. package/dist/src/vertex-CBAMfRz3.js +854 -0
  428. package/dist/src/vertex-CMP2IpN9.js +2 -0
  429. package/dist/src/vertex-CZ5YOotT.js +848 -0
  430. package/dist/src/vertex-DNsh1F1M.cjs +871 -0
  431. package/dist/src/vertexLive-BTXYj3-F.js +55 -0
  432. package/dist/src/vertexLive-Ba_LAqEF.cjs +55 -0
  433. package/dist/src/vertexLive-DM4sFaaj.js +56 -0
  434. package/dist/src/vertexLive-JqZOw7HC.js +55 -0
  435. package/dist/src/video-BY7Q2Gii.cjs +706 -0
  436. package/dist/src/video-CYElP-CK.js +703 -0
  437. package/dist/src/video-C_AdEDH0.js +702 -0
  438. package/dist/src/video-DGvjQ_4j.js +702 -0
  439. package/package.json +1 -1
  440. package/dist/src/accounts-Dpb43yde.cjs +0 -276
  441. package/dist/src/accounts-GKrO1olH.js +0 -229
  442. package/dist/src/accounts-e4FGOsji.js +0 -227
  443. package/dist/src/accounts-sUiEJfkB.js +0 -222
  444. package/dist/src/agentWorkspace-B8A_wuMR.cjs +0 -619
  445. package/dist/src/agentWorkspace-DSPIscXR.js +0 -593
  446. package/dist/src/agentWorkspace-DxUqd-XK.js +0 -592
  447. package/dist/src/agentWorkspace-n0uOJXhD.js +0 -592
  448. package/dist/src/agentic-utils-9bCj4twQ.js +0 -197
  449. package/dist/src/agentic-utils-CyCLuLoj.js +0 -197
  450. package/dist/src/agentic-utils-D23qtOlx.cjs +0 -237
  451. package/dist/src/agentic-utils-DuBMMNCw.js +0 -198
  452. package/dist/src/agents-6hDRNoYD.js +0 -777
  453. package/dist/src/agents-B8w2o7Sc.cjs +0 -777
  454. package/dist/src/agents-Ck0NXl7v.js +0 -777
  455. package/dist/src/agents-CwT3B1qB.js +0 -778
  456. package/dist/src/agents-DOLjfOEq.js +0 -266
  457. package/dist/src/agents-OcF9hq--.js +0 -267
  458. package/dist/src/agents-api-CnZZt8bn.cjs +0 -710
  459. package/dist/src/agents-api-D04u3bmL.js +0 -711
  460. package/dist/src/agents-api-DWw0_Vio.js +0 -710
  461. package/dist/src/agents-api-IWHEwp8i.js +0 -710
  462. package/dist/src/agents-mHt2Mh9O.cjs +0 -266
  463. package/dist/src/agents-sICccZCA.js +0 -266
  464. package/dist/src/ai.studio-Ci4rypDT.js +0 -2
  465. package/dist/src/ai.studio-KkQ6KaHa.js +0 -369
  466. package/dist/src/ai.studio-gSXK2cGI.js +0 -374
  467. package/dist/src/ai.studio-nQF_B4DH.cjs +0 -391
  468. package/dist/src/aimlapi-BeSk5BDV.js +0 -30
  469. package/dist/src/aimlapi-CzNUhdKu.cjs +0 -29
  470. package/dist/src/aimlapi-DCkazDPI.js +0 -29
  471. package/dist/src/aimlapi-WeH5B6K1.js +0 -28
  472. package/dist/src/anthropicMessages-AcNWTpYO.js +0 -115
  473. package/dist/src/anthropicMessages-BUtvzbpL.js +0 -114
  474. package/dist/src/anthropicMessages-DTWhZDCd.js +0 -114
  475. package/dist/src/anthropicMessages-lGVnSgyx.cjs +0 -114
  476. package/dist/src/app/assets/index-CyEI5aDN.js +0 -392
  477. package/dist/src/audio-BE_Duiu1.js +0 -104
  478. package/dist/src/audio-CiUd00k2.cjs +0 -104
  479. package/dist/src/audio-Dop6omna.js +0 -105
  480. package/dist/src/audio-EZNTInr_.js +0 -104
  481. package/dist/src/authoritativeMarkupInjection-BUqH6hNk.js +0 -78
  482. package/dist/src/authoritativeMarkupInjection-BmbJdJO0.cjs +0 -78
  483. package/dist/src/authoritativeMarkupInjection-Cpz12Y3w.js +0 -78
  484. package/dist/src/authoritativeMarkupInjection-W4XfcH91.js +0 -79
  485. package/dist/src/aws-B2Jqbnet.js +0 -2433
  486. package/dist/src/aws-DHtuUIzQ.js +0 -1149
  487. package/dist/src/aws-DLqXP6wY.js +0 -2433
  488. package/dist/src/aws-DfduhmwL.cjs +0 -2454
  489. package/dist/src/base-B9L2Xvpc.js +0 -255
  490. package/dist/src/base-BTZnIsuH.cjs +0 -305
  491. package/dist/src/base-BhaWNQPJ.js +0 -462
  492. package/dist/src/base-Br-G_Xou.js +0 -256
  493. package/dist/src/base-Bzo8hiUS.cjs +0 -467
  494. package/dist/src/base-C82fd2jq.js +0 -463
  495. package/dist/src/base-CRdQwC1W.js +0 -462
  496. package/dist/src/base-CjGwZNOQ.js +0 -255
  497. package/dist/src/bedrock-BdMru_ro.js +0 -1300
  498. package/dist/src/bedrock-CBUmfLX1.cjs +0 -2
  499. package/dist/src/bedrock-DBDxBaF9.js +0 -2
  500. package/dist/src/bedrock-DDrECM21.js +0 -1
  501. package/dist/src/bedrock-DVwkgvRS.js +0 -1
  502. package/dist/src/bestOfN-AKF5beq4.js +0 -159
  503. package/dist/src/bestOfN-C1UIVGJV.js +0 -160
  504. package/dist/src/bestOfN-DlkETbXy.cjs +0 -161
  505. package/dist/src/bestOfN-XIfEgU-g.js +0 -159
  506. package/dist/src/cache-BZaVoKaA.js +0 -2
  507. package/dist/src/cache-CEYhFUsH.js +0 -741
  508. package/dist/src/cache-Cbw1zH6q.js +0 -782
  509. package/dist/src/cache-CdaZNb75.js +0 -754
  510. package/dist/src/cache-DpuIzPxh.cjs +0 -856
  511. package/dist/src/cerebras-Bpn6FPkJ.js +0 -97
  512. package/dist/src/cerebras-CqJ0BRCJ.cjs +0 -97
  513. package/dist/src/cerebras-D4ILa_TY.js +0 -97
  514. package/dist/src/cerebras-DR8uc7ee.js +0 -98
  515. package/dist/src/chat-B7PZfaYp.cjs +0 -987
  516. package/dist/src/chat-BVuSjdEb.js +0 -227
  517. package/dist/src/chat-BsaZRjXN.js +0 -76
  518. package/dist/src/chat-C5liqe3Y.js +0 -226
  519. package/dist/src/chat-C9hyHf5B.js +0 -124
  520. package/dist/src/chat-CFLalAck.js +0 -226
  521. package/dist/src/chat-CN8JGrHu.js +0 -76
  522. package/dist/src/chat-CaHBBpCb.js +0 -123
  523. package/dist/src/chat-CcbLSghx.js +0 -123
  524. package/dist/src/chat-CoSN6sr2.js +0 -934
  525. package/dist/src/chat-CsJRFX8a.cjs +0 -226
  526. package/dist/src/chat-DFoZLvO_.js +0 -934
  527. package/dist/src/chat-DiLQvL7r.js +0 -77
  528. package/dist/src/chat-F6qVSDk3.cjs +0 -123
  529. package/dist/src/chat-Oq4FaJgI.cjs +0 -76
  530. package/dist/src/chat-knMfQUKz.js +0 -935
  531. package/dist/src/claude-agent-sdk-CJkOLpnI.js +0 -999
  532. package/dist/src/claude-agent-sdk-Ckn76PRD.js +0 -999
  533. package/dist/src/claude-agent-sdk-DX0gTr11.cjs +0 -1005
  534. package/dist/src/claude-agent-sdk-LMeikxoR.js +0 -1000
  535. package/dist/src/client-BFf1r6Z8.js +0 -352
  536. package/dist/src/client-CdXyDdDi.js +0 -351
  537. package/dist/src/client-DV8XMu7i.js +0 -351
  538. package/dist/src/client-iEXtOBKW.cjs +0 -357
  539. package/dist/src/cloud-B02qcpjf.js +0 -2
  540. package/dist/src/cloud-BlHP-g1v.cjs +0 -666
  541. package/dist/src/cloud-C9BI4isF.js +0 -287
  542. package/dist/src/cloud-CTTLNdvj.js +0 -657
  543. package/dist/src/cloud-D5X5yQ_d.js +0 -599
  544. package/dist/src/cloud-QxPkpsHA.js +0 -599
  545. package/dist/src/cloudflare-ai-DRTRT1fc.js +0 -132
  546. package/dist/src/cloudflare-ai-DgaXapAV.cjs +0 -133
  547. package/dist/src/cloudflare-ai-DkuJXijS.js +0 -134
  548. package/dist/src/cloudflare-ai-wy9QCHku.js +0 -133
  549. package/dist/src/cloudflare-gateway-9cAN8g29.cjs +0 -276
  550. package/dist/src/cloudflare-gateway-BVFMD292.js +0 -277
  551. package/dist/src/cloudflare-gateway-BuUI4p6I.js +0 -276
  552. package/dist/src/cloudflare-gateway-TlYHizbW.js +0 -276
  553. package/dist/src/codex-app-server-BPz_feug.cjs +0 -2162
  554. package/dist/src/codex-app-server-C9noXM9C.js +0 -2155
  555. package/dist/src/codex-app-server-CEmD-Y5U.js +0 -2
  556. package/dist/src/codex-app-server-DZolc18r.js +0 -2156
  557. package/dist/src/codex-app-server-Dd0A37nO.js +0 -2155
  558. package/dist/src/codex-sdk-BbxjBvBk.cjs +0 -1588
  559. package/dist/src/codex-sdk-BpXeIs-_.js +0 -1557
  560. package/dist/src/codex-sdk-BtQp5X0f.js +0 -1552
  561. package/dist/src/codex-sdk-CH0v9Bk1.js +0 -2
  562. package/dist/src/codex-sdk-CgrN7JB8.js +0 -1557
  563. package/dist/src/codex-security-B5fENNUr.js +0 -406
  564. package/dist/src/codex-security-C6uuJx-y.cjs +0 -411
  565. package/dist/src/codex-security-CifWtZR_.js +0 -407
  566. package/dist/src/codex-security-Kg2bOEKy.js +0 -406
  567. package/dist/src/cometapi-BFpIDLIx.js +0 -55
  568. package/dist/src/cometapi-C6O2PIrN.js +0 -56
  569. package/dist/src/cometapi-D5Yupc6F.js +0 -53
  570. package/dist/src/cometapi-DzgIwJzJ.cjs +0 -55
  571. package/dist/src/compatible-0jhmVfOp.js +0 -25
  572. package/dist/src/compatible-DdtwWdJi.cjs +0 -24
  573. package/dist/src/compatible-Di4nR3sU.js +0 -24
  574. package/dist/src/compatible-iJKLWdnf.js +0 -24
  575. package/dist/src/completion-B0_hVUDG.js +0 -119
  576. package/dist/src/completion-BdI6v0-Q.cjs +0 -123
  577. package/dist/src/completion-CWcJscwv.js +0 -118
  578. package/dist/src/conversationScoring-99EOPJRa.js +0 -373
  579. package/dist/src/conversationScoring-BIq4J0Cq.js +0 -374
  580. package/dist/src/conversationScoring-DARyeATr.cjs +0 -398
  581. package/dist/src/conversationScoring-DS48t4Hb.js +0 -373
  582. package/dist/src/crescendo-CebvXIu4.js +0 -730
  583. package/dist/src/crescendo-CxrLS0oo.js +0 -732
  584. package/dist/src/crescendo-DZQXBSkj.js +0 -733
  585. package/dist/src/crescendo-wjEx68uD.cjs +0 -734
  586. package/dist/src/custom-BiQ-aUNw.cjs +0 -620
  587. package/dist/src/custom-ByyB5Z93.js +0 -619
  588. package/dist/src/custom-C5jOevNb.js +0 -617
  589. package/dist/src/custom-CCrsCz8o.js +0 -618
  590. package/dist/src/dataUrl-C9J7wz9H.js +0 -3133
  591. package/dist/src/dataUrl-CGsBT7UZ.cjs +0 -3187
  592. package/dist/src/dataUrl-CzKvvWNO.js +0 -2868
  593. package/dist/src/decisions-BTusXCUx.cjs +0 -514
  594. package/dist/src/decisions-BYtL9edD.js +0 -515
  595. package/dist/src/decisions-jYix_gCF.js +0 -515
  596. package/dist/src/decisions-nxxfHRAN.js +0 -516
  597. package/dist/src/docker-DDIv4zkg.js +0 -93
  598. package/dist/src/docker-DVLp_5OG.js +0 -94
  599. package/dist/src/docker-INm9w0SY.cjs +0 -94
  600. package/dist/src/docker-sGuoCul_.js +0 -95
  601. package/dist/src/embedding-CAhqLLrR.js +0 -82
  602. package/dist/src/embedding-D94GAsti.cjs +0 -86
  603. package/dist/src/embedding-zb4MAzQy.js +0 -81
  604. package/dist/src/envoy-BMPszwEa.cjs +0 -60
  605. package/dist/src/envoy-Bp-s39_8.js +0 -60
  606. package/dist/src/envoy-BswDt-Ou.js +0 -61
  607. package/dist/src/envoy-rVbYcV80.js +0 -60
  608. package/dist/src/eval-B1-XfrC7.js +0 -2
  609. package/dist/src/eval-pC4qCvLw.js +0 -1920
  610. package/dist/src/evalResult-CzjCb5bd.js +0 -1300
  611. package/dist/src/evaluator-CNq2K7J_.js +0 -10578
  612. package/dist/src/evaluator-LtfxdxEu.js +0 -2
  613. package/dist/src/evaluatorHelpers-C5J0NYpQ.js +0 -601
  614. package/dist/src/evaluatorHelpers-DcdCglCs.cjs +0 -622
  615. package/dist/src/evaluatorHelpers-Rvb88W6B.js +0 -600
  616. package/dist/src/fetch-5HtnNhHg.js +0 -1556
  617. package/dist/src/fetch-BCwjoIFd.js +0 -1559
  618. package/dist/src/fetch-BhfUnKen.js +0 -1280
  619. package/dist/src/fetch-C08w4vVl.js +0 -9
  620. package/dist/src/fetch-DBxy8Tlb.cjs +0 -14
  621. package/dist/src/fetch-DYte6oQX.cjs +0 -1853
  622. package/dist/src/fetch-ElpfcAVG.js +0 -10
  623. package/dist/src/fetch-lM1OIaSt.js +0 -2
  624. package/dist/src/functionCallbackUtils-BCAdWfHO.js +0 -246
  625. package/dist/src/functionCallbackUtils-BYq9qOhG.js +0 -245
  626. package/dist/src/functionCallbackUtils-ByWkmVud.cjs +0 -262
  627. package/dist/src/functionCallbackUtils-D_C_JaFd.js +0 -245
  628. package/dist/src/gemini-image-CAlwaJhZ.js +0 -471
  629. package/dist/src/gemini-image-DJoVoOnZ.cjs +0 -471
  630. package/dist/src/gemini-image-Waio-9Xx.js +0 -472
  631. package/dist/src/gemini-image-vbY6Ftbf.js +0 -471
  632. package/dist/src/goat-BWbtyY9c.js +0 -512
  633. package/dist/src/goat-CXzzeCOQ.cjs +0 -515
  634. package/dist/src/goat-CfgvzSgZ.js +0 -514
  635. package/dist/src/goat-DLeoIHBp.js +0 -515
  636. package/dist/src/goblin-CmfCfwqk.cjs +0 -22
  637. package/dist/src/goblin-CtcaORRH.js +0 -22
  638. package/dist/src/goblin-DF6DDhM-.js +0 -22
  639. package/dist/src/goblin-Nai0xxBV.js +0 -23
  640. package/dist/src/google-ClpkLtKL.cjs +0 -142
  641. package/dist/src/google-CrFgKu9k.js +0 -143
  642. package/dist/src/google-Dj_EIBqS.js +0 -142
  643. package/dist/src/google-e4-vARv6.js +0 -142
  644. package/dist/src/graders-BN1IIa71.js +0 -15470
  645. package/dist/src/graders-C5sCfSP5.js +0 -2
  646. package/dist/src/hydra-B7RfXVBK.js +0 -2
  647. package/dist/src/hydra-C1cFN-Jg.js +0 -613
  648. package/dist/src/hydra-CK6Uc1ia.js +0 -612
  649. package/dist/src/hydra-Dg65rAP_.js +0 -610
  650. package/dist/src/hydra-Fqx8l_Xu.cjs +0 -612
  651. package/dist/src/image-BdlUDAtp.js +0 -163
  652. package/dist/src/image-C-KwEhwt.cjs +0 -163
  653. package/dist/src/image-C76fp6NK.cjs +0 -209
  654. package/dist/src/image-D75ehv3D.js +0 -482
  655. package/dist/src/image-DKtc5v5W.js +0 -209
  656. package/dist/src/image-DT_xOkJY.js +0 -163
  657. package/dist/src/image-DVpdllEE.js +0 -209
  658. package/dist/src/image-DXZ8AzMf.js +0 -481
  659. package/dist/src/image-DZl_1mvy.js +0 -164
  660. package/dist/src/image-DcTD7Ak0.cjs +0 -504
  661. package/dist/src/image-ndxEvBKB.js +0 -210
  662. package/dist/src/indirectWebPwn-70WA5x_q.js +0 -435
  663. package/dist/src/indirectWebPwn-BRVyd3qV.cjs +0 -463
  664. package/dist/src/indirectWebPwn-BsJVsu0q.js +0 -285
  665. package/dist/src/indirectWebPwn-Cw6yJu0i.js +0 -285
  666. package/dist/src/indirectWebPwn-D-PhJzo4.js +0 -286
  667. package/dist/src/indirectWebPwn-DRPHt2I8.js +0 -434
  668. package/dist/src/indirectWebPwn-DXtfFhd9.cjs +0 -285
  669. package/dist/src/indirectWebPwn-DvpNouzc.js +0 -434
  670. package/dist/src/interactions-5ixfW7Cl.js +0 -850
  671. package/dist/src/interactions-BOUzfmg1.cjs +0 -850
  672. package/dist/src/interactions-CSSXcGNc.js +0 -851
  673. package/dist/src/interactions-WB_sgkN0.js +0 -850
  674. package/dist/src/iterative-BShRNqjC.cjs +0 -549
  675. package/dist/src/iterative-BcW_ELtB.js +0 -548
  676. package/dist/src/iterative-BpHbHz8B.js +0 -547
  677. package/dist/src/iterative-YcchP12d.js +0 -545
  678. package/dist/src/iterativeImage-BMFE5Hl7.cjs +0 -419
  679. package/dist/src/iterativeImage-CkSaYLMY.js +0 -417
  680. package/dist/src/iterativeImage-DAEUlQX_.js +0 -418
  681. package/dist/src/iterativeImage-DeUSkEnj.js +0 -417
  682. package/dist/src/iterativeMeta-DbdaKdEO.cjs +0 -439
  683. package/dist/src/iterativeMeta-DcF9JWEK.js +0 -437
  684. package/dist/src/iterativeMeta-DkfPqWIC.js +0 -439
  685. package/dist/src/iterativeMeta-DkrpnVSs.js +0 -440
  686. package/dist/src/iterativeTree-CEMY1FhE.cjs +0 -879
  687. package/dist/src/iterativeTree-Cv5W4tX9.js +0 -877
  688. package/dist/src/iterativeTree-DGvomd3M.js +0 -876
  689. package/dist/src/iterativeTree-KbtC2Mkz.js +0 -878
  690. package/dist/src/knowledgeBase-AYRXn4wO.js +0 -163
  691. package/dist/src/knowledgeBase-BNqQfx01.js +0 -163
  692. package/dist/src/knowledgeBase-C0IaCCVs.js +0 -164
  693. package/dist/src/knowledgeBase-C2uj6wK4.cjs +0 -163
  694. package/dist/src/litellm-CN1ScAwZ.js +0 -147
  695. package/dist/src/litellm-CTjwiKSa.js +0 -146
  696. package/dist/src/litellm-Cd4Nj5ue.js +0 -145
  697. package/dist/src/litellm-DMqLpEBg.cjs +0 -146
  698. package/dist/src/live-Bt50-InO.cjs +0 -891
  699. package/dist/src/live-Bu2ujadG.js +0 -2
  700. package/dist/src/live-C3M52i3p.js +0 -891
  701. package/dist/src/live-CRfHMO3B.js +0 -892
  702. package/dist/src/live-iCIpO14m.js +0 -891
  703. package/dist/src/luma-ray-B8eHC4c8.cjs +0 -309
  704. package/dist/src/luma-ray-BIvkPw0s.js +0 -309
  705. package/dist/src/luma-ray-ChsiBxMA.js +0 -309
  706. package/dist/src/luma-ray-DAvKrInz.js +0 -310
  707. package/dist/src/mantleChat-5YeEXAZI.js +0 -154
  708. package/dist/src/mantleChat-D--mW3hl.js +0 -154
  709. package/dist/src/mantleChat-DHP3siNg.cjs +0 -154
  710. package/dist/src/mantleChat-wBfAX46o.js +0 -155
  711. package/dist/src/mcp-49jlEcTQ.js +0 -85
  712. package/dist/src/mcp-__PIz81I.js +0 -84
  713. package/dist/src/memoryPoisoning-Chxge74M.js +0 -111
  714. package/dist/src/memoryPoisoning-QH6NCUVE.js +0 -112
  715. package/dist/src/memoryPoisoning-tCxZt8Nz.js +0 -111
  716. package/dist/src/memoryPoisoning-ymrRItRR.cjs +0 -111
  717. package/dist/src/messages-BE_eZ7NA.js +0 -1127
  718. package/dist/src/messages-Bo638ShI.js +0 -1127
  719. package/dist/src/messages-BzR0-NQ1.js +0 -1128
  720. package/dist/src/messages-Cbg5Pfd0.cjs +0 -1161
  721. package/dist/src/mischievousUser-BtED2Z5Z.cjs +0 -51
  722. package/dist/src/mischievousUser-KqSOjrDJ.js +0 -51
  723. package/dist/src/mischievousUser-YMD41YNZ.js +0 -51
  724. package/dist/src/mischievousUser-zBxlNdCu.js +0 -52
  725. package/dist/src/modelslab-B8gZxtfE.js +0 -171
  726. package/dist/src/modelslab-C7bmVpTB.js +0 -170
  727. package/dist/src/modelslab-CCeQCTZg.js +0 -170
  728. package/dist/src/modelslab-CvBJGv7y.cjs +0 -170
  729. package/dist/src/nova-reel-BOEWsEKA.cjs +0 -272
  730. package/dist/src/nova-reel-BtyBXq5M.js +0 -273
  731. package/dist/src/nova-reel-C1eF3-EE.js +0 -272
  732. package/dist/src/nova-reel-YhgrxEKT.js +0 -272
  733. package/dist/src/nova-sonic-BaERP20w.js +0 -374
  734. package/dist/src/nova-sonic-CxgM3O-P.cjs +0 -374
  735. package/dist/src/nova-sonic-DJS22fC3.js +0 -375
  736. package/dist/src/nova-sonic-DwvwHyRX.js +0 -374
  737. package/dist/src/novita-D1l2tG0O.js +0 -116
  738. package/dist/src/novita-D84VM3v0.cjs +0 -115
  739. package/dist/src/novita-DGInvwfR.js +0 -115
  740. package/dist/src/novita-vXHkXhTg.js +0 -114
  741. package/dist/src/nscale-CU0Ldr00.js +0 -209
  742. package/dist/src/nscale-ClfKlQeM.cjs +0 -209
  743. package/dist/src/nscale-DxaVnF0x.js +0 -210
  744. package/dist/src/nscale-_poVxIrO.js +0 -207
  745. package/dist/src/openaiResponses-CJGsneNj.js +0 -317
  746. package/dist/src/openaiResponses-CKSl5Sdg.js +0 -319
  747. package/dist/src/openaiResponses-DC8rzFUc.js +0 -318
  748. package/dist/src/openaiResponses-rMHIA3dV.cjs +0 -318
  749. package/dist/src/openclaw-CCbvsNFW.cjs +0 -1344
  750. package/dist/src/openclaw-CZw1lCDZ.js +0 -1336
  751. package/dist/src/openclaw-ChFLgC-a.js +0 -1338
  752. package/dist/src/openclaw-DrPMoRrH.js +0 -1337
  753. package/dist/src/opencode-sdk-2EfIDhFp.cjs +0 -1018
  754. package/dist/src/opencode-sdk-CHw3F845.js +0 -1013
  755. package/dist/src/opencode-sdk-D0EGGxvy.js +0 -1012
  756. package/dist/src/opencode-sdk-jSS7NI2Q.js +0 -1012
  757. package/dist/src/openinterpreter-CY622xHC.js +0 -431
  758. package/dist/src/openinterpreter-DATyhuhP.js +0 -432
  759. package/dist/src/openinterpreter-DDUMyIf1.js +0 -431
  760. package/dist/src/openinterpreter-s5Tz6Lid.cjs +0 -435
  761. package/dist/src/pricing-C498twd2.js +0 -976
  762. package/dist/src/pricing-DBd_bMIh.cjs +0 -999
  763. package/dist/src/pricing-ISrMCskv.js +0 -976
  764. package/dist/src/promptfoo-6p0WG2ih.js +0 -286
  765. package/dist/src/promptfoo-BQRfgNs_.js +0 -285
  766. package/dist/src/promptfoo-Bqo88z20.cjs +0 -341
  767. package/dist/src/promptfoo-Cq5AVI3I.js +0 -286
  768. package/dist/src/providers-ATbeMX5E.cjs +0 -24180
  769. package/dist/src/providers-B7F1FiNI.js +0 -2
  770. package/dist/src/providers-DazIw8mI.js +0 -24039
  771. package/dist/src/providers-QvGlNAD0.js +0 -24437
  772. package/dist/src/quiverai-BPlNcLkV.cjs +0 -325
  773. package/dist/src/quiverai-By99-YmJ.js +0 -325
  774. package/dist/src/quiverai-CmmByDqo.js +0 -325
  775. package/dist/src/quiverai-HZkoJJqw.js +0 -326
  776. package/dist/src/registry-BLTnTfjT.cjs +0 -131
  777. package/dist/src/registry-BMC-COHT.js +0 -132
  778. package/dist/src/registry-C8UZtmBp.js +0 -131
  779. package/dist/src/registry-CfwEUYXG.js +0 -131
  780. package/dist/src/remoteGeneration-Bgg6kC5V.js +0 -370
  781. package/dist/src/remoteGeneration-CKJFE960.cjs +0 -482
  782. package/dist/src/remoteGeneration-DaxwJDKv.js +0 -370
  783. package/dist/src/remoteGeneration-vHMGVKr1.js +0 -371
  784. package/dist/src/responses-BvnyIHrq.js +0 -1099
  785. package/dist/src/responses-CERAgvf-.js +0 -1098
  786. package/dist/src/responses-CwgjeuOg.cjs +0 -1109
  787. package/dist/src/runtimeTransform-3RF9Avyg.cjs +0 -161
  788. package/dist/src/runtimeTransform-C51Z5j4H.js +0 -156
  789. package/dist/src/runtimeTransform-CKE_Iyv8.js +0 -157
  790. package/dist/src/runtimeTransform-Cs11xJpj.js +0 -156
  791. package/dist/src/sagemaker-BJ4dOCxE.js +0 -713
  792. package/dist/src/sagemaker-Bt3JUAWq.js +0 -714
  793. package/dist/src/sagemaker-QVKyLgLF.cjs +0 -716
  794. package/dist/src/sagemaker-l3IB8iTs.js +0 -713
  795. package/dist/src/scanner-DGjIiTZ1.js +0 -1971
  796. package/dist/src/server-BNaLZH6u.js +0 -63
  797. package/dist/src/server-CBMuQul3.js +0 -189
  798. package/dist/src/server-CMKKIzKv.cjs +0 -74
  799. package/dist/src/server-CN2IS9Cs.js +0 -155
  800. package/dist/src/server-ua4cqbjC.js +0 -2
  801. package/dist/src/session-CQIjq_HZ.cjs +0 -69
  802. package/dist/src/session-FTlg8sEu.js +0 -62
  803. package/dist/src/session-GBQXtPWp.js +0 -63
  804. package/dist/src/session-_0-rTdwY.js +0 -62
  805. package/dist/src/shared-BtuO-OAM.js +0 -1927
  806. package/dist/src/shared-Ci78wTZX.js +0 -1928
  807. package/dist/src/shared-Dd3rimdM.js +0 -1929
  808. package/dist/src/shared-KAjFxRre.cjs +0 -2185
  809. package/dist/src/simulatedUser--OjW6xCx.js +0 -247
  810. package/dist/src/simulatedUser-D0-slRye.cjs +0 -252
  811. package/dist/src/simulatedUser-D1Kh_cwA.js +0 -248
  812. package/dist/src/slack-BX0FUevO.js +0 -144
  813. package/dist/src/slack-CcwXFiJE.cjs +0 -144
  814. package/dist/src/slack-CnfqKfXp.js +0 -145
  815. package/dist/src/strategies-2ntcK4tN.js +0 -2248
  816. package/dist/src/strategies-DZIW7Zcs.js +0 -2385
  817. package/dist/src/strategies-YRDF2j-N.cjs +0 -2420
  818. package/dist/src/telemetry-CL8CVA4p.js +0 -205
  819. package/dist/src/telemetry-Cix90qqz.js +0 -201
  820. package/dist/src/telemetry-CkmlqTjV.js +0 -205
  821. package/dist/src/telemetry-DQH7l5bb.js +0 -2
  822. package/dist/src/telemetry-DUCXc1-r.cjs +0 -216
  823. package/dist/src/togetherai-3yUt5Ck7.js +0 -49
  824. package/dist/src/togetherai-BJUByLwt.cjs +0 -49
  825. package/dist/src/togetherai-Bh6KZkmd.js +0 -50
  826. package/dist/src/togetherai-DDl9yk3b.js +0 -48
  827. package/dist/src/traceContext-BiJVppjT.cjs +0 -1032
  828. package/dist/src/traceContext-CKKluULP.js +0 -1016
  829. package/dist/src/traceContext-D3Sk8fO-.js +0 -1015
  830. package/dist/src/transcription-BoVQKIVz.js +0 -175
  831. package/dist/src/transcription-Bs5u8780.js +0 -175
  832. package/dist/src/transcription-Dl654d1p.cjs +0 -178
  833. package/dist/src/transcription-VEvhM1Fu.js +0 -176
  834. package/dist/src/transform-BAbkGpvl.js +0 -125
  835. package/dist/src/transform-C_j4hUH8.js +0 -126
  836. package/dist/src/transform-Da86hs4M.cjs +0 -154
  837. package/dist/src/transform-_i1raQNo.js +0 -125
  838. package/dist/src/typesafe-CKXBeh82.cjs +0 -253
  839. package/dist/src/typesafe-D3VaGeNm.js +0 -253
  840. package/dist/src/typesafe-DYM-SFNH.js +0 -253
  841. package/dist/src/typesafe-LXmSbw8F.js +0 -254
  842. package/dist/src/util-41rb0XHX.js +0 -743
  843. package/dist/src/util-B0WlHYFM.js +0 -2049
  844. package/dist/src/util-BD1R_PsR.js +0 -1770
  845. package/dist/src/util-BLbM_ItC.js +0 -395
  846. package/dist/src/util-C72P0iIa.js +0 -352
  847. package/dist/src/util-C7MpGlJ6.cjs +0 -2305
  848. package/dist/src/util-CCtqagKc.js +0 -286
  849. package/dist/src/util-CM2Ec_uY.js +0 -2051
  850. package/dist/src/util-CSSAWO8v.js +0 -351
  851. package/dist/src/util-CkQc-fx3.cjs +0 -934
  852. package/dist/src/util-DcOlFTFa.cjs +0 -478
  853. package/dist/src/util-DluGTGVB.js +0 -395
  854. package/dist/src/util-PfQ60chz.js +0 -743
  855. package/dist/src/util-f14-CPij.js +0 -396
  856. package/dist/src/util-l9pXX3Eu.cjs +0 -357
  857. package/dist/src/util-o_BnlYqS.js +0 -2050
  858. package/dist/src/vertex-B_-gQgzc.js +0 -2
  859. package/dist/src/vertex-CU4gqoYa.js +0 -854
  860. package/dist/src/vertex-DZ42UHb3.js +0 -848
  861. package/dist/src/vertex-DsYQJA0T.cjs +0 -871
  862. package/dist/src/vertexLive-BvIaHKGB.js +0 -55
  863. package/dist/src/vertexLive-CTULqNx7.js +0 -56
  864. package/dist/src/vertexLive-DjJ_YQBe.cjs +0 -55
  865. package/dist/src/vertexLive-STUGcteB.js +0 -55
  866. package/dist/src/video-CAQGmtEV.js +0 -703
  867. package/dist/src/video-_FSTWlwX.js +0 -702
  868. package/dist/src/video-aBPJPWdt.js +0 -702
  869. package/dist/src/video-gDjkWv5m.cjs +0 -706
@@ -1,3187 +0,0 @@
1
- const require_rolldown_runtime = require("./rolldown-runtime-C0BPl7ul.cjs");
2
- const require_logger = require("./logger-DXUHJBHJ.cjs");
3
- const require_invariant = require("./invariant-C2JOzzO2.cjs");
4
- const require_fetch = require("./fetch-DYte6oQX.cjs");
5
- const require_tables = require("./tables-CNr3LyS7.cjs");
6
- const require_time = require("./time-CY7bOv6D.cjs");
7
- const require_esm = require("./esm-DITT-qdS.cjs");
8
- const require_file = require("./file-CPfaem7U.cjs");
9
- const require_render = require("./render-j-UNQw7s.cjs");
10
- const require_genaiTracer = require("./genaiTracer-C32xFIB8.cjs");
11
- const require_createHash = require("./createHash-sm6Q6LWg.cjs");
12
- require("./blobs-CwaJ7l1g.cjs");
13
- const require_extractor = require("./extractor-q61MtrkD.cjs");
14
- const require_packageImportErrors = require("./packageImportErrors-B2JxLURu.cjs");
15
- const require_tokenUsageUtils = require("./tokenUsageUtils-DMF9uhT9.cjs");
16
- let fs_promises = require("fs/promises");
17
- fs_promises = require_rolldown_runtime.__toESM(fs_promises, 1);
18
- let path = require("path");
19
- path = require_rolldown_runtime.__toESM(path, 1);
20
- let fs = require("fs");
21
- fs = require_rolldown_runtime.__toESM(fs, 1);
22
- let os = require("os");
23
- os = require_rolldown_runtime.__toESM(os, 1);
24
- let js_yaml = require("js-yaml");
25
- js_yaml = require_rolldown_runtime.__toESM(js_yaml, 1);
26
- let fast_deep_equal = require("fast-deep-equal");
27
- fast_deep_equal = require_rolldown_runtime.__toESM(fast_deep_equal, 1);
28
- let dedent = require("dedent");
29
- dedent = require_rolldown_runtime.__toESM(dedent, 1);
30
- let node_util = require("node:util");
31
- let crypto$1 = require("crypto");
32
- let drizzle_orm = require("drizzle-orm");
33
- let fast_xml_parser = require("fast-xml-parser");
34
- let lru_cache = require("lru-cache");
35
- let csv_stringify_sync = require("csv-stringify/sync");
36
- //#region src/providers/constants.ts
37
- const FILE_METADATA_KEY = "_promptfooFileMetadata";
38
- /**
39
- * Identifier for manual user ratings in componentResults.
40
- * Used to distinguish human ratings from automated assertions.
41
- */
42
- const HUMAN_ASSERTION_TYPE = "human";
43
- //#endregion
44
- //#region src/util/providerRef.ts
45
- const PROVIDER_OPTION_KEYS = /* @__PURE__ */ new Set([
46
- "id",
47
- "label",
48
- "config",
49
- "prompts",
50
- "transform",
51
- "delay",
52
- "env",
53
- "inputs"
54
- ]);
55
- /** Returns true if the value is a non-empty string suitable as a provider identifier. */
56
- function isValidProviderId(id) {
57
- return typeof id === "string" && id !== "";
58
- }
59
- function getProviderLabel(provider) {
60
- if ((typeof provider === "object" || typeof provider === "function") && provider !== null && "label" in provider && typeof provider.label === "string") return provider.label;
61
- }
62
- /**
63
- * Resolves relative file paths in provider IDs to absolute paths for consistent matching.
64
- * Handles file://, exec:, python:, golang: prefixes and bare .js/.ts/.mjs paths.
65
- */
66
- function canonicalizeProviderId(id) {
67
- if (id.startsWith("file://")) {
68
- const filePath = id.slice(7);
69
- return path.default.isAbsolute(filePath) ? id : `file://${path.default.resolve(filePath)}`;
70
- }
71
- for (const prefix of [
72
- "exec:",
73
- "python:",
74
- "golang:"
75
- ]) if (id.startsWith(prefix)) {
76
- const filePath = id.slice(prefix.length);
77
- if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.default.resolve(filePath)}`;
78
- return id;
79
- }
80
- if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.default.resolve(id)}`;
81
- return id;
82
- }
83
- /**
84
- * Returns true for provider refs that should be expanded from YAML/JSON config files.
85
- */
86
- function isProviderConfigFileReference(providerPath) {
87
- return providerPath.startsWith("file://") && (providerPath.endsWith(".yaml") || providerPath.endsWith(".yml") || providerPath.endsWith(".json"));
88
- }
89
- /**
90
- * Reads a provider config file and normalizes single-provider and multi-provider files.
91
- * An explicit base path also scopes nested configuration files; otherwise their
92
- * caller scope is retained.
93
- * Returns a `wasArray` flag so callers can detect multi-provider files that require
94
- * `loadApiProviders` instead of `loadApiProvider`.
95
- */
96
- function readProviderConfigFile(providerPath, basePath) {
97
- const relativePath = providerPath.slice(7);
98
- const resolvedPath = path.default.isAbsolute(relativePath) ? relativePath : path.default.join(basePath || process.cwd(), relativePath);
99
- let rawContent;
100
- try {
101
- rawContent = require_logger.loadYaml(fs.default.readFileSync(resolvedPath, "utf8"));
102
- } catch (err) {
103
- throw new Error(`Failed to load provider config ${relativePath}: ${err instanceof Error ? err.message : err}`);
104
- }
105
- const fileContent = require_file.maybeLoadConfigFromExternalFile(rawContent, void 0, basePath);
106
- require_invariant.invariant(fileContent, `Provider config ${relativePath} is undefined`);
107
- return {
108
- configs: [fileContent].flat(),
109
- relativePath,
110
- wasArray: Array.isArray(fileContent)
111
- };
112
- }
113
- /**
114
- * Loads provider config objects from a file-backed provider reference.
115
- */
116
- function loadProviderConfigsFromFile(providerPath, basePath) {
117
- return readProviderConfigFile(providerPath, basePath).configs;
118
- }
119
- /**
120
- * Pure, synchronous classifier that converts every supported provider reference shape
121
- * into a discriminated descriptor. Does not read files or instantiate providers.
122
- */
123
- function normalizeProviderRef(provider, options = {}) {
124
- const { index } = options;
125
- if (typeof provider === "string") {
126
- if (!isValidProviderId(provider)) return {
127
- kind: "unknown",
128
- id: index === void 0 ? "unknown" : `unknown-${index}`
129
- };
130
- if (isProviderConfigFileReference(provider)) return {
131
- kind: "file",
132
- id: provider,
133
- loadProviderPath: provider
134
- };
135
- return {
136
- kind: "named",
137
- id: provider,
138
- loadProviderPath: provider
139
- };
140
- }
141
- if (typeof provider === "function") {
142
- const label = getProviderLabel(provider);
143
- return {
144
- kind: "function",
145
- id: label ?? (index === void 0 ? "custom-function" : `custom-function-${index}`),
146
- label
147
- };
148
- }
149
- if (typeof provider === "object" && provider !== null && !Array.isArray(provider)) {
150
- const providerId = provider.id;
151
- const label = getProviderLabel(provider);
152
- if (isValidProviderId(providerId)) return {
153
- kind: "options",
154
- id: providerId,
155
- label,
156
- loadOptions: provider,
157
- loadProviderPath: providerId
158
- };
159
- const keys = Object.keys(provider);
160
- if (keys.length === 1 && !PROVIDER_OPTION_KEYS.has(keys[0])) {
161
- const originalId = keys[0];
162
- const providerObject = provider[originalId];
163
- if (typeof providerObject === "object" && providerObject !== null && !Array.isArray(providerObject) && isValidProviderId(originalId)) {
164
- const id = isValidProviderId(providerObject.id) ? providerObject.id : originalId;
165
- return {
166
- kind: "map",
167
- id,
168
- label: getProviderLabel(providerObject),
169
- loadOptions: {
170
- ...providerObject,
171
- id
172
- },
173
- loadProviderPath: originalId
174
- };
175
- }
176
- }
177
- if (isValidProviderId(label)) return {
178
- kind: "unknown",
179
- id: label,
180
- label
181
- };
182
- }
183
- return {
184
- kind: "unknown",
185
- id: index === void 0 ? "unknown" : `unknown-${index}`
186
- };
187
- }
188
- //#endregion
189
- //#region src/util/provider.ts
190
- function providerToIdentifier(provider) {
191
- if (!provider) return;
192
- if (typeof provider === "string") return canonicalizeProviderId(provider);
193
- const { label } = normalizeProviderRef(provider);
194
- if (label) return label;
195
- if (require_logger.isApiProvider(provider)) return canonicalizeProviderId(provider.id());
196
- if (require_logger.isProviderOptions(provider)) {
197
- if (provider.id) return canonicalizeProviderId(provider.id);
198
- return;
199
- }
200
- if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
201
- }
202
- /**
203
- * Gets the identifier string for a provider (label or id).
204
- */
205
- function getProviderIdentifier(provider) {
206
- return provider.label || provider.id();
207
- }
208
- /**
209
- * Gets a descriptive identifier string for a provider, showing both label and ID when both exist.
210
- * Useful for error messages to help users debug provider reference issues.
211
- */
212
- function getProviderDescription(provider) {
213
- const label = provider.label;
214
- const id = provider.id();
215
- if (label && label !== id) return `${label} (${id})`;
216
- return id;
217
- }
218
- function sanitizeProviderIdForLog(providerId) {
219
- const descriptionMatch = providerId.match(/^(\S+)(\s+\(.+\))$/);
220
- if (descriptionMatch) {
221
- const [, id, description] = descriptionMatch;
222
- return `${sanitizeProviderIdForLog(id)}${description}`;
223
- }
224
- return providerId.includes("://") || providerId.startsWith("/") ? require_logger.sanitizeUrl(providerId) : providerId;
225
- }
226
- /**
227
- * Checks if a provider reference matches a given provider.
228
- * Supports exact matching and wildcard patterns.
229
- */
230
- function doesProviderRefMatch(ref, provider) {
231
- const label = provider.label;
232
- const id = provider.id();
233
- const canonicalRef = canonicalizeProviderId(ref);
234
- const canonicalId = canonicalizeProviderId(id);
235
- if (label && label === ref) return true;
236
- if (id === ref || canonicalId === canonicalRef) return true;
237
- if (ref.endsWith("*")) {
238
- const prefix = ref.slice(0, -1);
239
- if (label?.startsWith(prefix) || id.startsWith(prefix) || canonicalId.startsWith(prefix)) return true;
240
- }
241
- if (label?.startsWith(`${ref}:`) || id.startsWith(`${ref}:`) || canonicalId.startsWith(`${ref}:`)) return true;
242
- return false;
243
- }
244
- /**
245
- * Checks if a provider is allowed based on a list of allowed references.
246
- */
247
- function isProviderAllowed(provider, allowedProviders) {
248
- if (!Array.isArray(allowedProviders)) return true;
249
- if (allowedProviders.length === 0) return false;
250
- return allowedProviders.some((ref) => doesProviderRefMatch(ref, provider));
251
- }
252
- /**
253
- * Detects if a provider uses OpenAI models.
254
- * This includes direct OpenAI providers and Azure OpenAI.
255
- */
256
- function isOpenAiProvider(providerId) {
257
- const lowerProviderId = providerId.toLowerCase();
258
- if (lowerProviderId.startsWith("openai:")) return true;
259
- if (lowerProviderId.startsWith("azureopenai:")) return true;
260
- if (lowerProviderId.startsWith("azure:")) {
261
- if ([
262
- "gpt",
263
- "openai",
264
- "davinci",
265
- "curie",
266
- "babbage",
267
- "ada",
268
- "text-embedding",
269
- "whisper",
270
- "dall-e",
271
- "tts"
272
- ].some((indicator) => lowerProviderId.includes(indicator))) return true;
273
- }
274
- return false;
275
- }
276
- /**
277
- * Detects if a provider uses Anthropic/Claude models.
278
- * This includes direct Anthropic providers, Bedrock with Claude, and Vertex with Claude.
279
- */
280
- function isAnthropicProvider(providerId) {
281
- const lowerProviderId = providerId.toLowerCase();
282
- if (lowerProviderId.startsWith("anthropic:")) return true;
283
- if (lowerProviderId.startsWith("bedrock:")) {
284
- if (lowerProviderId.includes("claude") || lowerProviderId.includes("anthropic")) return true;
285
- }
286
- if (lowerProviderId.startsWith("vertex:")) {
287
- if (lowerProviderId.includes("claude")) return true;
288
- }
289
- return false;
290
- }
291
- const KNOWN_ENV_VARS = {
292
- openai: "OPENAI_API_KEY",
293
- anthropic: "ANTHROPIC_API_KEY",
294
- google: "GOOGLE_API_KEY",
295
- mistral: "MISTRAL_API_KEY",
296
- cohere: "COHERE_API_KEY",
297
- replicate: "REPLICATE_API_TOKEN",
298
- voyage: "VOYAGE_API_KEY",
299
- ai21: "AI21_API_KEY",
300
- xai: "XAI_API_KEY",
301
- groq: "GROQ_API_KEY",
302
- deepseek: "DEEPSEEK_API_KEY",
303
- meta: "MODEL_API_KEY",
304
- moonshot: "MOONSHOT_API_KEY",
305
- perplexity: "PERPLEXITY_API_KEY",
306
- hyperbolic: "HYPERBOLIC_API_KEY",
307
- cerebras: "CEREBRAS_API_KEY",
308
- togetherai: "TOGETHER_API_KEY",
309
- fal: "FAL_KEY",
310
- huggingface: "HF_TOKEN",
311
- "cloudflare-ai": "CLOUDFLARE_API_KEY"
312
- };
313
- function getDefaultEnvVar(providerId) {
314
- const prefix = providerId.split(":")[0];
315
- return KNOWN_ENV_VARS[prefix] || `${prefix.toUpperCase()}_API_KEY`;
316
- }
317
- /**
318
- * Pre-checks providers for missing API keys before evaluation starts.
319
- * Assumes getApiKey() is side-effect free (no network calls or token refresh).
320
- * Returns stable provider IDs by default; descriptions are for user-facing diagnostics only.
321
- */
322
- function checkProviderApiKeys(providers, options = {}) {
323
- const missingApiKeys = /* @__PURE__ */ new Map();
324
- for (const provider of providers) {
325
- const p = provider;
326
- if (typeof p.getApiKey !== "function") continue;
327
- if (provider.id().startsWith("azure:")) continue;
328
- const requiresKey = typeof p.requiresApiKey === "function" ? p.requiresApiKey() : p.config?.apiKeyRequired !== false;
329
- let apiKey;
330
- try {
331
- apiKey = p.getApiKey();
332
- } catch {
333
- apiKey = void 0;
334
- }
335
- if (requiresKey && !apiKey) {
336
- const envVar = p.config?.apiKeyEnvar || getDefaultEnvVar(provider.id());
337
- const providerIdentifier = options.useDescriptions ? getProviderDescription(provider) : provider.id();
338
- if (!missingApiKeys.has(envVar)) missingApiKeys.set(envVar, []);
339
- missingApiKeys.get(envVar).push(providerIdentifier);
340
- }
341
- }
342
- return missingApiKeys;
343
- }
344
- /**
345
- * Detects if a provider uses Google models.
346
- * This includes direct Google/Vertex providers with Gemini and other Google models.
347
- * Note: Vertex with Claude models is NOT counted as Google (it's Anthropic).
348
- */
349
- function isGoogleProvider(providerId) {
350
- const lowerProviderId = providerId.toLowerCase();
351
- if (lowerProviderId.startsWith("google:")) return true;
352
- if (lowerProviderId.startsWith("vertex:")) {
353
- if (!lowerProviderId.includes("claude")) return true;
354
- }
355
- return false;
356
- }
357
- //#endregion
358
- //#region src/util/comparison.ts
359
- /**
360
- * Explicit runtime variable names that don't follow the underscore convention.
361
- * These are added during evaluation but aren't part of the original test definition.
362
- *
363
- * - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo)
364
- *
365
- * Note: Variables starting with underscore (e.g., _conversation) are automatically
366
- * treated as runtime variables and filtered out.
367
- */
368
- const EXPLICIT_RUNTIME_VAR_KEYS = ["sessionId"];
369
- /**
370
- * Checks if a variable key is a runtime-only variable that should be filtered
371
- * when comparing test cases.
372
- *
373
- * Runtime variables are identified by:
374
- * 1. Starting with underscore (_) - convention for internal/runtime vars
375
- * 2. Being in the explicit runtime var list (for legacy vars like sessionId)
376
- */
377
- function isRuntimeVar(key) {
378
- return key.startsWith("_") || EXPLICIT_RUNTIME_VAR_KEYS.includes(key);
379
- }
380
- /**
381
- * Filters out runtime-only variables that are added during evaluation
382
- * but aren't part of the original test definition.
383
- *
384
- * This is used when comparing test cases to determine if a result
385
- * corresponds to a particular test, regardless of runtime state.
386
- *
387
- * Runtime variables are identified by:
388
- * - Starting with underscore (e.g., _conversation, _metadata)
389
- * - Being in the explicit list (e.g., sessionId for backward compatibility)
390
- */
391
- function filterRuntimeVars(vars) {
392
- if (!vars || typeof vars !== "object" || Array.isArray(vars)) return vars;
393
- const filtered = {};
394
- for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
395
- return filtered;
396
- }
397
- /**
398
- * Extracts only runtime variables from a vars object.
399
- * This is the inverse of filterRuntimeVars.
400
- *
401
- * Used to restore runtime state when re-running filtered tests.
402
- */
403
- function extractRuntimeVars(vars) {
404
- if (!vars || typeof vars !== "object" || Array.isArray(vars)) return;
405
- const extracted = {};
406
- for (const [key, value] of Object.entries(vars)) if (isRuntimeVar(key)) extracted[key] = value;
407
- return Object.keys(extracted).length > 0 ? extracted : void 0;
408
- }
409
- function varsMatch(vars1, vars2) {
410
- return (0, fast_deep_equal.default)(vars1, vars2);
411
- }
412
- /**
413
- * Generate a unique key for a test case for deduplication purposes.
414
- * Excludes runtime variables and includes strategyId to distinguish tests
415
- * with the same prompt but different strategies.
416
- *
417
- * @param testCase - The test case to generate a key for
418
- * @returns A JSON string that uniquely identifies the test case
419
- */
420
- function getTestCaseDeduplicationKey(testCase) {
421
- const filteredVars = filterRuntimeVars(testCase.vars);
422
- const strategyId = testCase.metadata?.strategyId || "none";
423
- return JSON.stringify({
424
- vars: filteredVars,
425
- strategyId
426
- });
427
- }
428
- /**
429
- * Deduplicates an array of test cases based on their vars and strategyId.
430
- * Tests with the same vars but different strategies are considered different.
431
- * Runtime variables (like _conversation, sessionId) are filtered out before comparison.
432
- *
433
- * @param tests - Array of test cases to deduplicate
434
- * @returns Deduplicated array of test cases
435
- */
436
- function deduplicateTestCases(tests) {
437
- const seen = /* @__PURE__ */ new Set();
438
- return tests.filter((test) => {
439
- const key = getTestCaseDeduplicationKey(test);
440
- if (seen.has(key)) return false;
441
- seen.add(key);
442
- return true;
443
- });
444
- }
445
- function resultIsForTestCase(result, testCase) {
446
- const testProviderId = testCase.provider ? providerToIdentifier(testCase.provider) : void 0;
447
- const resultProviderId = providerToIdentifier(result.provider);
448
- const providersMatch = !testProviderId || !resultProviderId || testProviderId === resultProviderId;
449
- const resultVars = filterRuntimeVars(result.vars);
450
- const testVars = filterRuntimeVars(testCase.vars);
451
- const doVarsMatch = varsMatch(testVars, resultVars);
452
- const isMatch = doVarsMatch && providersMatch;
453
- if (!isMatch) {
454
- const varKeys = testVars ? Object.keys(testVars).join(", ") : "none";
455
- require_logger.logger.debug(`[resultIsForTestCase] No match: vars=${doVarsMatch}, providers=${providersMatch}`, {
456
- testProvider: testProviderId || "none",
457
- resultProvider: resultProviderId || "none",
458
- testVarKeys: varKeys
459
- });
460
- }
461
- return isMatch;
462
- }
463
- //#endregion
464
- //#region src/util/env.ts
465
- /**
466
- * Load environment variables from .env file(s).
467
- * @param envPath - Single path, array of paths, or undefined for default .env loading.
468
- * When paths are explicitly specified, all files must exist or an error is thrown.
469
- * When multiple files are provided, later files override values from earlier files.
470
- */
471
- function setupEnv(envPath, options = {}) {
472
- const paths = (Array.isArray(envPath) ? envPath : [envPath ?? ""]).flatMap((p) => p.split(",").map((s) => s.trim())).filter(Boolean);
473
- if (paths.length > 0) {
474
- for (const p of paths) if (!fs.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
475
- if (paths.length === 1) require_logger.logger.info(`Loading environment variables from ${paths[0]}`);
476
- else require_logger.logger.info(`Loading environment variables from: ${paths.join(", ")}`);
477
- }
478
- const previousEnv = { ...options.processEnv };
479
- require_logger.loadEnvFiles(paths.length > 0 ? paths : void 0, {
480
- ...paths.length > 0 && { override: true },
481
- ...options.processEnv && { processEnv: options.processEnv }
482
- });
483
- if (options.processEnv && paths.length === 0) {
484
- for (const key of Object.keys(options.processEnv)) if (!Object.prototype.hasOwnProperty.call(previousEnv, key) && process.env[key] !== void 0) delete options.processEnv[key];
485
- }
486
- if (options.refreshConfigDirectory) require_logger.refreshConfigDirectoryPathFromEnv();
487
- }
488
- //#endregion
489
- //#region src/googleSheets.ts
490
- var GoogleSheetAuthenticationRequiredError = class extends Error {
491
- constructor(message) {
492
- super(message);
493
- this.name = "GoogleSheetAuthenticationRequiredError";
494
- }
495
- };
496
- async function loadGoogleSheetsApi() {
497
- try {
498
- return await import("@googleapis/sheets");
499
- } catch (error) {
500
- if (require_packageImportErrors.isMissingPackageImportError(error, "@googleapis/sheets")) throw new Error("The @googleapis/sheets package is required for authenticated Google Sheets access. Install it with: npm install @googleapis/sheets");
501
- throw error;
502
- }
503
- }
504
- async function checkGoogleSheetAccess(url) {
505
- try {
506
- const response = await require_fetch.fetchWithProxy(url);
507
- if (response.ok) return {
508
- public: true,
509
- status: response.status
510
- };
511
- else return {
512
- public: false,
513
- status: response.status
514
- };
515
- } catch (error) {
516
- require_logger.logger.error(`Error checking sheet access: ${error}`);
517
- return { public: false };
518
- }
519
- }
520
- function isHtmlExportResponse(response, body) {
521
- const contentType = response.headers?.get("content-type")?.toLowerCase() ?? "";
522
- if (contentType.includes("text/html")) return true;
523
- if (contentType.includes("text/csv") || contentType.includes("application/csv")) return false;
524
- const prefix = body.trimStart().slice(0, 32).toLowerCase();
525
- return prefix.startsWith("<!doctype html") || prefix.startsWith("<html");
526
- }
527
- /**
528
- * Returns the sheet id (`gid`) a Google Sheets URL selects. The browser keeps it in the fragment
529
- * (`/edit#gid=123`); newer URLs also repeat it in the query (`/edit?gid=123#gid=123`). It stays a
530
- * string so that sheet id 0 is still truthy.
531
- */
532
- function getSheetGid(url) {
533
- const { searchParams, hash } = new URL(url);
534
- return searchParams.get("gid") || new URLSearchParams(hash.slice(1)).get("gid") || void 0;
535
- }
536
- async function fetchCsvFromGoogleSheetUnauthenticated(url) {
537
- const { parse: parseCsv } = await import("csv-parse/sync");
538
- const gid = getSheetGid(url);
539
- const csvUrl = `${url.replace(/\/edit.*$/, "/export")}?format=csv${gid ? `&gid=${gid}` : ""}`;
540
- const response = await require_fetch.fetchWithProxy(csvUrl);
541
- if (response.status !== 200) throw new Error(`Failed to fetch CSV from Google Sheets URL: ${url}`);
542
- const csvData = await response.text();
543
- if (isHtmlExportResponse(response, csvData)) throw new GoogleSheetAuthenticationRequiredError(`Failed to fetch CSV from Google Sheets URL: ${url}`);
544
- return parseCsv(csvData, { columns: true });
545
- }
546
- async function fetchCsvFromGoogleSheetAuthenticated(url) {
547
- const { sheets: googleSheets, auth: googleAuth } = await loadGoogleSheetsApi();
548
- const auth = new googleAuth.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/spreadsheets.readonly"] });
549
- const sheets = googleSheets("v4");
550
- const match = url.match(/\/d\/([^/]+)/);
551
- if (!match) throw new Error(`Invalid Google Sheets URL: ${url}`);
552
- const spreadsheetId = match[1];
553
- let range;
554
- const gid = getSheetGid(url);
555
- if (gid) {
556
- const sheet = (await sheets.spreadsheets.get({
557
- spreadsheetId,
558
- auth
559
- })).data.sheets?.find((sheet) => sheet.properties?.sheetId === Number(gid));
560
- if (!sheet || !sheet.properties?.title) throw new Error(`Sheet not found for gid: ${gid}`);
561
- range = sheet.properties.title;
562
- } else {
563
- const firstSheet = (await sheets.spreadsheets.get({
564
- spreadsheetId,
565
- auth
566
- })).data.sheets?.[0];
567
- if (!firstSheet || !firstSheet.properties?.title) throw new Error(`No sheets found in spreadsheet`);
568
- range = firstSheet.properties.title;
569
- }
570
- const rows = (await sheets.spreadsheets.values.get({
571
- spreadsheetId,
572
- range,
573
- auth
574
- })).data.values;
575
- if (!rows?.length) throw new Error(`No data found in Google Sheets URL: ${url}`);
576
- const headers = rows[0];
577
- return rows.slice(1).map((row) => {
578
- const csvRow = {};
579
- headers.forEach((header, index) => {
580
- csvRow[header] = row[index] ?? "";
581
- });
582
- return csvRow;
583
- });
584
- }
585
- async function fetchCsvFromGoogleSheet(url) {
586
- const access = await checkGoogleSheetAccess(url);
587
- const { public: isPublic } = access;
588
- require_logger.logger.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
589
- if (isPublic) try {
590
- return await fetchCsvFromGoogleSheetUnauthenticated(url);
591
- } catch (error) {
592
- if (error instanceof GoogleSheetAuthenticationRequiredError) return fetchCsvFromGoogleSheetAuthenticated(url);
593
- throw error;
594
- }
595
- if (!("status" in access)) try {
596
- return await fetchCsvFromGoogleSheetUnauthenticated(url);
597
- } catch {
598
- return fetchCsvFromGoogleSheetAuthenticated(url);
599
- }
600
- return fetchCsvFromGoogleSheetAuthenticated(url);
601
- }
602
- async function writeCsvToGoogleSheet(rows, url) {
603
- const { sheets: googleSheets, auth: googleAuth } = await loadGoogleSheetsApi();
604
- const auth = new googleAuth.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/spreadsheets"] });
605
- const sheets = googleSheets("v4");
606
- const match = url.match(/\/d\/([^/]+)/);
607
- if (!match) throw new Error(`Invalid Google Sheets URL: ${url}`);
608
- const spreadsheetId = match[1];
609
- const headers = Object.keys(rows[0]);
610
- const values = [headers, ...rows.map((row) => headers.map((header) => row[header]))];
611
- const getColumnLetter = (col) => {
612
- let letter = "";
613
- while (col > 0) {
614
- col--;
615
- letter = String.fromCharCode(65 + col % 26) + letter;
616
- col = Math.floor(col / 26);
617
- }
618
- return letter;
619
- };
620
- const numRows = values.length;
621
- const numCols = headers.length;
622
- const endColumn = getColumnLetter(numCols);
623
- let range;
624
- const gid = getSheetGid(url);
625
- if (gid) {
626
- const sheet = (await sheets.spreadsheets.get({
627
- spreadsheetId,
628
- auth
629
- })).data.sheets?.find((sheet) => sheet.properties?.sheetId === Number(gid));
630
- if (!sheet || !sheet.properties?.title) throw new Error(`Sheet not found for gid: ${gid}`);
631
- range = `${sheet.properties.title}!A1:${endColumn}${numRows}`;
632
- } else {
633
- const newSheetTitle = `Sheet${Date.now()}`;
634
- await sheets.spreadsheets.batchUpdate({
635
- spreadsheetId,
636
- auth,
637
- requestBody: { requests: [{ addSheet: { properties: { title: newSheetTitle } } }] }
638
- });
639
- range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
640
- }
641
- require_logger.logger.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
642
- await sheets.spreadsheets.values.update({
643
- spreadsheetId,
644
- range,
645
- valueInputOption: "USER_ENTERED",
646
- auth,
647
- requestBody: { values }
648
- });
649
- }
650
- //#endregion
651
- //#region src/models/prompt.ts
652
- /**
653
- * Generates a unique identifier for a prompt based on its properties.
654
- *
655
- * Priority order:
656
- * 1. If label is truthy, hash the label
657
- * 2. If id is truthy, hash the id
658
- * 3. Otherwise, hash the raw content (stringified if object)
659
- *
660
- * @param prompt - The prompt object to generate an ID for
661
- * @returns A SHA-256 hash string
662
- */
663
- function generateIdFromPrompt(prompt) {
664
- if (prompt.label) return require_createHash.sha256(prompt.label);
665
- if (prompt.id) return require_createHash.sha256(prompt.id);
666
- return require_createHash.sha256(typeof prompt.raw === "object" ? JSON.stringify(prompt.raw) : prompt.raw);
667
- }
668
- //#endregion
669
- //#region src/prompts/constants.ts
670
- const VALID_FILE_EXTENSIONS = [
671
- ".cjs",
672
- ".cts",
673
- ".j2",
674
- ".js",
675
- ".json",
676
- ".jsonl",
677
- ".md",
678
- ".mjs",
679
- ".mts",
680
- ".py",
681
- ".ts",
682
- ".txt",
683
- ".yml",
684
- ".yaml"
685
- ];
686
- //#endregion
687
- //#region src/prompts/utils.ts
688
- /**
689
- * Determines if a string is a valid file path.
690
- * @param str - The string to check.
691
- * @returns True if the string is a valid file path, false otherwise.
692
- */
693
- function maybeFilePath(str) {
694
- if (typeof str !== "string") throw new Error(`Invalid input: ${JSON.stringify(str)}`);
695
- if ([
696
- "\n",
697
- "portkey://",
698
- "langfuse://",
699
- "helicone://"
700
- ].some((substring) => str.includes(substring))) return false;
701
- return str.startsWith("file://") || VALID_FILE_EXTENSIONS.some((ext) => {
702
- const tokens = str.split(":");
703
- const pathOrFunctionToken = tokens[tokens.length - 1];
704
- const pathToken = tokens[tokens.length - 2];
705
- return pathOrFunctionToken?.endsWith(ext) || pathToken?.endsWith(ext);
706
- }) || str.charAt(str.length - 3) === "." || str.charAt(str.length - 4) === "." || str.includes("*") || str.includes("/") || str.includes("\\");
707
- }
708
- /**
709
- * Normalizes the input prompt to an array of prompts, rejecting invalid and empty inputs.
710
- * @param promptPathOrGlobs - The input prompt.
711
- * @returns The normalized prompts.
712
- * @throws If the input is invalid or empty.
713
- */
714
- function normalizeInput(promptPathOrGlobs) {
715
- if (!promptPathOrGlobs || (typeof promptPathOrGlobs === "string" || Array.isArray(promptPathOrGlobs)) && promptPathOrGlobs.length === 0) throw new Error(`Invalid input prompt: ${JSON.stringify(promptPathOrGlobs)}`);
716
- if (typeof promptPathOrGlobs === "string") return [{ raw: promptPathOrGlobs }];
717
- if (Array.isArray(promptPathOrGlobs)) return promptPathOrGlobs.map((promptPathOrGlob, _index) => {
718
- if (typeof promptPathOrGlob === "string") return { raw: promptPathOrGlob };
719
- return {
720
- raw: promptPathOrGlob.raw || promptPathOrGlob.id,
721
- ...promptPathOrGlob
722
- };
723
- });
724
- if (typeof promptPathOrGlobs === "object" && Object.keys(promptPathOrGlobs).length) return Object.entries(promptPathOrGlobs).map(([raw, key]) => ({
725
- label: key,
726
- raw
727
- }));
728
- throw new Error(`Invalid input prompt: ${JSON.stringify(promptPathOrGlobs)}`);
729
- }
730
- /**
731
- * Generates a hash identifier for a prompt.
732
- * This is an alias for generateIdFromPrompt for backward compatibility.
733
- *
734
- * @param prompt - The prompt to hash
735
- * @returns A SHA-256 hash string
736
- */
737
- function hashPrompt(prompt) {
738
- return generateIdFromPrompt(prompt);
739
- }
740
- //#endregion
741
- //#region src/database/signal.ts
742
- const SIGNAL_WATCHER_DEBOUNCE_MS = 250;
743
- /**
744
- * The legacy `evalId:timestamp` text format can't be told apart from a stray short token, so
745
- * {@link readLegacySignalEvalId} only recovers eval ids at least this long. Generated ids
746
- * (`eval-<rand>-<iso>`) always clear it; a short caller-provided id (e.g. an imported `demo`) is
747
- * written as JSON instead, which stores the id explicitly and survives the round trip.
748
- */
749
- const MIN_LEGACY_EVAL_ID_LENGTH = 9;
750
- /** Whether {@link readLegacySignalEvalId} can recover this id from the legacy text format. */
751
- function isLegacyRecoverableEvalId(evalId) {
752
- return evalId.length >= MIN_LEGACY_EVAL_ID_LENGTH;
753
- }
754
- /** Returns the string entries of an unknown value, or undefined if it is not an array. */
755
- function toStringEvalIds(value) {
756
- return Array.isArray(value) ? value.filter((evalId) => typeof evalId === "string") : void 0;
757
- }
758
- function writeSignalFile(content) {
759
- const filePath = require_tables.getDbSignalPath();
760
- try {
761
- fs.default.writeFileSync(filePath, content);
762
- } catch (err) {
763
- require_logger.logger.warn(`Failed to write database signal file: ${err}`);
764
- }
765
- }
766
- /**
767
- * Updates the signal file with the current timestamp and optional eval ID.
768
- * This is used to notify clients that there are new data available.
769
- *
770
- * If another signal is still pending in the watcher's debounce window, this folds it into a
771
- * combined JSON payload so no refresh is lost — scoped updates, a pending unscoped "refresh
772
- * latest", and pending deletes all survive coalescing. When nothing is pending it keeps the
773
- * compact legacy `evalId:timestamp` text.
774
- * This read-modify-write is best-effort within a single process; concurrent writes from separate
775
- * processes can still race the unlocked file, in which case a dropped signal self-heals on the
776
- * next one.
777
- * @param evalId - Optional eval ID that triggered the update
778
- */
779
- function updateSignalFile(evalId) {
780
- const now = Date.now();
781
- const nowIso = new Date(now).toISOString();
782
- const pending = readPendingSignal(now);
783
- const pendingUpdateIds = pending ? updateEvalIds(pending) : [];
784
- const pendingHasDelete = pending ? hasDeleteComponent(pending) : false;
785
- const pendingHasUnscopedUpdate = pending ? hasUnscopedUpdate(pending) : false;
786
- const updatedEvalIds = [.../* @__PURE__ */ new Set([...pendingUpdateIds, ...evalId ? [evalId] : []])];
787
- const refreshLatest = evalId === void 0 || pendingHasUnscopedUpdate;
788
- const needsCoalesce = pendingHasDelete || pendingUpdateIds.some((id) => id !== evalId) || refreshLatest && updatedEvalIds.length > 0;
789
- const evalIdNeedsJson = evalId !== void 0 && !isLegacyRecoverableEvalId(evalId);
790
- if (needsCoalesce || evalIdNeedsJson) {
791
- writeSignalFile(JSON.stringify({
792
- type: "update",
793
- evalId,
794
- updatedEvalIds: updatedEvalIds.length > 0 ? updatedEvalIds : void 0,
795
- deletedEvalIds: pendingHasDelete ? pending?.deletedEvalIds ?? [] : void 0,
796
- refreshLatest: refreshLatest ? true : void 0,
797
- timestamp: nowIso
798
- }));
799
- return;
800
- }
801
- writeSignalFile(evalId ? `${evalId}:${nowIso}` : nowIso);
802
- }
803
- /** True when the signal carries an update component (a scoped or unscoped eval refresh). */
804
- function hasUpdateComponent(signal) {
805
- return signal.type === "update" || signal.evalId !== void 0 || (signal.updatedEvalIds?.length ?? 0) > 0 || signal.refreshLatest === true;
806
- }
807
- /**
808
- * True when the signal asks to refresh the latest eval (an unscoped update component): the
809
- * `refreshLatest` marker, or a bare update with no scoped ids (e.g. a legacy bare timestamp). The
810
- * watcher emits {@link Eval.latest} for these, so a root `/eval` view follows new evals even when a
811
- * coalesced signal also carries scoped updates or a delete.
812
- */
813
- function hasUnscopedUpdate(signal) {
814
- return signal.refreshLatest === true || hasUpdateComponent(signal) && updateEvalIds(signal).length === 0;
815
- }
816
- /** True when the signal carries a delete component (specific ids or "all evals deleted"). */
817
- function hasDeleteComponent(signal) {
818
- return signal.type === "delete" || signal.deletedEvalIds !== void 0;
819
- }
820
- /**
821
- * True when a delete component means "all evals were deleted" rather than a specific id list.
822
- * "All" is encoded as `undefined` by direct delete callers and as `[]` when a pending delete-all
823
- * is folded into a JSON update, so both count. Only meaningful for a signal that actually carries
824
- * a delete component (see {@link hasDeleteComponent}); an update-only signal has no ids either.
825
- */
826
- function isAllEvalsDeleted(deletedEvalIds) {
827
- return deletedEvalIds === void 0 || deletedEvalIds.length === 0;
828
- }
829
- /**
830
- * The distinct scoped eval ids an update signal refreshes, newest last. Empty for an unscoped
831
- * update (which means "refresh to the latest eval"). Prefers the accumulated `updatedEvalIds`
832
- * set and falls back to the single `evalId` for legacy/simple signals.
833
- */
834
- function updateEvalIds(signal) {
835
- if (signal.updatedEvalIds && signal.updatedEvalIds.length > 0) return signal.updatedEvalIds;
836
- return signal.evalId ? [signal.evalId] : [];
837
- }
838
- /**
839
- * Parses signal-file content as a combined JSON payload, returning the classified signal and
840
- * its timestamp (used for the debounce-window check), or null if it is not a JSON signal.
841
- */
842
- function parseSignalJson(content) {
843
- try {
844
- const parsed = JSON.parse(content);
845
- if (parsed !== null && typeof parsed === "object" && "type" in parsed && (parsed.type === "delete" || parsed.type === "update")) {
846
- const evalId = "evalId" in parsed && typeof parsed.evalId === "string" ? parsed.evalId : void 0;
847
- const updatedEvalIds = "updatedEvalIds" in parsed ? toStringEvalIds(parsed.updatedEvalIds) : void 0;
848
- const deletedEvalIds = "deletedEvalIds" in parsed ? toStringEvalIds(parsed.deletedEvalIds) : void 0;
849
- const timestamp = "timestamp" in parsed && typeof parsed.timestamp === "string" ? parsed.timestamp : void 0;
850
- const refreshLatest = "refreshLatest" in parsed && parsed.refreshLatest === true ? true : void 0;
851
- return {
852
- signal: {
853
- type: parsed.type,
854
- evalId,
855
- updatedEvalIds,
856
- deletedEvalIds,
857
- refreshLatest
858
- },
859
- timestamp
860
- };
861
- }
862
- } catch {}
863
- return null;
864
- }
865
- /**
866
- * Reads the signal file only if it was written within the watcher's debounce window, so a
867
- * writer can fold a not-yet-observed signal into its own write instead of clobbering it.
868
- * Returns null when the file is stale (outside the window), unreadable, or unparseable.
869
- */
870
- function readPendingSignal(now) {
871
- const withinWindow = (timestamp) => {
872
- if (timestamp === void 0) return false;
873
- const elapsed = now - Date.parse(timestamp);
874
- return Number.isFinite(elapsed) && elapsed >= 0 && elapsed <= SIGNAL_WATCHER_DEBOUNCE_MS;
875
- };
876
- try {
877
- const content = fs.default.readFileSync(require_tables.getDbSignalPath(), "utf8").trim();
878
- const json = parseSignalJson(content);
879
- if (json) return withinWindow(json.timestamp) ? json.signal : null;
880
- const legacyTimestamp = content.match(/(\d{4}-\d{2}-\d{2}T[0-9:.]+Z?)$/)?.[1];
881
- if (!withinWindow(legacyTimestamp)) return null;
882
- return {
883
- type: "update",
884
- evalId: readLegacySignalEvalId(content)
885
- };
886
- } catch {
887
- return null;
888
- }
889
- }
890
- /**
891
- * Signals that one or more evals were deleted. Deletions use a JSON payload so the watcher can
892
- * tell clients exactly which evals to drop (whereas a plain update keeps the legacy
893
- * `evalId:timestamp` text format unless it has to fold in a pending delete). A delete in the
894
- * debounce window also carries forward a pending update — scoped ids or an unscoped "refresh
895
- * latest" — so neither refresh is lost (same-process best-effort, like {@link updateSignalFile};
896
- * cross-process writes can still race).
897
- * Pre-PR view servers don't understand this JSON and read it back as "no eval id", falling back
898
- * to broadcasting the latest eval — a benign degradation, so the view server and CLI should be
899
- * on matching versions for delete-aware refreshes.
900
- */
901
- function updateSignalFileForDeletedEvals(deletedEvalIds) {
902
- const now = Date.now();
903
- const pending = readPendingSignal(now);
904
- const pendingDelete = pending && hasDeleteComponent(pending) ? pending : null;
905
- const mergedDeletedEvalIds = pendingDelete ? isAllEvalsDeleted(pendingDelete.deletedEvalIds) || isAllEvalsDeleted(deletedEvalIds) ? void 0 : [.../* @__PURE__ */ new Set([...pendingDelete.deletedEvalIds ?? [], ...deletedEvalIds ?? []])] : deletedEvalIds;
906
- const carriedUpdateIds = pending && hasUpdateComponent(pending) ? updateEvalIds(pending) : [];
907
- const carriedRefreshLatest = pending ? hasUnscopedUpdate(pending) : false;
908
- writeSignalFile(JSON.stringify({
909
- type: "delete",
910
- deletedEvalIds: mergedDeletedEvalIds,
911
- evalId: carriedUpdateIds[carriedUpdateIds.length - 1],
912
- updatedEvalIds: carriedUpdateIds.length > 0 ? carriedUpdateIds : void 0,
913
- refreshLatest: carriedRefreshLatest ? true : void 0,
914
- timestamp: new Date(now).toISOString()
915
- }));
916
- }
917
- function readLegacySignalEvalId(content) {
918
- if (/^\d{4}-\d{2}-\d{2}T/.test(content)) return;
919
- const evalId = content.match(/^(.*):\d{4}-\d{2}-\d{2}T/)?.[1];
920
- return evalId && isLegacyRecoverableEvalId(evalId) ? evalId : void 0;
921
- }
922
- //#endregion
923
- //#region src/util/standaloneEvalCache.ts
924
- const standaloneEvalCache = new lru_cache.LRUCache({
925
- ttl: 72e5,
926
- max: 2e3
927
- });
928
- function clearStandaloneEvalCache() {
929
- standaloneEvalCache.clear();
930
- }
931
- //#endregion
932
- //#region src/models/evalPerformance.ts
933
- const countCache = /* @__PURE__ */ new Map();
934
- const CACHE_TTL = 3e5;
935
- async function getResultCount(evalId, distinct) {
936
- const cacheKey = `${distinct ? "distinct" : "total"}:${evalId}`;
937
- const cached = countCache.get(cacheKey);
938
- if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
939
- require_logger.logger.debug(`Using cached ${distinct ? "distinct count" : "total row count"} for eval ${evalId}: ${cached.count}`);
940
- return cached.count;
941
- }
942
- const db = await require_tables.getDb();
943
- const start = Date.now();
944
- const result = await db.select({ count: distinct ? drizzle_orm.sql`COUNT(DISTINCT test_idx)` : drizzle_orm.sql`COUNT(*)` }).from(require_tables.evalResultsTable).where(drizzle_orm.sql`eval_id = ${evalId}`).all();
945
- const count = Number(result[0]?.count ?? 0);
946
- const duration = Date.now() - start;
947
- require_logger.logger.debug(`${distinct ? "Distinct count" : "Total row count"} query for eval ${evalId}: ${count} in ${duration}ms`);
948
- countCache.set(cacheKey, {
949
- count,
950
- timestamp: Date.now()
951
- });
952
- return count;
953
- }
954
- /**
955
- * Get the count of distinct test indices for an eval.
956
- * This represents the number of unique test cases (rows in the UI table).
957
- *
958
- * Use getTotalResultRowCount() if you need the total number of result rows
959
- * (which may be higher when there are multiple prompts/providers per test case).
960
- */
961
- function getCachedResultsCount(evalId) {
962
- return getResultCount(evalId, true);
963
- }
964
- /**
965
- * Get the total count of all result rows for an eval.
966
- * This counts every result row in the database, including multiple results
967
- * per test case (e.g., when using multiple prompts or providers).
968
- *
969
- * Use this for progress tracking when iterating over all results (e.g., sharing).
970
- */
971
- function getTotalResultRowCount(evalId) {
972
- return getResultCount(evalId, false);
973
- }
974
- function clearCountCache(evalId) {
975
- if (evalId) {
976
- countCache.delete(`distinct:${evalId}`);
977
- countCache.delete(`total:${evalId}`);
978
- } else countCache.clear();
979
- }
980
- async function queryTestIndicesOptimized(evalId, opts) {
981
- const db = await require_tables.getDb();
982
- const offset = opts.offset ?? 0;
983
- const limit = opts.limit ?? 50;
984
- const mode = opts.filterMode ?? "all";
985
- let baseQuery = drizzle_orm.sql`eval_id = ${evalId}`;
986
- if (mode === "errors") baseQuery = drizzle_orm.sql`${baseQuery} AND failure_reason = ${2}`;
987
- else if (mode === "failures") baseQuery = drizzle_orm.sql`${baseQuery} AND success = 0 AND failure_reason != ${2}`;
988
- else if (mode === "passes") baseQuery = drizzle_orm.sql`${baseQuery} AND success = 1`;
989
- else if (mode === "highlights") baseQuery = drizzle_orm.sql`${baseQuery} AND json_extract(grading_result, '$.comment') LIKE '!highlight%'`;
990
- else if (mode === "user-rated") baseQuery = drizzle_orm.sql`${baseQuery} AND EXISTS (
991
- SELECT 1
992
- FROM json_each(grading_result, '$.componentResults')
993
- WHERE json_extract(value, '$.assertion.type') = ${HUMAN_ASSERTION_TYPE}
994
- )`;
995
- let searchCondition = drizzle_orm.sql`1=1`;
996
- if (opts.searchQuery && opts.searchQuery.trim() !== "" && !opts.filters?.length) searchCondition = drizzle_orm.sql`response LIKE ${`%${opts.searchQuery}%`}`;
997
- const whereClause = drizzle_orm.sql`${baseQuery} AND ${searchCondition}`;
998
- const countStart = Date.now();
999
- const countQuery = drizzle_orm.sql`
1000
- SELECT COUNT(DISTINCT test_idx) as count
1001
- FROM ${require_tables.evalResultsTable}
1002
- WHERE ${whereClause}
1003
- `;
1004
- const countResult = await db.all(countQuery);
1005
- const filteredCount = Number(countResult[0]?.count ?? 0);
1006
- require_logger.logger.debug(`Optimized count query took ${Date.now() - countStart}ms`);
1007
- const idxStart = Date.now();
1008
- const idxQuery = drizzle_orm.sql`
1009
- SELECT DISTINCT test_idx
1010
- FROM ${require_tables.evalResultsTable}
1011
- WHERE ${whereClause}
1012
- ORDER BY test_idx
1013
- LIMIT ${limit}
1014
- OFFSET ${offset}
1015
- `;
1016
- const testIndices = (await db.all(idxQuery)).map((row) => row.test_idx);
1017
- require_logger.logger.debug(`Optimized index query took ${Date.now() - idxStart}ms`);
1018
- return {
1019
- testIndices,
1020
- filteredCount
1021
- };
1022
- }
1023
- //#endregion
1024
- //#region src/models/evalMutation.ts
1025
- /**
1026
- * Drops this process's cached eval aggregates so the next read re-queries the DB.
1027
- * Always clears the standalone-eval LRU (keyed by filter combos, no per-eval granularity)
1028
- * and the count cache for `evalId` (or every eval's counts when `evalId` is omitted).
1029
- * Use this for in-process cache coherence only — it does NOT notify the view server.
1030
- */
1031
- function invalidateEvaluationCache(evalId) {
1032
- clearStandaloneEvalCache();
1033
- clearCountCache(evalId);
1034
- }
1035
- /**
1036
- * Like {@link invalidateEvaluationCache} but for several evals at once: clears the standalone-eval
1037
- * LRU once (it has no per-eval granularity, so repeating it per id is wasteful) and the count
1038
- * cache for each id.
1039
- */
1040
- function invalidateEvaluationCaches(evalIds) {
1041
- clearStandaloneEvalCache();
1042
- for (const id of evalIds) clearCountCache(id);
1043
- }
1044
- /**
1045
- * Call after creating or mutating an eval's data (results, prompts, ratings). Invalidates
1046
- * this process's caches and writes a scoped update signal so a running `promptfoo view`
1047
- * server refreshes clients viewing this eval (and the root route, which follows the latest).
1048
- */
1049
- function notifyEvaluationChanged(evalId) {
1050
- invalidateEvaluationCache(evalId);
1051
- updateSignalFile(evalId);
1052
- }
1053
- /**
1054
- * Call after deleting one or more evals. Invalidates this process's caches and writes a
1055
- * delete signal so the view server can navigate clients off the removed evals. Pass no
1056
- * argument (or omit ids) for "all evals deleted".
1057
- */
1058
- function notifyEvaluationsDeleted(evalIds) {
1059
- if (evalIds) invalidateEvaluationCaches(evalIds);
1060
- else invalidateEvaluationCache();
1061
- updateSignalFileForDeletedEvals(evalIds);
1062
- }
1063
- //#endregion
1064
- //#region src/models/evalResult.ts
1065
- function sanitizeProviderConfig(config) {
1066
- return require_logger.sanitizeObject(JSON.parse(require_logger.safeJsonStringify(config)), {
1067
- context: "provider config",
1068
- sanitizeUrls: true,
1069
- maxDepth: Number.POSITIVE_INFINITY
1070
- });
1071
- }
1072
- function stripMediaReferences(value) {
1073
- if (require_extractor.extractBlobHashesFromValue(value).length > 0 || typeof value === "string" && /^data:[^,]*,/i.test(value)) return "[output stripped]";
1074
- if (Array.isArray(value)) return value.map(stripMediaReferences);
1075
- if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, stripMediaReferences(item)]));
1076
- return value;
1077
- }
1078
- function projectOutputMetadata(metadata, stripOutput, responseMetadata, testMetadata) {
1079
- if (!stripOutput || !metadata || !responseMetadata || typeof metadata !== "object") return metadata;
1080
- return Object.fromEntries(Object.entries(metadata).flatMap(([key, value]) => {
1081
- if (!Object.prototype.hasOwnProperty.call(responseMetadata, key) || testMetadata && (0, node_util.isDeepStrictEqual)(value, testMetadata[key])) return [[key, value]];
1082
- return key === "audio" || key === "blobUris" ? [] : [[key, stripMediaReferences(sanitizeForDb(value))]];
1083
- }));
1084
- }
1085
- function projectProviderResponse(response, options) {
1086
- if (!response) return response;
1087
- if (!options.stripMetadata && !options.stripOutput) return response;
1088
- const projectedResponse = options.stripMetadata ? (({ metadata: _metadata, ...rest }) => rest)(response) : { ...response };
1089
- if (options.stripOutput) {
1090
- projectedResponse.output = "[output stripped]";
1091
- delete projectedResponse.raw;
1092
- delete projectedResponse.providerTransformedOutput;
1093
- delete projectedResponse.conversationOutput;
1094
- delete projectedResponse.audio;
1095
- delete projectedResponse.video;
1096
- delete projectedResponse.images;
1097
- if (projectedResponse.metadata) projectedResponse.metadata = projectOutputMetadata(projectedResponse.metadata, true, projectedResponse.metadata);
1098
- }
1099
- return projectedResponse;
1100
- }
1101
- function projectPrompt(prompt, stripPromptText) {
1102
- return stripPromptText ? {
1103
- ...prompt,
1104
- raw: "[prompt stripped]",
1105
- template: void 0,
1106
- ...prompt.config?.provider && { config: {
1107
- ...prompt.config,
1108
- provider: require_logger.stripProviderPromptSelectors(prompt.config.provider)
1109
- } }
1110
- } : prompt;
1111
- }
1112
- function projectTracesForOutput(traces, { shouldStripMetadata, shouldStripPromptText, shouldStripResponseOutput, shouldStripTestVars }) {
1113
- if (!shouldStripMetadata && !shouldStripPromptText && !shouldStripResponseOutput && !shouldStripTestVars) return traces;
1114
- return traces.map((trace) => {
1115
- let projectedTrace = trace;
1116
- if (shouldStripMetadata) {
1117
- const { metadata: _metadata, ...traceWithoutMetadata } = trace;
1118
- projectedTrace = traceWithoutMetadata;
1119
- } else if (shouldStripTestVars && trace.metadata && "vars" in trace.metadata) {
1120
- const { metadata: traceMetadata, ...traceWithoutMetadata } = trace;
1121
- const { vars: _vars, ...metadata } = traceMetadata;
1122
- projectedTrace = {
1123
- ...traceWithoutMetadata,
1124
- ...Object.keys(metadata).length > 0 && { metadata }
1125
- };
1126
- }
1127
- if (!shouldStripPromptText && !shouldStripResponseOutput) return projectedTrace;
1128
- return {
1129
- ...projectedTrace,
1130
- spans: projectedTrace.spans.map((span) => {
1131
- if (!span.attributes) return span;
1132
- const projectedAttributes = { ...span.attributes };
1133
- if (shouldStripPromptText) delete projectedAttributes[require_genaiTracer.PromptfooAttributes.REQUEST_BODY];
1134
- if (shouldStripResponseOutput) delete projectedAttributes[require_genaiTracer.PromptfooAttributes.RESPONSE_BODY];
1135
- const { attributes: _attributes, ...projectedSpan } = span;
1136
- return {
1137
- ...projectedSpan,
1138
- ...Object.keys(projectedAttributes).length > 0 && { attributes: projectedAttributes }
1139
- };
1140
- })
1141
- };
1142
- });
1143
- }
1144
- function projectTestCase(testCase, options) {
1145
- if (!options.stripMetadata && !options.stripVars && !options.stripOutput && !options.stripPromptText) return testCase;
1146
- const projectedTestCase = options.stripMetadata ? (({ metadata: _metadata, ...rest }) => rest)(testCase) : { ...testCase };
1147
- if (options.stripVars) projectedTestCase.vars = void 0;
1148
- if (options.stripOutput) delete projectedTestCase.providerOutput;
1149
- if (options.stripMetadata && testCase.metadata?.__promptfoo?.remote === true) projectedTestCase.metadata = { __promptfoo: { remote: true } };
1150
- return options.stripPromptText ? require_logger.stripTestProviderPromptSelectors(projectedTestCase) : projectedTestCase;
1151
- }
1152
- /** Map only grading assertion providers and component results. */
1153
- function mapGradingResultProviderRefs(gradingResult, mapProvider) {
1154
- const visited = /* @__PURE__ */ new WeakMap();
1155
- const project = (value) => {
1156
- const result = asRecord(value);
1157
- if (!result) return value;
1158
- const previous = visited.get(result);
1159
- if (previous) return previous;
1160
- const projected = { ...result };
1161
- visited.set(result, projected);
1162
- if (result.assertion) projected.assertion = require_logger.mapTestProviderRefs(result.assertion, mapProvider);
1163
- if (Array.isArray(result.componentResults)) projected.componentResults = result.componentResults.map(project);
1164
- return projected;
1165
- };
1166
- return project(gradingResult);
1167
- }
1168
- /** Project assertion providers only, preserving grading details and unrelated metadata. */
1169
- function projectGradingResult(gradingResult, stripPromptText) {
1170
- return stripPromptText ? mapGradingResultProviderRefs(gradingResult, require_logger.stripProviderPromptSelectors) : gradingResult;
1171
- }
1172
- function sanitizeProvider(provider) {
1173
- try {
1174
- if (require_logger.isApiProvider(provider)) return {
1175
- id: provider.id(),
1176
- label: provider.label,
1177
- ...provider.config && { config: sanitizeProviderConfig(provider.config) }
1178
- };
1179
- if (require_logger.isProviderOptions(provider)) return {
1180
- id: provider.id,
1181
- label: provider.label,
1182
- ...provider.config && { config: sanitizeProviderConfig(provider.config) }
1183
- };
1184
- if (typeof provider === "object" && provider) {
1185
- const providerObj = provider;
1186
- return {
1187
- id: typeof providerObj.id === "function" ? providerObj.id() : providerObj.id,
1188
- label: providerObj.label,
1189
- ...providerObj.config && { config: sanitizeProviderConfig(providerObj.config) }
1190
- };
1191
- }
1192
- } catch {}
1193
- return JSON.parse(require_logger.safeJsonStringify(provider));
1194
- }
1195
- /** Snapshot live provider references for replay without retaining runtime client state. */
1196
- function toSerializableProviderRef(provider) {
1197
- if (require_logger.isApiProvider(provider)) return {
1198
- ...sanitizeProvider(provider),
1199
- ...require_logger.sanitizeObject({
1200
- transform: typeof provider.transform === "string" ? provider.transform : void 0,
1201
- delay: provider.delay,
1202
- inputs: provider.inputs
1203
- }, {
1204
- context: "provider options",
1205
- sanitizeUrls: true,
1206
- maxDepth: Number.POSITIVE_INFINITY
1207
- }),
1208
- ...provider.prompts && { prompts: [...provider.prompts] }
1209
- };
1210
- if (Array.isArray(provider)) return provider.map(toSerializableProviderRef);
1211
- if (require_logger.isProviderTypeMap(provider)) {
1212
- let serialized;
1213
- for (const type of require_logger.GRADING_PROVIDER_TYPE_KEYS) if (require_logger.isApiProvider(provider[type])) {
1214
- serialized ??= { ...provider };
1215
- serialized[type] = toSerializableProviderRef(provider[type]);
1216
- }
1217
- return serialized ?? provider;
1218
- }
1219
- return provider;
1220
- }
1221
- /** Snapshot live grading references before generic result serialization invokes provider toJSON. */
1222
- function serializeResultProviderRefs(result) {
1223
- const record = result;
1224
- const serializeProvider = (provider) => {
1225
- if (!require_logger.isApiProvider(provider) && !require_logger.isProviderTypeMap(provider)) return provider;
1226
- return require_logger.sanitizeObject(toSerializableProviderRef(provider), {
1227
- context: "grading provider",
1228
- sanitizeUrls: true,
1229
- maxDepth: Number.POSITIVE_INFINITY,
1230
- throwOnError: true
1231
- });
1232
- };
1233
- const projected = { ...record };
1234
- if (record.testCase) projected.testCase = require_logger.mapTestProviderRefs(record.testCase, serializeProvider);
1235
- const prompt = asRecord(record.prompt);
1236
- const config = asRecord(prompt?.config);
1237
- if (config?.provider !== void 0) projected.prompt = {
1238
- ...prompt,
1239
- config: {
1240
- ...config,
1241
- provider: serializeProvider(config.provider)
1242
- }
1243
- };
1244
- if (record.gradingResult) projected.gradingResult = mapGradingResultProviderRefs(record.gradingResult, serializeProvider);
1245
- return projected;
1246
- }
1247
- /**
1248
- * Sanitize an object for database storage by removing circular references
1249
- * and non-serializable values (functions, Timeout objects, etc.).
1250
- * Uses safeJsonStringify which handles circular references gracefully.
1251
- *
1252
- * This prevents "Converting circular structure to JSON" errors that can occur
1253
- * when Node.js Timeout objects or other non-serializable data leaks into results.
1254
- * See: https://github.com/promptfoo/promptfoo/issues/7266
1255
- */
1256
- function sanitizeForDb(obj) {
1257
- if (obj === null || obj === void 0) return obj;
1258
- try {
1259
- const serialized = require_logger.safeJsonStringify(obj);
1260
- if (serialized === void 0) {
1261
- require_logger.logger.debug("sanitizeForDb: Failed to serialize object, using fallback", {
1262
- valueType: typeof obj,
1263
- isArray: Array.isArray(obj)
1264
- });
1265
- return Array.isArray(obj) ? [] : null;
1266
- }
1267
- return JSON.parse(serialized);
1268
- } catch (error) {
1269
- require_logger.logger.debug("sanitizeForDb: Parse error, using fallback", { error });
1270
- return Array.isArray(obj) ? [] : null;
1271
- }
1272
- }
1273
- /**
1274
- * Sanitize a per-test-case field for persistence: strips circular refs,
1275
- * collapses class instances (e.g. live SDK clients that leaked in via
1276
- * `defaultTest.options.provider`), and redacts credential fields (`apiKey`,
1277
- * `token`, etc.) at any depth. Use this for any slot that can carry a provider
1278
- * config — notably `testCase.options.provider` and `prompt.config.provider`,
1279
- * where the resolved runtime provider (with its Anthropic / Bedrock SDK
1280
- * client) flows in from the evaluator. Without this, credentials configured on
1281
- * the judge provider end up in the Eval results both in the DB and in the
1282
- * polling response served by `/api/eval/job/:id`.
1283
- */
1284
- function sanitizeForDbWithSecrets(obj) {
1285
- if (obj === null || obj === void 0) return obj;
1286
- return require_logger.sanitizeObject(obj, {
1287
- context: "evalResult field",
1288
- maxDepth: Number.POSITIVE_INFINITY
1289
- });
1290
- }
1291
- const SENSITIVE_RESPONSE_HEADER_NAMES = /* @__PURE__ */ new Set([
1292
- "authorization",
1293
- "proxy-authorization",
1294
- "cookie",
1295
- "set-cookie",
1296
- "openai-organization",
1297
- "openai-project",
1298
- "openai-version",
1299
- "x-request-id",
1300
- "x-amzn-requestid",
1301
- "x-amzn-trace-id",
1302
- "x-amz-security-token",
1303
- "x-amz-cf-id",
1304
- "x-azure-ref",
1305
- "x-correlation-id",
1306
- "x-trace-id",
1307
- "cf-ray",
1308
- "cf-cache-status",
1309
- "x-openai-proxy-wasm",
1310
- "via"
1311
- ]);
1312
- const SENSITIVE_RESPONSE_HEADER_PREFIXES = ["x-ratelimit-"];
1313
- function isSensitiveResponseHeader(headerName) {
1314
- const normalized = headerName.toLowerCase();
1315
- if (SENSITIVE_RESPONSE_HEADER_NAMES.has(normalized)) return true;
1316
- return SENSITIVE_RESPONSE_HEADER_PREFIXES.some((prefix) => normalized.startsWith(prefix));
1317
- }
1318
- function isSensitiveRequestHeader(headerName) {
1319
- return isSensitiveResponseHeader(headerName) || require_logger.isSecretField(headerName);
1320
- }
1321
- function redactSensitiveHeaders(headers, sourceHeaders = headers, isSensitiveHeader = isSensitiveResponseHeader) {
1322
- let mutated = false;
1323
- const next = {};
1324
- for (const [key, value] of Object.entries(headers)) if (Object.prototype.hasOwnProperty.call(sourceHeaders, key) && (0, node_util.isDeepStrictEqual)(sourceHeaders[key], value) && isSensitiveHeader(key)) {
1325
- next[key] = require_logger.REDACTED;
1326
- mutated = true;
1327
- } else next[key] = value;
1328
- return mutated ? next : null;
1329
- }
1330
- function redactHttpHeadersOnMetadata(metadata, options) {
1331
- if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) return metadata;
1332
- const m = metadata;
1333
- let nextMetadata;
1334
- const legacyHeaders = m.headers;
1335
- const legacyHeadersSource = options?.redactLegacyHeaders ? legacyHeaders : (options?.legacyHeadersSource)?.headers;
1336
- if (legacyHeaders && typeof legacyHeaders === "object" && !Array.isArray(legacyHeaders) && legacyHeadersSource && typeof legacyHeadersSource === "object" && !Array.isArray(legacyHeadersSource)) {
1337
- const redacted = redactSensitiveHeaders(legacyHeaders, legacyHeadersSource, isSensitiveRequestHeader);
1338
- if (redacted) nextMetadata = {
1339
- ...m,
1340
- headers: redacted
1341
- };
1342
- }
1343
- const http = m.http;
1344
- if (!http || typeof http !== "object" || Array.isArray(http)) return nextMetadata ?? metadata;
1345
- const httpRecord = http;
1346
- let nextHttp;
1347
- for (const slot of ["headers", "requestHeaders"]) {
1348
- const slotValue = httpRecord[slot];
1349
- if (slotValue && typeof slotValue === "object" && !Array.isArray(slotValue)) {
1350
- const redacted = slot === "requestHeaders" ? redactSensitiveHeaders(slotValue, slotValue, isSensitiveRequestHeader) : redactSensitiveHeaders(slotValue);
1351
- if (redacted) {
1352
- nextHttp ??= { ...httpRecord };
1353
- nextHttp[slot] = redacted;
1354
- }
1355
- }
1356
- }
1357
- if (!nextHttp) return nextMetadata ?? metadata;
1358
- nextMetadata ??= { ...m };
1359
- nextMetadata.http = nextHttp;
1360
- return nextMetadata;
1361
- }
1362
- function redactHttpHeadersOnGradingResult(gradingResult) {
1363
- if (!gradingResult || typeof gradingResult !== "object" || Array.isArray(gradingResult)) return gradingResult;
1364
- const gr = gradingResult;
1365
- let mutated = false;
1366
- const next = { ...gr };
1367
- if (gr.metadata !== void 0) {
1368
- const redacted = redactHttpHeadersOnMetadata(gr.metadata);
1369
- if (redacted !== gr.metadata) {
1370
- next.metadata = redacted;
1371
- mutated = true;
1372
- }
1373
- }
1374
- if (Array.isArray(gr.componentResults)) {
1375
- let componentMutated = false;
1376
- const nextComponents = gr.componentResults.map((component) => {
1377
- const redacted = redactHttpHeadersOnGradingResult(component);
1378
- if (redacted !== component) componentMutated = true;
1379
- return redacted;
1380
- });
1381
- if (componentMutated) {
1382
- next.componentResults = nextComponents;
1383
- mutated = true;
1384
- }
1385
- }
1386
- return mutated ? next : gradingResult;
1387
- }
1388
- function sanitizeResponseForDb(response) {
1389
- if (!response) return response;
1390
- const redactedMetadata = redactHttpHeadersOnMetadata(response.metadata, { redactLegacyHeaders: true });
1391
- if (redactedMetadata === response.metadata) return response;
1392
- return {
1393
- ...response,
1394
- metadata: redactedMetadata
1395
- };
1396
- }
1397
- function sanitizeMetadataForDb(metadata, responseMetadata) {
1398
- return redactHttpHeadersOnMetadata(metadata, { legacyHeadersSource: sanitizeForDb(responseMetadata) });
1399
- }
1400
- function sanitizeGradingResultAssertions(gradingResult) {
1401
- if (!gradingResult || typeof gradingResult !== "object" || Array.isArray(gradingResult)) return gradingResult;
1402
- const gr = gradingResult;
1403
- const assertion = asRecord(gr.assertion);
1404
- const sanitizedAssertion = assertion && { ...assertion };
1405
- if (sanitizedAssertion) {
1406
- for (const field of ["provider", "config"]) if (Object.prototype.hasOwnProperty.call(sanitizedAssertion, field)) sanitizedAssertion[field] = require_logger.sanitizeObject(sanitizedAssertion[field], {
1407
- context: "grading assertion",
1408
- maxDepth: Number.POSITIVE_INFINITY,
1409
- sanitizeUrls: true,
1410
- throwOnError: true
1411
- });
1412
- }
1413
- return {
1414
- ...gr,
1415
- ...sanitizedAssertion && { assertion: sanitizedAssertion },
1416
- ...Array.isArray(gr.componentResults) && { componentResults: gr.componentResults.map(sanitizeGradingResultAssertions) }
1417
- };
1418
- }
1419
- function sanitizeGradingResultForDb(gradingResult) {
1420
- return redactHttpHeadersOnGradingResult(sanitizeGradingResultAssertions(gradingResult));
1421
- }
1422
- const PROMPTFOO_METADATA_KEY = "__promptfoo";
1423
- const TRACE_LINKAGE_KEY = "traceLinkage";
1424
- const REPEAT_LINKAGE_KEY = "repeatLinkage";
1425
- function asRecord(value) {
1426
- return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
1427
- }
1428
- function persistTraceMetadata(metadata, traceId, evaluationId) {
1429
- if (!traceId && !evaluationId) return stripTraceLinkageFromMetadata(metadata);
1430
- const metadataRecord = metadata ?? {};
1431
- const promptfooMetadata = asRecord(metadataRecord[PROMPTFOO_METADATA_KEY]);
1432
- if (metadataRecord["__promptfoo"] !== void 0 && promptfooMetadata === void 0) require_logger.logger.warn(`[EvalResult] Overwriting non-object metadata.${PROMPTFOO_METADATA_KEY} with internal trace linkage; the key is reserved for promptfoo internals.`);
1433
- if (promptfooMetadata && TRACE_LINKAGE_KEY in promptfooMetadata) require_logger.logger.warn(`[EvalResult] Overwriting metadata.${PROMPTFOO_METADATA_KEY}.${TRACE_LINKAGE_KEY} with internal trace linkage; the path is reserved for promptfoo internals.`);
1434
- return {
1435
- ...metadataRecord,
1436
- [PROMPTFOO_METADATA_KEY]: {
1437
- ...promptfooMetadata ?? {},
1438
- [TRACE_LINKAGE_KEY]: {
1439
- traceId,
1440
- evaluationId
1441
- }
1442
- }
1443
- };
1444
- }
1445
- function stripTraceLinkageFromMetadata(metadata) {
1446
- const metadataRecord = asRecord(metadata);
1447
- const promptfooMetadata = asRecord(metadataRecord?.[PROMPTFOO_METADATA_KEY]);
1448
- if (!metadataRecord || !promptfooMetadata || !(TRACE_LINKAGE_KEY in promptfooMetadata)) return metadata;
1449
- const { [TRACE_LINKAGE_KEY]: _traceLinkage, ...remainingPromptfooMetadata } = promptfooMetadata;
1450
- const strippedMetadata = { ...metadataRecord };
1451
- delete strippedMetadata[PROMPTFOO_METADATA_KEY];
1452
- if (Object.keys(remainingPromptfooMetadata).length > 0) strippedMetadata[PROMPTFOO_METADATA_KEY] = remainingPromptfooMetadata;
1453
- return strippedMetadata;
1454
- }
1455
- function persistRepeatMetadata(metadata, repeatIndex, repeatGroupId) {
1456
- if (repeatIndex === void 0 || !repeatGroupId) return stripRepeatLinkageFromMetadata(metadata);
1457
- const metadataRecord = metadata ?? {};
1458
- const promptfooMetadata = asRecord(metadataRecord[PROMPTFOO_METADATA_KEY]);
1459
- if (metadataRecord["__promptfoo"] !== void 0 && promptfooMetadata === void 0) require_logger.logger.warn(`[EvalResult] Overwriting non-object metadata.${PROMPTFOO_METADATA_KEY} with internal repeat linkage; the key is reserved for promptfoo internals.`);
1460
- if (promptfooMetadata && REPEAT_LINKAGE_KEY in promptfooMetadata) require_logger.logger.warn(`[EvalResult] Overwriting metadata.${PROMPTFOO_METADATA_KEY}.${REPEAT_LINKAGE_KEY} with internal repeat linkage; the path is reserved for promptfoo internals.`);
1461
- return {
1462
- ...metadataRecord,
1463
- [PROMPTFOO_METADATA_KEY]: {
1464
- ...promptfooMetadata ?? {},
1465
- [REPEAT_LINKAGE_KEY]: {
1466
- repeatIndex,
1467
- repeatGroupId
1468
- }
1469
- }
1470
- };
1471
- }
1472
- function stripRepeatLinkageFromMetadata(metadata) {
1473
- const metadataRecord = asRecord(metadata);
1474
- const promptfooMetadata = asRecord(metadataRecord?.[PROMPTFOO_METADATA_KEY]);
1475
- if (!metadataRecord || !promptfooMetadata || !(REPEAT_LINKAGE_KEY in promptfooMetadata)) return metadata;
1476
- const { [REPEAT_LINKAGE_KEY]: _repeatLinkage, ...remainingPromptfooMetadata } = promptfooMetadata;
1477
- const strippedMetadata = { ...metadataRecord };
1478
- delete strippedMetadata[PROMPTFOO_METADATA_KEY];
1479
- if (Object.keys(remainingPromptfooMetadata).length > 0) strippedMetadata[PROMPTFOO_METADATA_KEY] = remainingPromptfooMetadata;
1480
- return strippedMetadata;
1481
- }
1482
- function surfaceRepeatMetadata(metadata) {
1483
- const metadataRecord = metadata ?? {};
1484
- const promptfooMetadata = asRecord(metadataRecord[PROMPTFOO_METADATA_KEY]);
1485
- const repeatLinkage = asRecord(promptfooMetadata?.[REPEAT_LINKAGE_KEY]);
1486
- return {
1487
- repeatIndex: typeof repeatLinkage?.repeatIndex === "number" && Number.isInteger(repeatLinkage.repeatIndex) && repeatLinkage.repeatIndex >= 0 ? repeatLinkage.repeatIndex : void 0,
1488
- repeatGroupId: typeof repeatLinkage?.repeatGroupId === "string" && repeatLinkage.repeatGroupId.length > 0 ? repeatLinkage.repeatGroupId : void 0,
1489
- metadata: promptfooMetadata != null && REPEAT_LINKAGE_KEY in promptfooMetadata ? stripRepeatLinkageFromMetadata(metadataRecord) : metadataRecord
1490
- };
1491
- }
1492
- function surfaceTraceMetadata(metadata) {
1493
- const metadataRecord = metadata ?? {};
1494
- const promptfooMetadata = asRecord(metadataRecord[PROMPTFOO_METADATA_KEY]);
1495
- const traceLinkage = asRecord(promptfooMetadata?.[TRACE_LINKAGE_KEY]);
1496
- const traceId = typeof traceLinkage?.traceId === "string" ? traceLinkage.traceId : void 0;
1497
- const evaluationId = typeof traceLinkage?.evaluationId === "string" ? traceLinkage.evaluationId : void 0;
1498
- if (!(promptfooMetadata != null && TRACE_LINKAGE_KEY in promptfooMetadata)) return {
1499
- traceId,
1500
- evaluationId,
1501
- metadata: metadataRecord
1502
- };
1503
- return {
1504
- traceId,
1505
- evaluationId,
1506
- metadata: stripTraceLinkageFromMetadata(metadataRecord)
1507
- };
1508
- }
1509
- function redactSensitiveResultFieldsForDb(fields) {
1510
- return {
1511
- response: sanitizeResponseForDb(fields.response),
1512
- gradingResult: sanitizeGradingResultForDb(fields.gradingResult),
1513
- metadata: sanitizeMetadataForDb(fields.metadata, fields.response?.metadata)
1514
- };
1515
- }
1516
- function getStripFlags(env) {
1517
- const getFlag = (key) => {
1518
- const value = env?.[key];
1519
- return value === void 0 ? require_logger.getEnvBool(key, false) : require_logger.parseEnvBool(String(value), false);
1520
- };
1521
- return {
1522
- shouldStripPromptText: getFlag("PROMPTFOO_STRIP_PROMPT_TEXT"),
1523
- shouldStripResponseOutput: getFlag("PROMPTFOO_STRIP_RESPONSE_OUTPUT"),
1524
- shouldStripTestVars: getFlag("PROMPTFOO_STRIP_TEST_VARS"),
1525
- shouldStripGradingResult: getFlag("PROMPTFOO_STRIP_GRADING_RESULT"),
1526
- shouldStripMetadata: getFlag("PROMPTFOO_STRIP_METADATA")
1527
- };
1528
- }
1529
- /**
1530
- * Sanitize a result before it is serialized into a JSONL output artifact. This is the
1531
- * JSONL-boundary equivalent of the database-persistence sanitization and must stay in sync
1532
- * with it: it redacts credential-bearing HTTP headers from the response / grading / metadata
1533
- * and applies the `PROMPTFOO_STRIP_*` projections (prompt text, response output, test vars,
1534
- * grading result, metadata). In-memory rows keep their real values for hooks; only the
1535
- * on-disk copy is sanitized.
1536
- */
1537
- function sanitizeResultForJsonlArtifact(result, stripFlags = getStripFlags()) {
1538
- const { shouldStripPromptText, shouldStripResponseOutput, shouldStripTestVars, shouldStripGradingResult, shouldStripMetadata } = stripFlags;
1539
- const artifactResult = serializeResultProviderRefs(result);
1540
- const redacted = redactSensitiveResultFieldsForDb({
1541
- response: sanitizeForDb(artifactResult.response),
1542
- gradingResult: sanitizeForDb(artifactResult.gradingResult),
1543
- metadata: sanitizeForDb(artifactResult.metadata)
1544
- });
1545
- const response = projectProviderResponse(redacted.response ?? void 0, {
1546
- stripMetadata: shouldStripMetadata,
1547
- stripOutput: shouldStripResponseOutput
1548
- });
1549
- return {
1550
- ...result,
1551
- ...artifactResult.testCase ? { testCase: projectTestCase(sanitizeForDbWithSecrets(artifactResult.testCase), {
1552
- stripMetadata: shouldStripMetadata,
1553
- stripVars: shouldStripTestVars,
1554
- stripOutput: shouldStripResponseOutput,
1555
- stripPromptText: shouldStripPromptText
1556
- }) } : {},
1557
- ...artifactResult.vars === void 0 ? {} : { vars: shouldStripTestVars ? {} : sanitizeForDbWithSecrets(artifactResult.vars) },
1558
- ...artifactResult.prompt ? { prompt: projectPrompt(sanitizeForDbWithSecrets(artifactResult.prompt), shouldStripPromptText) } : {},
1559
- ...artifactResult.provider ? { provider: sanitizeProvider(artifactResult.provider) } : {},
1560
- response,
1561
- gradingResult: shouldStripGradingResult ? null : projectGradingResult(redacted.gradingResult, shouldStripPromptText),
1562
- namedScores: sanitizeForDb(artifactResult.namedScores),
1563
- metadata: shouldStripMetadata ? {} : projectOutputMetadata(redacted.metadata, shouldStripResponseOutput, redacted.response?.metadata, artifactResult.testCase?.metadata)
1564
- };
1565
- }
1566
- var EvalResult = class EvalResult {
1567
- static async createFromEvaluateResult(evalId, result, opts) {
1568
- const persist = opts?.persist == null ? true : opts.persist;
1569
- const { prompt, error, score, latencyMs, success, provider, gradingResult, namedScores, cost, metadata, failureReason, testCase, traceId, evaluationId, repeatIndex, repeatGroupId } = serializeResultProviderRefs(result);
1570
- const persistedMetadata = persistTraceMetadata(persistRepeatMetadata(metadata, repeatIndex, repeatGroupId), traceId, evaluationId);
1571
- const processedResponse = await require_extractor.extractAndStoreBinaryData(result.response, {
1572
- evalId,
1573
- testIdx: result.testIdx,
1574
- promptIdx: result.promptIdx
1575
- });
1576
- const args = {
1577
- id: crypto.randomUUID(),
1578
- evalId,
1579
- testCase: sanitizeForDbWithSecrets(testCase),
1580
- promptIdx: result.promptIdx,
1581
- testIdx: result.testIdx,
1582
- prompt: sanitizeForDbWithSecrets(prompt),
1583
- promptId: hashPrompt(prompt),
1584
- error: error?.toString(),
1585
- success,
1586
- score: score == null ? 0 : score,
1587
- response: sanitizeForDb(processedResponse || null),
1588
- gradingResult: sanitizeForDb(gradingResult || null),
1589
- namedScores: sanitizeForDb(namedScores),
1590
- provider: sanitizeProvider(provider),
1591
- latencyMs,
1592
- cost,
1593
- metadata: sanitizeForDb(persistedMetadata),
1594
- failureReason
1595
- };
1596
- if (persist) {
1597
- const db = await require_tables.getDb();
1598
- const redacted = redactSensitiveResultFieldsForDb({
1599
- response: args.response,
1600
- gradingResult: args.gradingResult,
1601
- metadata: args.metadata
1602
- });
1603
- args.response = redacted.response;
1604
- args.gradingResult = redacted.gradingResult;
1605
- args.metadata = redacted.metadata;
1606
- const dbResult = await db.insert(require_tables.evalResultsTable).values(args).returning();
1607
- clearCountCache(evalId);
1608
- return new EvalResult({
1609
- ...dbResult[0],
1610
- persisted: true
1611
- });
1612
- }
1613
- return new EvalResult(args);
1614
- }
1615
- static async createManyFromEvaluateResult(results, evalId) {
1616
- const db = await require_tables.getDb();
1617
- const returnResults = [];
1618
- const processedResults = [];
1619
- for (const result of results) {
1620
- const processedResponse = require_extractor.isBlobStorageEnabled() ? await require_extractor.extractAndStoreBinaryData(result.response, {
1621
- evalId,
1622
- testIdx: result.testIdx,
1623
- promptIdx: result.promptIdx
1624
- }) : result.response;
1625
- processedResults.push({
1626
- ...serializeResultProviderRefs(result),
1627
- response: processedResponse ?? void 0
1628
- });
1629
- }
1630
- await db.transaction(async (tx) => {
1631
- for (const result of processedResults) {
1632
- const { traceId: _traceId, evaluationId: _evaluationId, repeatIndex: _repeatIndex, repeatGroupId: _repeatGroupId, ...rest } = result;
1633
- const sanitizedResult = {
1634
- ...rest,
1635
- testCase: sanitizeForDbWithSecrets(result.testCase),
1636
- prompt: sanitizeForDbWithSecrets(result.prompt),
1637
- ...redactSensitiveResultFieldsForDb({
1638
- response: sanitizeForDb(result.response),
1639
- gradingResult: sanitizeForDb(result.gradingResult),
1640
- metadata: sanitizeForDb(persistTraceMetadata(persistRepeatMetadata(result.metadata, result.repeatIndex, result.repeatGroupId), result.traceId, result.evaluationId))
1641
- }),
1642
- namedScores: sanitizeForDb(result.namedScores),
1643
- provider: result.provider ? sanitizeProvider(result.provider) : result.provider
1644
- };
1645
- const dbResult = await tx.insert(require_tables.evalResultsTable).values({
1646
- ...sanitizedResult,
1647
- evalId,
1648
- id: crypto.randomUUID()
1649
- }).returning().get();
1650
- returnResults.push(new EvalResult({
1651
- ...dbResult,
1652
- persisted: true
1653
- }));
1654
- }
1655
- });
1656
- clearCountCache(evalId);
1657
- return returnResults;
1658
- }
1659
- static async findById(id) {
1660
- const result = await (await require_tables.getDb()).select().from(require_tables.evalResultsTable).where((0, drizzle_orm.eq)(require_tables.evalResultsTable.id, id));
1661
- return result.length > 0 ? new EvalResult({
1662
- ...result[0],
1663
- persisted: true
1664
- }) : null;
1665
- }
1666
- static async findManyByEvalId(evalId, opts) {
1667
- return (await (await require_tables.getDb()).select().from(require_tables.evalResultsTable).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, evalId), opts?.testIdx == null ? void 0 : (0, drizzle_orm.eq)(require_tables.evalResultsTable.testIdx, opts.testIdx)))).map((result) => new EvalResult({
1668
- ...result,
1669
- persisted: true
1670
- }));
1671
- }
1672
- static async findManyByEvalIdAndTestIndices(evalId, testIndices) {
1673
- if (!testIndices.length) return [];
1674
- return (await (await require_tables.getDb()).select().from(require_tables.evalResultsTable).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, evalId), testIndices.length === 1 ? (0, drizzle_orm.eq)(require_tables.evalResultsTable.testIdx, testIndices[0]) : (0, drizzle_orm.inArray)(require_tables.evalResultsTable.testIdx, testIndices)))).map((result) => new EvalResult({
1675
- ...result,
1676
- persisted: true
1677
- }));
1678
- }
1679
- /**
1680
- * Returns a set of completed (testIdx,promptIdx) pairs for a given eval.
1681
- * Key format: `${testIdx}:${promptIdx}`
1682
- *
1683
- * @param evalId - The evaluation ID to query
1684
- * @param opts.excludeErrors - If true, excludes results with ERROR failureReason (used in retry mode)
1685
- */
1686
- static async getCompletedIndexPairs(evalId, opts) {
1687
- const db = await require_tables.getDb();
1688
- const whereClause = opts?.excludeErrors ? (0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, evalId), (0, drizzle_orm.ne)(require_tables.evalResultsTable.failureReason, require_tables.ResultFailureReason.ERROR)) : (0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, evalId);
1689
- const rows = await db.select({
1690
- testIdx: require_tables.evalResultsTable.testIdx,
1691
- promptIdx: require_tables.evalResultsTable.promptIdx
1692
- }).from(require_tables.evalResultsTable).where(whereClause);
1693
- const ret = /* @__PURE__ */ new Set();
1694
- for (const r of rows) ret.add(`${r.testIdx}:${r.promptIdx}`);
1695
- return ret;
1696
- }
1697
- static async *findManyByEvalIdBatched(evalId, opts) {
1698
- const db = await require_tables.getDb();
1699
- const batchSize = opts?.batchSize || 100;
1700
- let offset = 0;
1701
- while (true) {
1702
- const nextResult = await db.select({ testIdx: require_tables.evalResultsTable.testIdx }).from(require_tables.evalResultsTable).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, evalId), (0, drizzle_orm.gte)(require_tables.evalResultsTable.testIdx, offset))).orderBy(require_tables.evalResultsTable.testIdx).limit(1).get();
1703
- if (!nextResult) break;
1704
- offset = nextResult.testIdx;
1705
- yield (await db.select().from(require_tables.evalResultsTable).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, evalId), (0, drizzle_orm.gte)(require_tables.evalResultsTable.testIdx, offset), (0, drizzle_orm.lt)(require_tables.evalResultsTable.testIdx, offset + batchSize))).all()).map((result) => new EvalResult({
1706
- ...result,
1707
- persisted: true
1708
- }));
1709
- offset += batchSize;
1710
- }
1711
- }
1712
- id;
1713
- evalId;
1714
- description;
1715
- promptIdx;
1716
- testIdx;
1717
- testCase;
1718
- prompt;
1719
- promptId;
1720
- error;
1721
- success;
1722
- score;
1723
- response;
1724
- gradingResult;
1725
- namedScores;
1726
- provider;
1727
- latencyMs;
1728
- cost;
1729
- metadata;
1730
- traceId;
1731
- evaluationId;
1732
- repeatIndex;
1733
- repeatGroupId;
1734
- failureReason;
1735
- persisted;
1736
- pluginId;
1737
- constructor(opts) {
1738
- this.id = opts.id;
1739
- this.evalId = opts.evalId;
1740
- this.promptIdx = opts.promptIdx;
1741
- this.testIdx = opts.testIdx;
1742
- this.testCase = opts.testCase;
1743
- this.prompt = opts.prompt;
1744
- this.promptId = opts.promptId || hashPrompt(opts.prompt);
1745
- this.error = opts.error;
1746
- this.score = opts.score;
1747
- this.success = opts.success;
1748
- this.response = opts.response || void 0;
1749
- this.gradingResult = opts.gradingResult;
1750
- this.namedScores = opts.namedScores || {};
1751
- this.provider = opts.provider;
1752
- this.latencyMs = opts.latencyMs || 0;
1753
- this.cost = opts.cost || 0;
1754
- const surfacedTrace = surfaceTraceMetadata(opts.metadata);
1755
- const surfacedRepeat = surfaceRepeatMetadata(surfacedTrace.metadata);
1756
- this.metadata = surfacedRepeat.metadata;
1757
- this.traceId = surfacedTrace.traceId;
1758
- this.evaluationId = surfacedTrace.evaluationId;
1759
- this.repeatIndex = surfacedRepeat.repeatIndex;
1760
- this.repeatGroupId = surfacedRepeat.repeatGroupId;
1761
- this.failureReason = require_tables.isResultFailureReason(opts.failureReason) ? opts.failureReason : require_tables.ResultFailureReason.NONE;
1762
- this.persisted = opts.persisted || false;
1763
- this.pluginId = opts.testCase.metadata?.pluginId;
1764
- }
1765
- async save() {
1766
- const db = await require_tables.getDb();
1767
- const { traceId: _traceId, evaluationId: _evaluationId, repeatIndex: _repeatIndex, repeatGroupId: _repeatGroupId, pluginId: _pluginId, ...rest } = serializeResultProviderRefs(this);
1768
- const persistedValues = {
1769
- ...rest,
1770
- error: this.error ?? null,
1771
- gradingResult: sanitizeGradingResultForDb(rest.gradingResult),
1772
- metadata: persistTraceMetadata(persistRepeatMetadata(this.metadata, this.repeatIndex, this.repeatGroupId), this.traceId, this.evaluationId)
1773
- };
1774
- if (this.persisted) await db.update(require_tables.evalResultsTable).set({
1775
- ...persistedValues,
1776
- updatedAt: require_time.getCurrentTimestamp()
1777
- }).where((0, drizzle_orm.eq)(require_tables.evalResultsTable.id, this.id)).run();
1778
- else {
1779
- const result = await db.insert(require_tables.evalResultsTable).values(persistedValues).returning();
1780
- this.id = result[0].id;
1781
- this.persisted = true;
1782
- }
1783
- invalidateEvaluationCache(this.evalId);
1784
- }
1785
- toEvaluateResult(stripFlags = getStripFlags()) {
1786
- const { shouldStripPromptText, shouldStripResponseOutput, shouldStripTestVars, shouldStripGradingResult, shouldStripMetadata } = stripFlags;
1787
- const response = projectProviderResponse(this.response, {
1788
- stripMetadata: shouldStripMetadata,
1789
- stripOutput: shouldStripResponseOutput
1790
- });
1791
- const prompt = projectPrompt(this.prompt, shouldStripPromptText);
1792
- const testCase = projectTestCase(this.testCase, {
1793
- stripMetadata: shouldStripMetadata,
1794
- stripVars: shouldStripTestVars,
1795
- stripOutput: shouldStripResponseOutput,
1796
- stripPromptText: shouldStripPromptText
1797
- });
1798
- const tokenUsage = require_tokenUsageUtils.createEmptyTokenUsage();
1799
- if (this.response) require_tokenUsageUtils.accumulateResponseTokenUsage(tokenUsage, this.response);
1800
- if (this.gradingResult) require_tokenUsageUtils.accumulateGradingTokenUsage(tokenUsage, this.gradingResult.tokensUsed, { cached: this.gradingResult.metadata?.cachedResponse });
1801
- return {
1802
- cost: this.cost,
1803
- ...this.response?.incurredCost !== void 0 && { incurredCost: this.response.incurredCost },
1804
- description: this.description || void 0,
1805
- error: this.error || void 0,
1806
- gradingResult: shouldStripGradingResult ? null : projectGradingResult(this.gradingResult, shouldStripPromptText),
1807
- id: this.id,
1808
- latencyMs: this.latencyMs,
1809
- namedScores: this.namedScores,
1810
- prompt,
1811
- promptId: this.promptId,
1812
- promptIdx: this.promptIdx,
1813
- ...this.traceId ? { traceId: this.traceId } : {},
1814
- ...this.evaluationId ? { evaluationId: this.evaluationId } : {},
1815
- ...this.repeatGroupId !== void 0 && {
1816
- repeatGroupId: this.repeatGroupId,
1817
- repeatIndex: this.repeatIndex
1818
- },
1819
- provider: {
1820
- id: this.provider.id,
1821
- label: this.provider.label
1822
- },
1823
- response,
1824
- score: this.score,
1825
- success: this.success,
1826
- testCase,
1827
- testIdx: this.testIdx,
1828
- tokenUsage,
1829
- vars: shouldStripTestVars ? {} : this.testCase.vars || {},
1830
- metadata: shouldStripMetadata ? {} : projectOutputMetadata(this.metadata, shouldStripResponseOutput, this.response?.metadata, this.testCase.metadata),
1831
- failureReason: this.failureReason
1832
- };
1833
- }
1834
- };
1835
- /** Normalize an `EvalResult` model instance or a plain `EvaluateResult` to `EvaluateResult`. */
1836
- function asEvaluateResult(result, stripFlags = getStripFlags()) {
1837
- return "toEvaluateResult" in result ? result.toEvaluateResult(stripFlags) : result;
1838
- }
1839
- /** Canonical `testIdx:promptIdx` key used to dedupe/look up a result across the streaming,
1840
- * recovery, and comparison paths. */
1841
- function getResultIndexKey(result) {
1842
- return `${result.testIdx}:${result.promptIdx}`;
1843
- }
1844
- //#endregion
1845
- //#region src/util/eval/evalTableUtils.ts
1846
- /**
1847
- * Keep this in its current order, as it is used to map the columns in the CSV, so it needs to be static.
1848
- *
1849
- * The keys are the names of the columns in the metadata object, and the values are the names of the columns in the CSV.
1850
- *
1851
- * This is imported by enterprise so it doesn't need to be copied.
1852
- */
1853
- const REDTEAM_METADATA_KEYS_TO_CSV_COLUMN_NAMES = {
1854
- messages: "Messages",
1855
- redteamHistory: "RedteamHistory",
1856
- redteamTreeHistory: "RedteamTreeHistory",
1857
- pluginId: "pluginId",
1858
- strategyId: "strategyId",
1859
- sessionId: "sessionId",
1860
- sessionIds: "sessionIds"
1861
- };
1862
- const REDTEAM_METADATA_COLUMNS = Object.values(REDTEAM_METADATA_KEYS_TO_CSV_COLUMN_NAMES);
1863
- /**
1864
- * Get the status string for an output
1865
- */
1866
- function getOutputStatus(output) {
1867
- if (output.pass) return "PASS";
1868
- return output.failureReason === require_tables.ResultFailureReason.ASSERT ? "FAIL" : "ERROR";
1869
- }
1870
- /**
1871
- * Format named scores for CSV output.
1872
- * Returns empty string if no named scores, otherwise JSON string.
1873
- */
1874
- function formatNamedScores(namedScores) {
1875
- if (!namedScores || Object.keys(namedScores).length === 0) return "";
1876
- const rounded = {};
1877
- for (const [key, value] of Object.entries(namedScores)) if (typeof value === "number" && !Number.isNaN(value)) rounded[key] = Number(value.toFixed(2));
1878
- if (Object.keys(rounded).length === 0) return "";
1879
- return JSON.stringify(rounded);
1880
- }
1881
- function formatNamedScoreValue(value) {
1882
- return typeof value === "number" && !Number.isNaN(value) ? value.toFixed(2) : "";
1883
- }
1884
- /**
1885
- * Build a per-prompt list of `Metric: <name>` column names, deduplicated and
1886
- * sorted with `localeCompare`. Shared shell for both the row-scanning helper
1887
- * (WebUI path) and the aggregate-reading helper (streaming CLI path); see the
1888
- * two call sites below.
1889
- */
1890
- function collectMetricNamesByPrompt(items, namesFor) {
1891
- return items.map((item, index) => [...new Set(namesFor(item, index))].sort((a, b) => a.localeCompare(b)));
1892
- }
1893
- /**
1894
- * Collect per-row named metric names from prompt-level aggregate metrics, for
1895
- * use by the streaming CSV path which cannot scan rows up front.
1896
- *
1897
- * Uses `namedScoresCount` because it is incremented only when a per-row
1898
- * result contributes a metric (`src/util/namedMetrics.ts`). Derived metrics
1899
- * write to `namedScores` but not `namedScoresCount` (`src/evaluator.ts`
1900
- * `updateDerivedMetrics`), so this keeps the streaming column set aligned
1901
- * with what `collectNamedScoreNamesByPrompt` would produce from row outputs.
1902
- *
1903
- * Returns an empty list both for prompts with no per-row contributions
1904
- * (`namedScoresCount: {}` — modern eval with no named metrics configured)
1905
- * and for prompts whose count map is missing entirely (legacy/imported
1906
- * evals). Use `promptNeedsDiscoveryFallback` to distinguish the two: only
1907
- * the latter requires a row-scan pre-pass.
1908
- */
1909
- function collectNamedScoreNamesByPromptFromAggregate(prompts) {
1910
- return collectMetricNamesByPrompt(prompts, (prompt) => {
1911
- const counts = prompt.metrics?.namedScoresCount;
1912
- if (!counts) return [];
1913
- return Object.keys(counts).filter((name) => (counts[name] ?? 0) > 0);
1914
- });
1915
- }
1916
- /**
1917
- * True when a prompt's aggregate `metrics.namedScoresCount` is missing/absent,
1918
- * not merely empty. Legacy evals persisted before count tracking shipped, and
1919
- * external imports via `POST /api/eval` that don't backfill the count, both
1920
- * land here. Modern evals always have an object (possibly empty) and skip the
1921
- * discovery pass.
1922
- */
1923
- function promptNeedsDiscoveryFallback(prompt) {
1924
- const metrics = prompt.metrics;
1925
- return !metrics || !metrics.namedScoresCount;
1926
- }
1927
- /**
1928
- * Scan one batch of `EvalResult` rows for the metric names that will need
1929
- * dedicated `Metric: <name>` columns. Only inspects the prompts in
1930
- * `fallbackPromptIndices` (those whose aggregate `namedScoresCount` is
1931
- * missing/empty), and only touches `namedScores` + `testCase.description` —
1932
- * not the full row shape — so the discovery pass stays allocation-light even
1933
- * for very large legacy evals.
1934
- *
1935
- * Mirrors the row-output filter in `collectNamedScoreNamesByPrompt` so the
1936
- * streaming CSV column set matches the WebUI CSV column set.
1937
- */
1938
- function scanBatchForDiscovery(batchResults, acc, fallbackPromptIndices) {
1939
- for (const result of batchResults) {
1940
- if (!acc.hasDescriptions && result.testCase?.description) acc.hasDescriptions = true;
1941
- if (!fallbackPromptIndices.has(result.promptIdx)) continue;
1942
- const namedScores = result.namedScores;
1943
- if (!namedScores) continue;
1944
- let names = acc.metricNamesByPrompt.get(result.promptIdx);
1945
- if (!names) {
1946
- names = /* @__PURE__ */ new Set();
1947
- acc.metricNamesByPrompt.set(result.promptIdx, names);
1948
- }
1949
- for (const [name, value] of Object.entries(namedScores)) if (typeof value === "number" && !Number.isNaN(value)) names.add(name);
1950
- }
1951
- }
1952
- /**
1953
- * Convert a batch of `EvalResult` rows (grouped by testIdx) into the table-row
1954
- * shape that `tableRowToCsvValues` accepts. Used by `streamEvalCsv` to keep its
1955
- * per-batch processing tight; the outputs array is pre-sized so columns line up
1956
- * with prompts even when results arrive out of promptIdx order.
1957
- */
1958
- function batchToStreamRows(batchResults, varNames, numPrompts) {
1959
- const rowsByTestIdx = /* @__PURE__ */ new Map();
1960
- for (const result of batchResults) {
1961
- let row = rowsByTestIdx.get(result.testIdx);
1962
- if (!row) {
1963
- row = {
1964
- vars: varNames.map((varName) => {
1965
- const value = result.testCase?.vars?.[varName];
1966
- return value === void 0 ? "" : String(value);
1967
- }),
1968
- outputs: new Array(numPrompts).fill(null),
1969
- test: { description: result.testCase?.description }
1970
- };
1971
- rowsByTestIdx.set(result.testIdx, row);
1972
- }
1973
- row.outputs[result.promptIdx] = {
1974
- text: result.response?.output ?? "",
1975
- pass: result.success,
1976
- score: result.score,
1977
- namedScores: result.namedScores,
1978
- failureReason: result.failureReason,
1979
- gradingResult: result.gradingResult,
1980
- metadata: result.metadata
1981
- };
1982
- }
1983
- return Array.from(rowsByTestIdx.values());
1984
- }
1985
- /**
1986
- * Build CSV headers for an evaluation table.
1987
- *
1988
- * @param vars - Variable names from the table head
1989
- * @param prompts - Prompt definitions from the table head
1990
- * @param options - Export options
1991
- * @param options.namedScoreNamesByPrompt - Named metric names that should get
1992
- * dedicated `Metric: <name>` columns, one sorted list per prompt (aligned
1993
- * with `prompts`). Must match the value passed to `tableRowToCsvValues` so
1994
- * header and row column counts stay in sync.
1995
- * @returns Array of header strings
1996
- */
1997
- function buildCsvHeaders(vars, prompts, options = {}) {
1998
- const headers = [
1999
- ...options.hasDescriptions ? ["Description"] : [],
2000
- ...vars,
2001
- ...prompts.flatMap((prompt, promptIndex) => {
2002
- const provider = prompt.provider || "";
2003
- return [
2004
- provider ? `[${provider}] ${prompt.label}` : prompt.label,
2005
- "Status",
2006
- "Score",
2007
- "Named Scores",
2008
- ...(options.namedScoreNamesByPrompt?.[promptIndex] || []).map((name) => `Metric: ${name}`),
2009
- "Grader Reason",
2010
- "Comment"
2011
- ];
2012
- })
2013
- ];
2014
- if (options.isRedteam) headers.push(...REDTEAM_METADATA_COLUMNS);
2015
- return headers;
2016
- }
2017
- /**
2018
- * Convert a single table row to CSV row values.
2019
- *
2020
- * @param row - The table row to convert
2021
- * @param options - Export options (`namedScoreNamesByPrompt` must match the
2022
- * array passed to `buildCsvHeaders`)
2023
- * @returns Array of values for the CSV row
2024
- */
2025
- function tableRowToCsvValues(row, options = {}) {
2026
- const rowValues = [
2027
- ...options.hasDescriptions ? [row.test.description || ""] : [],
2028
- ...row.vars,
2029
- ...row.outputs.flatMap((output, outputIndex) => {
2030
- const namedScoreNames = options.namedScoreNamesByPrompt?.[outputIndex] || [];
2031
- if (!output) return [
2032
- "",
2033
- "",
2034
- "",
2035
- "",
2036
- ...namedScoreNames.map(() => ""),
2037
- "",
2038
- ""
2039
- ];
2040
- const status = getOutputStatus(output);
2041
- const score = output.score?.toFixed(2) ?? "";
2042
- const namedScores = formatNamedScores(output.namedScores);
2043
- const namedScoreValues = namedScoreNames.map((name) => formatNamedScoreValue(output.namedScores?.[name]));
2044
- return [
2045
- output.text || "",
2046
- status,
2047
- score,
2048
- namedScores,
2049
- ...namedScoreValues,
2050
- output.gradingResult?.reason || "",
2051
- output.gradingResult?.comment || ""
2052
- ];
2053
- })
2054
- ];
2055
- if (options.isRedteam) {
2056
- const redteamKeys = Object.keys(REDTEAM_METADATA_KEYS_TO_CSV_COLUMN_NAMES);
2057
- const firstOutputMetadata = row.outputs[0]?.metadata;
2058
- for (const key of redteamKeys) {
2059
- let value = firstOutputMetadata?.[key];
2060
- if (key === "strategyId" && (value === null || value === void 0)) value = "basic";
2061
- if (value === null || value === void 0) rowValues.push("");
2062
- else if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") rowValues.push(value.toString());
2063
- else rowValues.push(JSON.stringify(value));
2064
- }
2065
- }
2066
- return rowValues;
2067
- }
2068
- /**
2069
- * Escape every cell of a CSV row matrix (headers + body) against spreadsheet
2070
- * formula injection (CWE-1236), unless disabled via
2071
- * `PROMPTFOO_DISABLE_CSV_FORMULA_ESCAPING`. Numbers and booleans pass through
2072
- * untouched; only string cells are inspected. Applied at the `csvStringify`
2073
- * boundary so `buildCsvHeaders`/`tableRowToCsvValues` stay pure and re-importable
2074
- * CSVs are never produced from un-escaped data.
2075
- */
2076
- function escapeCsvRowsForFormulaInjection(rows, env) {
2077
- const configuredValue = env?.PROMPTFOO_DISABLE_CSV_FORMULA_ESCAPING ?? process.env.PROMPTFOO_DISABLE_CSV_FORMULA_ESCAPING;
2078
- if ([
2079
- "1",
2080
- "true",
2081
- "yes",
2082
- "yup",
2083
- "yeppers"
2084
- ].includes(configuredValue?.toLowerCase() ?? "")) return rows;
2085
- return rows.map((row) => row.map((cell) => typeof cell === "string" ? escapeCsvFormula(cell) : cell));
2086
- }
2087
- function isNumericCell(value) {
2088
- return value.trim() !== "" && Number.isFinite(Number(value));
2089
- }
2090
- /**
2091
- * Neutralize spreadsheet formula (CSV) injection at eval export time (CWE-1236).
2092
- *
2093
- * Spreadsheet applications execute cells beginning with `=`, `+`, `-`, or `@`
2094
- * as formulas. Exported eval data can contain attacker-controlled model output,
2095
- * variables, and grader comments, and CSV quoting alone does not neutralize
2096
- * those formulas. Prefix dangerous values with the spreadsheet-native text
2097
- * marker while preserving finite signed numbers.
2098
- *
2099
- * Leading whitespace and zero-width characters are ignored when identifying a
2100
- * trigger because spreadsheet applications can strip them before evaluation.
2101
- */
2102
- function escapeCsvFormula(value) {
2103
- if (value.length === 0) return value;
2104
- const unpadded = value.replace(/^[\s\u200B\u200C\u200D]+/, "");
2105
- const firstChar = unpadded[0];
2106
- if (firstChar === void 0) return value;
2107
- return firstChar === "=" || firstChar === "@" || (firstChar === "-" || firstChar === "+") && !isNumericCell(unpadded) ? `'${value}` : value;
2108
- }
2109
- /**
2110
- * Stream CSV data from an evaluation in batches.
2111
- *
2112
- * This is more memory-efficient for large evaluations as it processes
2113
- * results in batches rather than loading everything into memory.
2114
- *
2115
- * Used by the CLI export (`promptfoo eval -o output.csv`) to maintain
2116
- * consistent CSV format with WebUI exports while handling large datasets.
2117
- *
2118
- * @param eval_ - The evaluation to export
2119
- * @param options - Streaming options including the write callback
2120
- */
2121
- async function streamEvalCsv(eval_, options) {
2122
- const { isRedteam = false, write } = options;
2123
- const env = eval_.config?.env;
2124
- const varNames = eval_.vars;
2125
- const prompts = eval_.prompts;
2126
- const numPrompts = prompts.length;
2127
- const namedScoreNamesByPrompt = collectNamedScoreNamesByPromptFromAggregate(prompts);
2128
- const fallbackPromptIndices = prompts.flatMap((prompt, promptIndex) => promptNeedsDiscoveryFallback(prompt) ? [promptIndex] : []);
2129
- const fallbackPromptIndexSet = new Set(fallbackPromptIndices);
2130
- const accumulator = {
2131
- metricNamesByPrompt: /* @__PURE__ */ new Map(),
2132
- hasDescriptions: false
2133
- };
2134
- for await (const batchResults of eval_.fetchResultsBatched()) scanBatchForDiscovery(batchResults, accumulator, fallbackPromptIndexSet);
2135
- const hasDescriptions = accumulator.hasDescriptions;
2136
- for (const promptIndex of fallbackPromptIndices) {
2137
- const names = accumulator.metricNamesByPrompt.get(promptIndex);
2138
- namedScoreNamesByPrompt[promptIndex] = names ? [...names].sort((a, b) => a.localeCompare(b)) : [];
2139
- }
2140
- const headers = buildCsvHeaders(varNames, prompts, {
2141
- hasDescriptions,
2142
- isRedteam,
2143
- namedScoreNamesByPrompt
2144
- });
2145
- await write((0, csv_stringify_sync.stringify)(escapeCsvRowsForFormulaInjection([headers], env)));
2146
- for await (const batchResults of eval_.fetchResultsBatched()) {
2147
- const csvRows = batchToStreamRows(batchResults, varNames, numPrompts).map((row) => tableRowToCsvValues(row, {
2148
- hasDescriptions,
2149
- isRedteam,
2150
- namedScoreNamesByPrompt
2151
- }));
2152
- if (csvRows.length > 0) await write((0, csv_stringify_sync.stringify)(escapeCsvRowsForFormulaInjection(csvRows, env)));
2153
- }
2154
- }
2155
- //#endregion
2156
- //#region src/util/junit.ts
2157
- const MAX_JUNIT_NAME_LENGTH = 512;
2158
- const MAX_JUNIT_DETAIL_LENGTH = 8192;
2159
- const JUNIT_ASSERTION_FAILURE_MESSAGE = "Assertion failed";
2160
- const JUNIT_EVALUATION_ERROR_MESSAGE = "Evaluation error";
2161
- const INVALID_XML_CHARACTERS = /[^\t\n\r\u0020-\ud7ff\ue000-\ufffd\u{10000}-\u{10ffff}]/gu;
2162
- function truncateText(value, maxLength) {
2163
- if (value.length <= maxLength) return value;
2164
- const cutoff = maxLength - 3;
2165
- const end = (value.codePointAt(cutoff - 1) ?? 0) > 65535 ? cutoff + 1 : cutoff;
2166
- return `${value.slice(0, end)}...`;
2167
- }
2168
- function normalizeInlineText(value, fallback, maxLength = MAX_JUNIT_NAME_LENGTH) {
2169
- return truncateText((value ?? "").replace(/\s+/g, " ").replace(INVALID_XML_CHARACTERS, "").replace(/\s+/g, " ").trim() || fallback, maxLength);
2170
- }
2171
- function formatDurationSeconds(durationMs) {
2172
- return Number(((typeof durationMs === "number" && Number.isFinite(durationMs) && durationMs > 0 ? durationMs : 0) / 1e3).toFixed(3)).toString();
2173
- }
2174
- function getEvaluationTimestamp(evalRecord) {
2175
- const date = new Date(evalRecord.createdAt);
2176
- return Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
2177
- }
2178
- function getTestCaseName(result) {
2179
- const prefix = `test ${result.testIdx + 1}`;
2180
- const description = normalizeInlineText(result.description || result.testCase.description, "");
2181
- return description ? `${prefix}: ${description}` : prefix;
2182
- }
2183
- function getFailedComponentResults(gradingResult) {
2184
- return gradingResult?.componentResults?.filter((component) => !component.pass) ?? [];
2185
- }
2186
- function getAssertionLabel(gradingResult) {
2187
- return gradingResult.assertion?.type ?? "assertion";
2188
- }
2189
- function getFailedAssertionLabels(gradingResult) {
2190
- const failedComponents = getFailedComponentResults(gradingResult);
2191
- if (failedComponents.length > 0) return failedComponents.map(getAssertionLabel);
2192
- return gradingResult ? [getAssertionLabel(gradingResult)] : [];
2193
- }
2194
- function getFailureDetails(result) {
2195
- const lines = [`Score: ${result.score}`, `Reason: ${JUNIT_ASSERTION_FAILURE_MESSAGE}`];
2196
- const failedAssertionLabels = getFailedAssertionLabels(result.gradingResult);
2197
- if (failedAssertionLabels.length > 0) {
2198
- lines.push("Failed assertions:");
2199
- for (const label of failedAssertionLabels) lines.push(`- ${label}`);
2200
- }
2201
- return truncateText(lines.join("\n").replace(INVALID_XML_CHARACTERS, ""), MAX_JUNIT_DETAIL_LENGTH);
2202
- }
2203
- function projectEvalResult(result) {
2204
- const projected = "toEvaluateResult" in result ? result.toEvaluateResult() : result;
2205
- return {
2206
- description: projected.description,
2207
- error: projected.error,
2208
- failureReason: projected.failureReason,
2209
- gradingResult: projected.gradingResult,
2210
- latencyMs: projected.latencyMs,
2211
- prompt: projected.prompt,
2212
- promptId: projected.promptId,
2213
- promptIdx: projected.promptIdx,
2214
- provider: projected.provider,
2215
- score: projected.score,
2216
- success: projected.success,
2217
- testCase: projected.testCase,
2218
- testIdx: projected.testIdx
2219
- };
2220
- }
2221
- async function* iterateJunitProjectedResults(evalRecord) {
2222
- if (evalRecord.useOldResults()) {
2223
- const results = await evalRecord.getResults();
2224
- for (const result of results) yield projectEvalResult(result);
2225
- return;
2226
- }
2227
- if (!evalRecord.persisted) {
2228
- for (const result of evalRecord.results) yield projectEvalResult(result);
2229
- return;
2230
- }
2231
- for await (const batchResults of evalRecord.fetchResultsBatched()) for (const result of batchResults) yield projectEvalResult(result);
2232
- }
2233
- async function buildJunitSuites(evalRecord) {
2234
- const suites = /* @__PURE__ */ new Map();
2235
- const providersByDisplayName = /* @__PURE__ */ new Map();
2236
- for await (const result of iterateJunitProjectedResults(evalRecord)) {
2237
- const { provider } = result;
2238
- const providerKey = JSON.stringify([provider.id ?? "", provider.label ?? ""]);
2239
- const promptKey = result.promptId || `prompt-index:${result.promptIdx}`;
2240
- const key = JSON.stringify([providerKey, promptKey]);
2241
- let suite = suites.get(key);
2242
- if (!suite) {
2243
- const rawName = provider.label || provider.id || "";
2244
- const displayName = normalizeInlineText(rawName, "unknown provider");
2245
- const providers = providersByDisplayName.get(displayName) ?? /* @__PURE__ */ new Set();
2246
- providers.add(providerKey);
2247
- providersByDisplayName.set(displayName, providers);
2248
- suite = {
2249
- providerKey,
2250
- rawName,
2251
- promptKey,
2252
- promptIdx: result.promptIdx,
2253
- displayName: "",
2254
- errors: 0,
2255
- failures: 0,
2256
- testcases: [],
2257
- timeMs: 0
2258
- };
2259
- suites.set(key, suite);
2260
- }
2261
- suite.promptIdx = Math.min(suite.promptIdx, result.promptIdx);
2262
- suite.testcases.push({
2263
- testcase: buildJunitTestCase(result),
2264
- testIdx: result.testIdx
2265
- });
2266
- suite.timeMs += result.latencyMs;
2267
- if (!result.success) {
2268
- if (result.failureReason === require_tables.ResultFailureReason.ASSERT) suite.failures += 1;
2269
- else suite.errors += 1;
2270
- }
2271
- }
2272
- const promptCountsByProvider = /* @__PURE__ */ new Map();
2273
- const orderedSuites = [...suites.values()].sort((a, b) => a.promptIdx - b.promptIdx || a.promptKey.localeCompare(b.promptKey));
2274
- for (const suite of orderedSuites) {
2275
- const baseName = normalizeInlineText(suite.rawName, "unknown provider");
2276
- const suffix = (providersByDisplayName.get(baseName)?.size ?? 0) > 1 || suite.rawName.search(INVALID_XML_CHARACTERS) !== -1 ? ` (${require_createHash.sha256(suite.providerKey).slice(0, 16)})` : "";
2277
- const providerName = normalizeInlineText(suite.rawName, "unknown provider", MAX_JUNIT_NAME_LENGTH - suffix.length);
2278
- const ordinal = (promptCountsByProvider.get(suite.providerKey) ?? 0) + 1;
2279
- promptCountsByProvider.set(suite.providerKey, ordinal);
2280
- suite.displayName = `[${providerName}] prompt ${ordinal}${suffix}`;
2281
- for (const { testcase } of suite.testcases) testcase["@_classname"] = suite.displayName;
2282
- }
2283
- return [...suites.values()];
2284
- }
2285
- function buildJunitTestCase(result) {
2286
- const testcase = {
2287
- "@_name": getTestCaseName(result),
2288
- "@_time": formatDurationSeconds(result.latencyMs)
2289
- };
2290
- if (!result.success) {
2291
- if (result.failureReason === require_tables.ResultFailureReason.ASSERT) testcase.failure = {
2292
- "#text": getFailureDetails(result),
2293
- "@_message": JUNIT_ASSERTION_FAILURE_MESSAGE,
2294
- "@_type": "assertion"
2295
- };
2296
- else testcase.error = {
2297
- "#text": `Reason: ${JUNIT_EVALUATION_ERROR_MESSAGE}`,
2298
- "@_message": JUNIT_EVALUATION_ERROR_MESSAGE,
2299
- "@_type": "error"
2300
- };
2301
- }
2302
- return testcase;
2303
- }
2304
- async function createJunitXml(evalRecord) {
2305
- const suites = await buildJunitSuites(evalRecord);
2306
- const tests = suites.reduce((sum, suite) => sum + suite.testcases.length, 0);
2307
- const failures = suites.reduce((sum, suite) => sum + suite.failures, 0);
2308
- const errors = suites.reduce((sum, suite) => sum + suite.errors, 0);
2309
- const totalTimeMs = suites.reduce((sum, suite) => sum + suite.timeMs, 0);
2310
- const timestamp = getEvaluationTimestamp(evalRecord);
2311
- return new fast_xml_parser.XMLBuilder({
2312
- format: true,
2313
- ignoreAttributes: false,
2314
- indentBy: " "
2315
- }).build({
2316
- "?xml": {
2317
- "@_version": "1.0",
2318
- "@_encoding": "UTF-8"
2319
- },
2320
- testsuites: {
2321
- "@_errors": errors,
2322
- "@_failures": failures,
2323
- "@_name": "promptfoo",
2324
- "@_skipped": 0,
2325
- "@_tests": tests,
2326
- "@_time": formatDurationSeconds(totalTimeMs),
2327
- testsuite: suites.map((suite) => ({
2328
- "@_errors": suite.errors,
2329
- "@_failures": suite.failures,
2330
- "@_name": suite.displayName,
2331
- "@_skipped": 0,
2332
- "@_tests": suite.testcases.length,
2333
- "@_time": formatDurationSeconds(suite.timeMs),
2334
- ...timestamp ? { "@_timestamp": timestamp } : {},
2335
- testcase: suite.testcases.sort((a, b) => a.testIdx - b.testIdx).map(({ testcase }) => testcase)
2336
- }))
2337
- }
2338
- }).replace(INVALID_XML_CHARACTERS, "");
2339
- }
2340
- async function writeJunitXmlOutput(outputPath, evalRecord) {
2341
- await fs_promises.writeFile(outputPath, await createJunitXml(evalRecord));
2342
- }
2343
- const SUPPORTED_OUTPUT_FILE_FORMATS = [...require_tables.OutputFileExtension.options, "junit.xml"];
2344
- function getOutputFileFormat(outputPath) {
2345
- if (outputPath.toLowerCase().endsWith(".junit.xml")) return "junit.xml";
2346
- return require_tables.OutputFileExtension.safeParse(path.extname(outputPath).slice(1).toLowerCase()).data;
2347
- }
2348
- //#endregion
2349
- //#region src/util/output.ts
2350
- function warnOnDegradedJsonlRecovery(evalRecord, outputPaths) {
2351
- if (!evalRecord.resultPersistenceFailed) return;
2352
- if (outputPaths.some((outputPath) => getOutputFileFormat(outputPath) === "jsonl")) require_logger.logger.warn("[Output] Reconciling JSONL from streamed rows and the eval record because one or more rows failed to persist");
2353
- }
2354
- function isFileNotFoundError(error) {
2355
- return error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT";
2356
- }
2357
- function isPermissionDeniedError(error) {
2358
- return error !== null && typeof error === "object" && "code" in error && (error.code === "EACCES" || error.code === "EPERM");
2359
- }
2360
- async function resolveJsonlOutputPath(outputPath) {
2361
- try {
2362
- if (!(await fs_promises.lstat(outputPath)).isSymbolicLink()) return outputPath;
2363
- try {
2364
- return await fs_promises.realpath(outputPath);
2365
- } catch (error) {
2366
- if (isFileNotFoundError(error)) return path.resolve(path.dirname(outputPath), await fs_promises.readlink(outputPath));
2367
- throw error;
2368
- }
2369
- } catch (error) {
2370
- if (isFileNotFoundError(error)) return outputPath;
2371
- throw error;
2372
- }
2373
- }
2374
- async function appendJsonlResultBatch(outputPath, results, stripFlags) {
2375
- if (results.length === 0) return;
2376
- const text = results.map((result) => JSON.stringify(sanitizeResultForJsonlArtifact(result, stripFlags))).join(os.EOL) + os.EOL;
2377
- await fs_promises.appendFile(outputPath, text);
2378
- }
2379
- async function readStreamedJsonlResults(outputPath) {
2380
- let contents;
2381
- try {
2382
- contents = await fs_promises.readFile(outputPath, "utf8");
2383
- } catch (error) {
2384
- if (error.code === "ENOENT") return [];
2385
- throw error;
2386
- }
2387
- const results = [];
2388
- const lines = contents.split(/\r?\n/).filter(Boolean);
2389
- for (const [index, line] of lines.entries()) try {
2390
- results.push(JSON.parse(line));
2391
- } catch (error) {
2392
- require_logger.logger.warn(`[Output] Skipping malformed streamed JSONL row at ${outputPath}:${index + 1} during recovery: ${error instanceof Error ? error.message : String(error)}`);
2393
- }
2394
- return results;
2395
- }
2396
- async function collectJsonlResultsAfterPersistenceFailure(outputPath, evalRecord) {
2397
- const stripFlags = getStripFlags(evalRecord.config.env);
2398
- const finalResults = /* @__PURE__ */ new Map();
2399
- const put = (result) => finalResults.set(getResultIndexKey(result), result);
2400
- for (const result of await readStreamedJsonlResults(outputPath)) put(result);
2401
- for await (const batchResults of evalRecord.fetchResultsBatched()) for (const result of batchResults) {
2402
- const evaluateResult = asEvaluateResult(result, stripFlags);
2403
- if (!evalRecord.hasResultPersistenceFailure(evaluateResult)) put(evaluateResult);
2404
- }
2405
- for (const result of evalRecord.getFinalJsonlResults()) put(result);
2406
- return Array.from(finalResults.values());
2407
- }
2408
- async function getExistingFileMode(outputPath) {
2409
- try {
2410
- return (await fs_promises.stat(outputPath)).mode & 4095;
2411
- } catch (error) {
2412
- if (isFileNotFoundError(error)) return;
2413
- throw error;
2414
- }
2415
- }
2416
- async function appendJsonlResults(outputPath, evalRecord, recoveredResults) {
2417
- const stripFlags = getStripFlags(evalRecord.config.env);
2418
- if (recoveredResults) {
2419
- await appendJsonlResultBatch(outputPath, recoveredResults, stripFlags);
2420
- return;
2421
- }
2422
- for await (const batchResults of evalRecord.fetchResultsBatched()) await appendJsonlResultBatch(outputPath, batchResults.map((result) => asEvaluateResult(result, stripFlags)), stripFlags);
2423
- }
2424
- async function rewriteJsonlWithExternalBackup(outputPath, outputMode, evalRecord, preparedReplacementPath, recoveredResults) {
2425
- const tempDirectory = await fs_promises.mkdtemp(path.join(os.tmpdir(), "promptfoo-jsonl-"));
2426
- const backupPath = path.join(tempDirectory, "backup.jsonl");
2427
- const replacementPath = path.join(tempDirectory, "replacement.jsonl");
2428
- let overwriteAttempted = false;
2429
- let preserveTempDirectory = false;
2430
- let hasBackup = false;
2431
- try {
2432
- try {
2433
- await fs_promises.copyFile(outputPath, backupPath);
2434
- hasBackup = true;
2435
- } catch (error) {
2436
- if (!isFileNotFoundError(error)) throw error;
2437
- }
2438
- if (preparedReplacementPath) await fs_promises.copyFile(preparedReplacementPath, replacementPath);
2439
- else {
2440
- if (outputMode === void 0) await fs_promises.writeFile(replacementPath, "");
2441
- else await fs_promises.writeFile(replacementPath, "", { mode: outputMode });
2442
- await appendJsonlResults(replacementPath, evalRecord, recoveredResults);
2443
- }
2444
- overwriteAttempted = true;
2445
- await fs_promises.copyFile(replacementPath, outputPath);
2446
- if (outputMode !== void 0) await fs_promises.chmod(outputPath, outputMode);
2447
- } catch (error) {
2448
- if (overwriteAttempted && hasBackup) try {
2449
- await fs_promises.copyFile(backupPath, outputPath);
2450
- } catch (restoreError) {
2451
- preserveTempDirectory = true;
2452
- require_logger.logger.error("[Output] Failed to restore JSONL output after rewrite failure", {
2453
- backupPath,
2454
- error: restoreError
2455
- });
2456
- throw new Error(`Failed to rewrite JSONL output (${error instanceof Error ? error.message : String(error)}) and restore backup (${restoreError instanceof Error ? restoreError.message : String(restoreError)}). Backup retained at ${backupPath}`);
2457
- }
2458
- throw error;
2459
- } finally {
2460
- if (!preserveTempDirectory) await fs_promises.rm(tempDirectory, {
2461
- recursive: true,
2462
- force: true
2463
- }).catch((error) => {
2464
- require_logger.logger.warn("[Output] Failed to remove temporary JSONL backup directory", {
2465
- error,
2466
- tempDirectory
2467
- });
2468
- });
2469
- }
2470
- }
2471
- async function removeTemporaryJsonlOutput(tempOutputPath) {
2472
- await fs_promises.rm(tempOutputPath, { force: true }).catch((error) => {
2473
- require_logger.logger.warn("[Output] Failed to remove temporary JSONL output", {
2474
- error,
2475
- tempOutputPath
2476
- });
2477
- });
2478
- }
2479
- const outputToSimpleString = (output) => {
2480
- const passFailText = output.pass ? "[PASS]" : output.failureReason === require_tables.ResultFailureReason.ASSERT ? "[FAIL]" : "[ERROR]";
2481
- const namedScoresText = Object.entries(output.namedScores).map(([name, value]) => `${name}: ${value?.toFixed(2)}`).join(", ");
2482
- const scoreText = namedScoresText.length > 0 ? `(${output.score?.toFixed(2)}, ${namedScoresText})` : `(${output.score?.toFixed(2)})`;
2483
- const gradingResultText = output.gradingResult ? `${output.pass ? "Pass" : "Fail"} Reason: ${output.gradingResult.reason}` : "";
2484
- return dedent.default`
2485
- ${passFailText} ${scoreText}
2486
-
2487
- ${output.text}
2488
-
2489
- ${gradingResultText}
2490
- `.trim();
2491
- };
2492
- const outputToHtmlReportCell = (output) => {
2493
- const status = output.pass ? "pass" : output.failureReason === require_tables.ResultFailureReason.ERROR ? "error" : "fail";
2494
- const namedScores = Object.entries(output.namedScores).map(([name, value]) => ({
2495
- name,
2496
- value: value?.toFixed(2)
2497
- }));
2498
- return {
2499
- status,
2500
- statusLabel: status.toUpperCase(),
2501
- score: output.score?.toFixed(2),
2502
- namedScores,
2503
- text: output.text,
2504
- reason: output.gradingResult?.reason || "",
2505
- prompt: output.prompt,
2506
- provider: output.provider || "",
2507
- error: output.error || "",
2508
- failureReason: output.failureReason,
2509
- latencyDisplay: Number.isFinite(output.latencyMs) ? `${output.latencyMs.toFixed(0)} ms` : "",
2510
- costDisplay: Number.isFinite(output.cost) && output.cost > 0 ? `$${output.cost.toFixed(6)}` : "",
2511
- totalTokensDisplay: typeof output.tokenUsage?.total === "number" ? output.tokenUsage.total.toLocaleString("en-US") : "",
2512
- promptTokensDisplay: typeof output.tokenUsage?.prompt === "number" ? output.tokenUsage.prompt.toLocaleString("en-US") : "",
2513
- completionTokensDisplay: typeof output.tokenUsage?.completion === "number" ? output.tokenUsage.completion.toLocaleString("en-US") : ""
2514
- };
2515
- };
2516
- async function createOutputSummary(evalRecord, stripFlags) {
2517
- const summary = await evalRecord.toEvaluateSummary();
2518
- const prompts = ("prompts" in summary ? summary.prompts : summary.table.head.prompts).map((prompt) => prompt.config ? {
2519
- ...prompt,
2520
- config: require_logger.sanitizeConfigForOutput(prompt.config, stripFlags)
2521
- } : prompt);
2522
- return "prompts" in summary ? {
2523
- ...summary,
2524
- prompts
2525
- } : {
2526
- ...summary,
2527
- table: {
2528
- ...summary.table,
2529
- head: {
2530
- ...summary.table.head,
2531
- prompts
2532
- }
2533
- }
2534
- };
2535
- }
2536
- function resultsForMediaExportScan(results, shouldStripResponseOutput) {
2537
- if (!shouldStripResponseOutput) return results;
2538
- return {
2539
- ...results,
2540
- results: results.results.map((result) => {
2541
- const response = result.response;
2542
- if (!response?.metadata || !("blobUris" in response.metadata)) return result;
2543
- const { metadata: responseMetadata, ...projectedResponse } = response;
2544
- const { blobUris: _blobUris, ...metadata } = responseMetadata;
2545
- return {
2546
- ...result,
2547
- response: {
2548
- ...projectedResponse,
2549
- ...Object.keys(metadata).length > 0 && { metadata }
2550
- }
2551
- };
2552
- })
2553
- };
2554
- }
2555
- function createOutputMetadata(evalRecord) {
2556
- let evaluationCreatedAt;
2557
- if (evalRecord.createdAt) try {
2558
- const date = new Date(evalRecord.createdAt);
2559
- evaluationCreatedAt = Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
2560
- } catch {
2561
- evaluationCreatedAt = void 0;
2562
- }
2563
- return {
2564
- promptfooVersion: require_fetch.VERSION,
2565
- nodeVersion: process.version,
2566
- platform: os.platform(),
2567
- arch: os.arch(),
2568
- exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
2569
- evaluationCreatedAt,
2570
- author: evalRecord.author ?? void 0
2571
- };
2572
- }
2573
- async function createOutputData(evalRecord, shareableUrl, options = {}) {
2574
- const stripFlags = getStripFlags(evalRecord.config.env);
2575
- const summary = await createOutputSummary(evalRecord, stripFlags);
2576
- const redactedConfig = require_logger.sanitizeConfigForOutput(evalRecord.config, stripFlags);
2577
- let traces;
2578
- try {
2579
- const { getTraceStore } = await Promise.resolve().then(() => require("./store-D8Q5EmyP.cjs")).then((n) => n.store_exports);
2580
- traces = await getTraceStore().getTracesByEvaluation(evalRecord.id);
2581
- } catch (error) {
2582
- require_logger.logger.warn(`Failed to fetch traces for output ${evalRecord.id}; traces omitted from export: ${error}`);
2583
- }
2584
- const output = {
2585
- evalId: evalRecord.id,
2586
- results: summary,
2587
- config: redactedConfig,
2588
- shareableUrl,
2589
- metadata: createOutputMetadata(evalRecord),
2590
- ...evalRecord.vars?.length > 0 && { vars: [...evalRecord.vars] },
2591
- ...evalRecord.runtimeOptions && { runtimeOptions: require_logger.sanitizeRuntimeOptions(evalRecord.runtimeOptions) },
2592
- ...traces && traces.length > 0 && { traces: projectTracesForOutput(traces, stripFlags) }
2593
- };
2594
- if (options.includeMedia) {
2595
- const blobAssets = await exportBlobAssets(evalRecord.id, summary, stripFlags.shouldStripResponseOutput, output.traces);
2596
- if (blobAssets.length > 0) output.blobAssets = blobAssets;
2597
- }
2598
- return output;
2599
- }
2600
- async function exportBlobAssets(evalId, results, shouldStripResponseOutput, traces) {
2601
- const { getShareAuthorizedBlob } = await Promise.resolve().then(() => require("./blobs-CwaJ7l1g.cjs")).then((n) => n.blobs_exports);
2602
- const assets = [];
2603
- for (const hash of require_extractor.collectBlobHashes({
2604
- results: resultsForMediaExportScan(results, shouldStripResponseOutput),
2605
- traces
2606
- })) try {
2607
- const blob = await getShareAuthorizedBlob(hash, evalId);
2608
- if (!blob) continue;
2609
- if (blob.data.length > 52428800) {
2610
- require_logger.logger.warn("[Output] Skipping oversized blob in eval export", {
2611
- hash,
2612
- sizeBytes: blob.data.length
2613
- });
2614
- continue;
2615
- }
2616
- assets.push({
2617
- hash,
2618
- mimeType: blob.metadata.mimeType,
2619
- sizeBytes: blob.data.length,
2620
- data: blob.data.toString("base64")
2621
- });
2622
- } catch (error) {
2623
- require_logger.logger.warn("[Output] Skipping missing blob in eval export", {
2624
- error,
2625
- hash
2626
- });
2627
- }
2628
- return assets;
2629
- }
2630
- /**
2631
- * JSON writer with improved error handling for large datasets.
2632
- * Provides helpful error messages when memory limits are exceeded.
2633
- */
2634
- async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl, options) {
2635
- try {
2636
- const outputData = await createOutputData(evalRecord, shareableUrl, options);
2637
- const jsonString = JSON.stringify(outputData, null, 2);
2638
- await fs_promises.writeFile(outputPath, jsonString);
2639
- } catch (error) {
2640
- const msg = error?.message ?? "";
2641
- const isStringLen = error instanceof RangeError && msg.includes("Invalid string length");
2642
- const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
2643
- if (isStringLen || isHeapOOM) {
2644
- const resultCount = await evalRecord.getResultsCount();
2645
- require_logger.logger.error(`Dataset too large for JSON export (${resultCount} results).`);
2646
- throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
2647
- } else throw error;
2648
- }
2649
- }
2650
- async function writeOutput(outputPath, evalRecord, shareableUrl, options = {}) {
2651
- if (outputPath.match(/^https:\/\/docs\.google\.com\/spreadsheets\//)) {
2652
- const table = await evalRecord.getTable();
2653
- const rows = table.body.map((row) => {
2654
- const csvRow = {};
2655
- table.head.vars.forEach((varName, index) => {
2656
- csvRow[varName] = row.vars[index];
2657
- });
2658
- table.head.prompts.forEach((prompt, index) => {
2659
- csvRow[`[${prompt.provider}] ${prompt.label}`] = outputToSimpleString(row.outputs[index]);
2660
- });
2661
- return csvRow;
2662
- });
2663
- require_logger.logger.info(`Writing ${rows.length} rows to Google Sheets...`);
2664
- await writeCsvToGoogleSheet(rows, outputPath);
2665
- return;
2666
- }
2667
- const outputExtension = getOutputFileFormat(outputPath);
2668
- require_invariant.invariant(outputExtension, `Unsupported output file format ${path.extname(outputPath).slice(1).toLowerCase()}. Please use one of: ${SUPPORTED_OUTPUT_FILE_FORMATS.join(", ")}.`);
2669
- const outputDir = path.dirname(outputPath);
2670
- await fs_promises.mkdir(outputDir, { recursive: true });
2671
- if (outputExtension === "junit.xml") await writeJunitXmlOutput(outputPath, evalRecord);
2672
- else if (outputExtension === "csv") {
2673
- const fileHandle = await fs_promises.open(outputPath, "w");
2674
- try {
2675
- await streamEvalCsv(evalRecord, {
2676
- isRedteam: Boolean(evalRecord.config.redteam),
2677
- write: async (data) => {
2678
- await fileHandle.write(data);
2679
- }
2680
- });
2681
- } finally {
2682
- await fileHandle.close();
2683
- }
2684
- } else if (outputExtension === "json") await writeJsonOutputSafely(outputPath, evalRecord, shareableUrl, options);
2685
- else if (outputExtension === "yaml" || outputExtension === "yml" || outputExtension === "txt") await fs_promises.writeFile(outputPath, js_yaml.dump(await createOutputData(evalRecord, shareableUrl, options)));
2686
- else if (outputExtension === "html") {
2687
- const table = await evalRecord.getTable();
2688
- const stripFlags = getStripFlags(evalRecord.config.env);
2689
- const summary = await createOutputSummary(evalRecord, stripFlags);
2690
- const redactedConfig = require_logger.sanitizeConfigForOutput(evalRecord.config, stripFlags);
2691
- const metadata = createOutputMetadata(evalRecord);
2692
- const template = await fs_promises.readFile(path.join(require_esm.getDirectory(), "tableOutput.html"), "utf-8");
2693
- const htmlTable = [[...table.head.vars, ...table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`)], ...table.body.map((row, rowIndex) => [...row.vars.map((value, variableIndex) => ({
2694
- kind: "variable",
2695
- name: table.head.vars[variableIndex],
2696
- text: value
2697
- })), ...row.outputs.map((output, outputIndex) => ({
2698
- kind: "output",
2699
- detailId: `result-detail-${rowIndex}-${outputIndex}`,
2700
- detailTitle: `Result detail - row ${rowIndex + 1}, prompt ${outputIndex + 1}`,
2701
- description: row.description || row.test?.description || "",
2702
- ...outputToHtmlReportCell(output)
2703
- }))])];
2704
- const reportOutputs = table.body.flatMap((row) => row.outputs);
2705
- const totalResults = reportOutputs.length;
2706
- const successes = reportOutputs.filter((output) => output.pass).length;
2707
- const errors = reportOutputs.filter((output) => !output.pass && output.failureReason === require_tables.ResultFailureReason.ERROR).length;
2708
- const failures = totalResults - successes - errors;
2709
- const passRate = totalResults > 0 ? successes / totalResults * 100 : 0;
2710
- const htmlOutput = require_render.getNunjucksEngine().renderString(template, {
2711
- config: redactedConfig,
2712
- table: htmlTable,
2713
- results: summary,
2714
- metadata,
2715
- report: {
2716
- totalResults,
2717
- totalRows: table.body.length,
2718
- promptCount: table.head.prompts.length,
2719
- variableCount: table.head.vars.length,
2720
- successes,
2721
- failures,
2722
- errors,
2723
- passRateDisplay: `${passRate.toFixed(1)}%`
2724
- }
2725
- });
2726
- await fs_promises.writeFile(outputPath, htmlOutput);
2727
- } else if (outputExtension === "jsonl") {
2728
- const jsonlOutputPath = await resolveJsonlOutputPath(outputPath);
2729
- if (jsonlOutputPath !== outputPath) await fs_promises.mkdir(path.dirname(jsonlOutputPath), { recursive: true });
2730
- const outputMode = await getExistingFileMode(jsonlOutputPath);
2731
- const recoveredResults = evalRecord.resultPersistenceFailed ? await collectJsonlResultsAfterPersistenceFailure(jsonlOutputPath, evalRecord) : void 0;
2732
- const tempOutputPath = path.join(path.dirname(jsonlOutputPath), `.promptfoo-${(0, crypto$1.randomUUID)()}.tmp`);
2733
- try {
2734
- try {
2735
- await fs_promises.writeFile(tempOutputPath, "");
2736
- } catch (error) {
2737
- if (isPermissionDeniedError(error)) {
2738
- require_logger.logger.warn("[Output] Falling back to JSONL rewrite with external backup because the output directory is not writable");
2739
- await rewriteJsonlWithExternalBackup(jsonlOutputPath, outputMode, evalRecord, void 0, recoveredResults);
2740
- return;
2741
- }
2742
- throw error;
2743
- }
2744
- await appendJsonlResults(tempOutputPath, evalRecord, recoveredResults);
2745
- if (outputMode !== void 0) await fs_promises.chmod(tempOutputPath, outputMode);
2746
- try {
2747
- await fs_promises.rename(tempOutputPath, jsonlOutputPath);
2748
- } catch (error) {
2749
- if (isPermissionDeniedError(error)) {
2750
- require_logger.logger.warn("[Output] Falling back to JSONL rewrite with external backup because replacing the output file is not permitted");
2751
- await rewriteJsonlWithExternalBackup(jsonlOutputPath, outputMode, evalRecord, tempOutputPath);
2752
- await removeTemporaryJsonlOutput(tempOutputPath);
2753
- return;
2754
- }
2755
- throw error;
2756
- }
2757
- } catch (error) {
2758
- await removeTemporaryJsonlOutput(tempOutputPath);
2759
- throw error;
2760
- }
2761
- } else if (outputExtension === "xml") {
2762
- const stripFlags = getStripFlags(evalRecord.config.env);
2763
- const summary = await createOutputSummary(evalRecord, stripFlags);
2764
- const redactedConfig = require_logger.sanitizeConfigForOutput(evalRecord.config, stripFlags);
2765
- const sanitizeForXml = (obj) => {
2766
- if (obj === null || obj === void 0) return "";
2767
- if (Array.isArray(obj)) return obj.map(sanitizeForXml);
2768
- if (typeof obj === "object") {
2769
- const sanitized = {};
2770
- for (const [key, value] of Object.entries(obj)) sanitized[key] = sanitizeForXml(value);
2771
- return sanitized;
2772
- }
2773
- return String(obj);
2774
- };
2775
- const xmlData = new fast_xml_parser.XMLBuilder({
2776
- ignoreAttributes: false,
2777
- format: true,
2778
- indentBy: " "
2779
- }).build({ promptfoo: {
2780
- evalId: evalRecord.id,
2781
- results: sanitizeForXml(summary),
2782
- config: sanitizeForXml(redactedConfig),
2783
- shareableUrl: shareableUrl || ""
2784
- } });
2785
- await fs_promises.writeFile(outputPath, xmlData);
2786
- }
2787
- }
2788
- async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
2789
- const errors = (await Promise.allSettled(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)))).flatMap((result) => result.status === "rejected" ? [result.reason] : []);
2790
- if (errors.length > 0) throw Object.assign(/* @__PURE__ */ new Error("One or more output writes failed"), { errors });
2791
- }
2792
- //#endregion
2793
- //#region src/util/runtime.ts
2794
- function printBorder() {
2795
- const border = "=".repeat(require_fetch.TERMINAL_MAX_WIDTH);
2796
- require_logger.logger.info(border);
2797
- }
2798
- //#endregion
2799
- //#region src/util/dataUrl.ts
2800
- /**
2801
- * Check if a string is a data URL
2802
- * @param value String to check
2803
- * @returns true if value is a data URL (starts with "data:")
2804
- *
2805
- * @example
2806
- * isDataUrl("data:image/jpeg;base64,/9j/...") // true
2807
- * isDataUrl("/9j/4AAQSkZJRg...") // false
2808
- * isDataUrl("https://example.com/image.jpg") // false
2809
- */
2810
- function isDataUrl(value) {
2811
- return typeof value === "string" && value.startsWith("data:") && value.length > 5;
2812
- }
2813
- /**
2814
- * Parse a data URL into its components
2815
- *
2816
- * Handles data URLs with optional parameters (e.g., charset, name):
2817
- * - `data:image/jpeg;base64,<data>` - Standard format
2818
- * - `data:image/jpeg;charset=utf-8;base64,<data>` - With charset
2819
- * - `data:image/jpeg;name=photo.jpg;base64,<data>` - With filename
2820
- *
2821
- * @param dataUrl Data URL string
2822
- * @returns Parsed components (mimeType and base64Data) or null if invalid
2823
- *
2824
- * @example
2825
- * parseDataUrl("data:image/jpeg;base64,/9j/...")
2826
- * // { mimeType: "image/jpeg", base64Data: "/9j/..." }
2827
- *
2828
- * parseDataUrl("data:image/jpeg;charset=utf-8;base64,/9j/...")
2829
- * // { mimeType: "image/jpeg", base64Data: "/9j/..." }
2830
- *
2831
- * parseDataUrl("invalid") // null
2832
- */
2833
- function parseDataUrl(dataUrl) {
2834
- if (!isDataUrl(dataUrl)) return null;
2835
- const match = dataUrl.match(/^data:([^;,]+)(?:;[^,]*)?;base64,(.+)$/);
2836
- if (!match) return null;
2837
- return {
2838
- mimeType: match[1].trim(),
2839
- base64Data: match[2].trim()
2840
- };
2841
- }
2842
- /**
2843
- * Extract base64 data from a data URL or return original if not a data URL
2844
- * Useful for providers that expect raw base64 (Anthropic, Google)
2845
- *
2846
- * @param value Data URL or raw base64 string
2847
- * @returns Raw base64 string (data URL prefix stripped if present)
2848
- *
2849
- * @example
2850
- * extractBase64FromDataUrl("data:image/jpeg;base64,/9j/...")
2851
- * // "/9j/..."
2852
- *
2853
- * extractBase64FromDataUrl("/9j/...") // "/9j/..." (unchanged)
2854
- */
2855
- function extractBase64FromDataUrl(value) {
2856
- const parsed = parseDataUrl(value);
2857
- return parsed ? parsed.base64Data : value;
2858
- }
2859
- /**
2860
- * Build a data URI from a MIME type and base64 data.
2861
- *
2862
- * @param mimeType MIME type (e.g. "image/png")
2863
- * @param base64Data Raw base64-encoded data
2864
- * @returns Data URI string
2865
- *
2866
- * @example
2867
- * toDataUri("image/png", "iVBORw0KGgo...")
2868
- * // "data:image/png;base64,iVBORw0KGgo..."
2869
- */
2870
- function toDataUri(mimeType, base64Data) {
2871
- return `data:${mimeType};base64,${base64Data}`;
2872
- }
2873
- //#endregion
2874
- Object.defineProperty(exports, "EvalResult", {
2875
- enumerable: true,
2876
- get: function() {
2877
- return EvalResult;
2878
- }
2879
- });
2880
- Object.defineProperty(exports, "FILE_METADATA_KEY", {
2881
- enumerable: true,
2882
- get: function() {
2883
- return FILE_METADATA_KEY;
2884
- }
2885
- });
2886
- Object.defineProperty(exports, "HUMAN_ASSERTION_TYPE", {
2887
- enumerable: true,
2888
- get: function() {
2889
- return HUMAN_ASSERTION_TYPE;
2890
- }
2891
- });
2892
- Object.defineProperty(exports, "PROMPTFOO_METADATA_KEY", {
2893
- enumerable: true,
2894
- get: function() {
2895
- return PROMPTFOO_METADATA_KEY;
2896
- }
2897
- });
2898
- Object.defineProperty(exports, "asEvaluateResult", {
2899
- enumerable: true,
2900
- get: function() {
2901
- return asEvaluateResult;
2902
- }
2903
- });
2904
- Object.defineProperty(exports, "checkProviderApiKeys", {
2905
- enumerable: true,
2906
- get: function() {
2907
- return checkProviderApiKeys;
2908
- }
2909
- });
2910
- Object.defineProperty(exports, "deduplicateTestCases", {
2911
- enumerable: true,
2912
- get: function() {
2913
- return deduplicateTestCases;
2914
- }
2915
- });
2916
- Object.defineProperty(exports, "doesProviderRefMatch", {
2917
- enumerable: true,
2918
- get: function() {
2919
- return doesProviderRefMatch;
2920
- }
2921
- });
2922
- Object.defineProperty(exports, "extractBase64FromDataUrl", {
2923
- enumerable: true,
2924
- get: function() {
2925
- return extractBase64FromDataUrl;
2926
- }
2927
- });
2928
- Object.defineProperty(exports, "extractRuntimeVars", {
2929
- enumerable: true,
2930
- get: function() {
2931
- return extractRuntimeVars;
2932
- }
2933
- });
2934
- Object.defineProperty(exports, "fetchCsvFromGoogleSheet", {
2935
- enumerable: true,
2936
- get: function() {
2937
- return fetchCsvFromGoogleSheet;
2938
- }
2939
- });
2940
- Object.defineProperty(exports, "filterRuntimeVars", {
2941
- enumerable: true,
2942
- get: function() {
2943
- return filterRuntimeVars;
2944
- }
2945
- });
2946
- Object.defineProperty(exports, "generateIdFromPrompt", {
2947
- enumerable: true,
2948
- get: function() {
2949
- return generateIdFromPrompt;
2950
- }
2951
- });
2952
- Object.defineProperty(exports, "getCachedResultsCount", {
2953
- enumerable: true,
2954
- get: function() {
2955
- return getCachedResultsCount;
2956
- }
2957
- });
2958
- Object.defineProperty(exports, "getOutputFileFormat", {
2959
- enumerable: true,
2960
- get: function() {
2961
- return getOutputFileFormat;
2962
- }
2963
- });
2964
- Object.defineProperty(exports, "getProviderDescription", {
2965
- enumerable: true,
2966
- get: function() {
2967
- return getProviderDescription;
2968
- }
2969
- });
2970
- Object.defineProperty(exports, "getProviderIdentifier", {
2971
- enumerable: true,
2972
- get: function() {
2973
- return getProviderIdentifier;
2974
- }
2975
- });
2976
- Object.defineProperty(exports, "getResultIndexKey", {
2977
- enumerable: true,
2978
- get: function() {
2979
- return getResultIndexKey;
2980
- }
2981
- });
2982
- Object.defineProperty(exports, "getStripFlags", {
2983
- enumerable: true,
2984
- get: function() {
2985
- return getStripFlags;
2986
- }
2987
- });
2988
- Object.defineProperty(exports, "getTotalResultRowCount", {
2989
- enumerable: true,
2990
- get: function() {
2991
- return getTotalResultRowCount;
2992
- }
2993
- });
2994
- Object.defineProperty(exports, "hashPrompt", {
2995
- enumerable: true,
2996
- get: function() {
2997
- return hashPrompt;
2998
- }
2999
- });
3000
- Object.defineProperty(exports, "invalidateEvaluationCache", {
3001
- enumerable: true,
3002
- get: function() {
3003
- return invalidateEvaluationCache;
3004
- }
3005
- });
3006
- Object.defineProperty(exports, "isAnthropicProvider", {
3007
- enumerable: true,
3008
- get: function() {
3009
- return isAnthropicProvider;
3010
- }
3011
- });
3012
- Object.defineProperty(exports, "isDataUrl", {
3013
- enumerable: true,
3014
- get: function() {
3015
- return isDataUrl;
3016
- }
3017
- });
3018
- Object.defineProperty(exports, "isGoogleProvider", {
3019
- enumerable: true,
3020
- get: function() {
3021
- return isGoogleProvider;
3022
- }
3023
- });
3024
- Object.defineProperty(exports, "isOpenAiProvider", {
3025
- enumerable: true,
3026
- get: function() {
3027
- return isOpenAiProvider;
3028
- }
3029
- });
3030
- Object.defineProperty(exports, "isProviderAllowed", {
3031
- enumerable: true,
3032
- get: function() {
3033
- return isProviderAllowed;
3034
- }
3035
- });
3036
- Object.defineProperty(exports, "isProviderConfigFileReference", {
3037
- enumerable: true,
3038
- get: function() {
3039
- return isProviderConfigFileReference;
3040
- }
3041
- });
3042
- Object.defineProperty(exports, "loadProviderConfigsFromFile", {
3043
- enumerable: true,
3044
- get: function() {
3045
- return loadProviderConfigsFromFile;
3046
- }
3047
- });
3048
- Object.defineProperty(exports, "maybeFilePath", {
3049
- enumerable: true,
3050
- get: function() {
3051
- return maybeFilePath;
3052
- }
3053
- });
3054
- Object.defineProperty(exports, "normalizeInput", {
3055
- enumerable: true,
3056
- get: function() {
3057
- return normalizeInput;
3058
- }
3059
- });
3060
- Object.defineProperty(exports, "normalizeProviderRef", {
3061
- enumerable: true,
3062
- get: function() {
3063
- return normalizeProviderRef;
3064
- }
3065
- });
3066
- Object.defineProperty(exports, "notifyEvaluationChanged", {
3067
- enumerable: true,
3068
- get: function() {
3069
- return notifyEvaluationChanged;
3070
- }
3071
- });
3072
- Object.defineProperty(exports, "notifyEvaluationsDeleted", {
3073
- enumerable: true,
3074
- get: function() {
3075
- return notifyEvaluationsDeleted;
3076
- }
3077
- });
3078
- Object.defineProperty(exports, "parseDataUrl", {
3079
- enumerable: true,
3080
- get: function() {
3081
- return parseDataUrl;
3082
- }
3083
- });
3084
- Object.defineProperty(exports, "persistRepeatMetadata", {
3085
- enumerable: true,
3086
- get: function() {
3087
- return persistRepeatMetadata;
3088
- }
3089
- });
3090
- Object.defineProperty(exports, "persistTraceMetadata", {
3091
- enumerable: true,
3092
- get: function() {
3093
- return persistTraceMetadata;
3094
- }
3095
- });
3096
- Object.defineProperty(exports, "printBorder", {
3097
- enumerable: true,
3098
- get: function() {
3099
- return printBorder;
3100
- }
3101
- });
3102
- Object.defineProperty(exports, "projectPrompt", {
3103
- enumerable: true,
3104
- get: function() {
3105
- return projectPrompt;
3106
- }
3107
- });
3108
- Object.defineProperty(exports, "projectTracesForOutput", {
3109
- enumerable: true,
3110
- get: function() {
3111
- return projectTracesForOutput;
3112
- }
3113
- });
3114
- Object.defineProperty(exports, "providerToIdentifier", {
3115
- enumerable: true,
3116
- get: function() {
3117
- return providerToIdentifier;
3118
- }
3119
- });
3120
- Object.defineProperty(exports, "queryTestIndicesOptimized", {
3121
- enumerable: true,
3122
- get: function() {
3123
- return queryTestIndicesOptimized;
3124
- }
3125
- });
3126
- Object.defineProperty(exports, "readProviderConfigFile", {
3127
- enumerable: true,
3128
- get: function() {
3129
- return readProviderConfigFile;
3130
- }
3131
- });
3132
- Object.defineProperty(exports, "resultIsForTestCase", {
3133
- enumerable: true,
3134
- get: function() {
3135
- return resultIsForTestCase;
3136
- }
3137
- });
3138
- Object.defineProperty(exports, "sanitizeProviderIdForLog", {
3139
- enumerable: true,
3140
- get: function() {
3141
- return sanitizeProviderIdForLog;
3142
- }
3143
- });
3144
- Object.defineProperty(exports, "sanitizeResultForJsonlArtifact", {
3145
- enumerable: true,
3146
- get: function() {
3147
- return sanitizeResultForJsonlArtifact;
3148
- }
3149
- });
3150
- Object.defineProperty(exports, "setupEnv", {
3151
- enumerable: true,
3152
- get: function() {
3153
- return setupEnv;
3154
- }
3155
- });
3156
- Object.defineProperty(exports, "stripTraceLinkageFromMetadata", {
3157
- enumerable: true,
3158
- get: function() {
3159
- return stripTraceLinkageFromMetadata;
3160
- }
3161
- });
3162
- Object.defineProperty(exports, "toDataUri", {
3163
- enumerable: true,
3164
- get: function() {
3165
- return toDataUri;
3166
- }
3167
- });
3168
- Object.defineProperty(exports, "toSerializableProviderRef", {
3169
- enumerable: true,
3170
- get: function() {
3171
- return toSerializableProviderRef;
3172
- }
3173
- });
3174
- Object.defineProperty(exports, "warnOnDegradedJsonlRecovery", {
3175
- enumerable: true,
3176
- get: function() {
3177
- return warnOnDegradedJsonlRecovery;
3178
- }
3179
- });
3180
- Object.defineProperty(exports, "writeMultipleOutputs", {
3181
- enumerable: true,
3182
- get: function() {
3183
- return writeMultipleOutputs;
3184
- }
3185
- });
3186
-
3187
- //# sourceMappingURL=dataUrl-CGsBT7UZ.cjs.map