promptfoo 0.120.26 → 0.121.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (366) hide show
  1. package/README.md +1 -1
  2. package/dist/drizzle/0023_wooden_mandrill.sql +2 -0
  3. package/dist/drizzle/meta/0023_snapshot.json +1496 -0
  4. package/dist/drizzle/meta/_journal.json +7 -0
  5. package/dist/src/{ListApp-D3DG0F8h.js → ListApp-Du7YVwj5.js} +2 -4
  6. package/dist/src/accounts-BgNJDBE6.js +206 -0
  7. package/dist/src/{accounts-BzEY8H3v.cjs → accounts-Bx-x3bmW.cjs} +99 -80
  8. package/dist/src/{accounts-DHHiXsy6.js → accounts-CMqkzrVf.js} +61 -36
  9. package/dist/src/{accounts-R3gfCR_g.js → accounts-xrUGFA6n.js} +60 -35
  10. package/dist/src/{agentic-utils-D6_gzOUF.js → agentic-utils-BKIN5PKu.js} +9 -10
  11. package/dist/src/{agents-CwM7re15.cjs → agents-B0f4HICh.cjs} +37 -40
  12. package/dist/src/{agents-Cnph5GLD.js → agents-C-dDThPK.js} +37 -37
  13. package/dist/src/{agents-C7BiinFI.cjs → agents-CErsqg5U.cjs} +19 -27
  14. package/dist/src/{agents-v4cW_ZgC.js → agents-CVIn-Utx.js} +19 -22
  15. package/dist/src/{agents-GiUianme.js → agents-CXknwsFX.js} +37 -40
  16. package/dist/src/{agents-DETIQHqF.js → agents-DeH4Gu94.js} +21 -28
  17. package/dist/src/{agents-DYIT-hQy.js → agents-Dy2YpZpa.js} +38 -41
  18. package/dist/src/{agents-Cao4i7AX.js → agents-aF4-T121.js} +19 -30
  19. package/dist/src/{aimlapi-DMF6v_vb.js → aimlapi-BAGZDo5G.js} +16 -18
  20. package/dist/src/{aimlapi-CMJpKK-B.js → aimlapi-BNfTBexL.js} +15 -17
  21. package/dist/src/{aimlapi-DtSf1ykJ.js → aimlapi-DHRKlBEA.js} +15 -4
  22. package/dist/src/{aimlapi-DoGLcQW_.cjs → aimlapi-tg0Gkcvr.cjs} +15 -16
  23. package/dist/src/app/assets/index-BFCZg7hQ.js +439 -0
  24. package/dist/src/app/assets/index-NCn4eVBv.css +1 -0
  25. package/dist/src/app/assets/{vendor-charts-CYyo8R8v.js → vendor-charts-CCl15Imd.js} +1 -1
  26. package/dist/src/app/assets/{vendor-markdown-DSmzq4Jh.js → vendor-markdown-0tekx3KX.js} +1 -1
  27. package/dist/src/app/index.html +4 -34
  28. package/dist/src/{audio-DUH4q0Xq.js → audio-BRODU0UK.js} +7 -9
  29. package/dist/src/{audio-BWjyvHn9.cjs → audio-BWeaWovU.cjs} +6 -7
  30. package/dist/src/{audio-U580w8jM.js → audio-CHQ4r-RV.js} +6 -5
  31. package/dist/src/{audio-BrJBFN2b.js → audio-tf_NBjlC.js} +6 -8
  32. package/dist/src/base-B0tcrnq_.js +193 -0
  33. package/dist/src/base-B4QJRyFS.js +194 -0
  34. package/dist/src/base-DBtwl2FR.cjs +222 -0
  35. package/dist/src/base-fEDN28WM.js +193 -0
  36. package/dist/src/{blobs-kt8v3UyH.js → blobs-BAU-dXan.js} +9 -12
  37. package/dist/src/{blobs-C9J2mVgo.js → blobs-Bpg5rH6i.js} +9 -12
  38. package/dist/src/{blobs-673H0jCl.cjs → blobs-DvS-O6be.cjs} +34 -37
  39. package/dist/src/blobs-qTYm-1PY.js +236 -0
  40. package/dist/src/{cache-BLLayYEN.js → cache-8XhNqPKW.js} +64 -67
  41. package/dist/src/cache-Bbn1Nyrd.cjs +5 -0
  42. package/dist/src/cache-BwsMSda7.js +6 -0
  43. package/dist/src/{cache-mIszOnuz.js → cache-CG0SlR1d.js} +64 -66
  44. package/dist/src/{cache-7xULbvt3.cjs → cache-COish3-W.cjs} +114 -117
  45. package/dist/src/cache-D3eqDYGU.js +739 -0
  46. package/dist/src/{chat-Fl6TZJRS.cjs → chat-2K608PeQ.cjs} +20 -21
  47. package/dist/src/chat-BKm79wib.js +764 -0
  48. package/dist/src/{chat-XPN9YHhr.js → chat-CM_kyI8B.js} +20 -9
  49. package/dist/src/{chat-DIywASPG.js → chat-CRWNNq73.js} +49 -49
  50. package/dist/src/{chat-C8Ei4f87.js → chat-CznLWr_D.js} +49 -49
  51. package/dist/src/{chat-CgyGj2hC.js → chat-DHMH-N64.js} +20 -22
  52. package/dist/src/{chat-C4zqjObh.cjs → chat-DaqekjFr.cjs} +69 -69
  53. package/dist/src/{chat-Cpz3O-Xl.js → chat-DxysjBvt.js} +21 -23
  54. package/dist/src/{chatkit-Dpxrq4eD.js → chatkit-65VXf5SR.js} +58 -58
  55. package/dist/src/{chatkit-DIrJX8xk.js → chatkit-Be-Q-a9F.js} +58 -60
  56. package/dist/src/{chatkit-DEls11hE.js → chatkit-BxFvW8KY.js} +58 -60
  57. package/dist/src/{chatkit-e25Ziu17.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
  58. package/dist/src/chunk-DEq-mXcV.js +15 -0
  59. package/dist/src/chunk-DRamLcfz.js +16 -0
  60. package/dist/src/{claude-agent-sdk-6-xTaLwM.js → claude-agent-sdk-BLTu0WBO.js} +45 -31
  61. package/dist/src/{claude-agent-sdk-BzxF6NIJ.cjs → claude-agent-sdk-CJH22shf.cjs} +44 -29
  62. package/dist/src/{claude-agent-sdk-CmkTnKGH.js → claude-agent-sdk-D6_k9FKA.js} +45 -33
  63. package/dist/src/{claude-agent-sdk-rXCBLK_o.js → claude-agent-sdk-Dy5lT-Tx.js} +46 -21
  64. package/dist/src/{cloud-BMbRVJFw.js → cloud-Bc9526yV.js} +32 -12
  65. package/dist/src/cloud-DmE0EwsY.js +4 -0
  66. package/dist/src/{cloudflare-ai-CUg4BTcj.js → cloudflare-ai-C9r2sRhw.js} +16 -18
  67. package/dist/src/{cloudflare-ai-Z9X219gp.js → cloudflare-ai-CWWJCRim.js} +16 -4
  68. package/dist/src/{cloudflare-ai-BAQ0u_dg.cjs → cloudflare-ai-ClWSdor4.cjs} +16 -17
  69. package/dist/src/{cloudflare-ai-CobxMTR4.js → cloudflare-ai-ICsOuD-z.js} +17 -19
  70. package/dist/src/{cloudflare-gateway-C0sgfr_z.cjs → cloudflare-gateway-C2_-KG5o.cjs} +21 -22
  71. package/dist/src/{cloudflare-gateway-_itGuXry.js → cloudflare-gateway-D6O7AlYb.js} +23 -23
  72. package/dist/src/{cloudflare-gateway-D2_yi-Fh.js → cloudflare-gateway-D6xFc5pa.js} +21 -25
  73. package/dist/src/{cloudflare-gateway-Djf3F3_H.js → cloudflare-gateway-pXGHxJ47.js} +26 -14
  74. package/dist/src/{codex-sdk-ibXwdglL.js → codex-sdk-C6UMlxwV.js} +49 -32
  75. package/dist/src/{codex-sdk-BASDNkIl.js → codex-sdk-DUwKWezN.js} +49 -30
  76. package/dist/src/{codex-sdk-dSnGdgIp.js → codex-sdk-GGAw0qbD.js} +49 -32
  77. package/dist/src/{codex-sdk-wTEpMM_X.cjs → codex-sdk-fAO0c3yA.cjs} +49 -32
  78. package/dist/src/{cometapi-B01btbfb.js → cometapi-BasUi7-_.js} +17 -19
  79. package/dist/src/{cometapi-DHUAH6nK.js → cometapi-Bbjp5V4x.js} +16 -4
  80. package/dist/src/{cometapi-ChAaRjg5.js → cometapi-DkXrKi5z.js} +21 -24
  81. package/dist/src/{cometapi-JbvOJSCO.cjs → cometapi-vY6aDZgo.cjs} +21 -22
  82. package/dist/src/{completion-D9_MDlnd.js → completion-6Mx_iXxK.js} +11 -13
  83. package/dist/src/{completion-BBJ6zmG3.js → completion-C5rtR_9P.js} +11 -13
  84. package/dist/src/{completion-DDyL3Cb2.cjs → completion-CDOouNzq.cjs} +21 -23
  85. package/dist/src/completion-C_P3ypkJ.js +120 -0
  86. package/dist/src/createHash-CTQmL3G2.js +15 -0
  87. package/dist/src/createHash-CfZSc0b4.cjs +27 -0
  88. package/dist/src/createHash-Da8fMwqB.js +16 -0
  89. package/dist/src/createHash-DmPQkvBh.js +15 -0
  90. package/dist/src/{docker-JAAubMw3.js → docker-5KcG-_86.js} +18 -20
  91. package/dist/src/{docker-Ckw-j7Rr.cjs → docker-BwsKwxFs.cjs} +18 -19
  92. package/dist/src/{docker-vnOg96gi.js → docker-CZnqU1XV.js} +18 -7
  93. package/dist/src/{docker-BuButc4D.js → docker-DzxyDPIj.js} +19 -21
  94. package/dist/src/entrypoint.js +2 -3
  95. package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
  96. package/dist/src/{esm-CYhseqj4.js → esm-C03C-mv3.js} +17 -20
  97. package/dist/src/{esm-rDtG_2rg.js → esm-CaIwzWR5.js} +18 -21
  98. package/dist/src/esm-Cd1AjG1D.js +379 -0
  99. package/dist/src/{esm-BQkx5roy.cjs → esm-CnNt7sI4.cjs} +47 -49
  100. package/dist/src/eval-17JizQIv.js +15 -0
  101. package/dist/src/{eval-CYrbG57o.js → eval-DmFyWU7i.js} +49 -55
  102. package/dist/src/{evalResult-COsVttMA.js → evalResult-CDQiuUuf.js} +16 -12
  103. package/dist/src/{evalResult-6JaUIStC.js → evalResult-CTG2AHOS.js} +10 -11
  104. package/dist/src/evalResult-Cqj8pldJ.js +12 -0
  105. package/dist/src/{evalResult-DlRfu_Rq.cjs → evalResult-Dap2CekP.cjs} +20 -21
  106. package/dist/src/evalResult-DvcJAWJU.cjs +10 -0
  107. package/dist/src/evalResult-Hftn-S_i.js +10 -0
  108. package/dist/src/evaluator-B2CFNt-P.js +36 -0
  109. package/dist/src/{evaluator-3EJCMTs0.js → evaluator-DPFRbFIL.js} +210 -232
  110. package/dist/src/{extractor-LSYjrhK0.js → extractor-CFG6bcWJ.js} +23 -38
  111. package/dist/src/{extractor-DbhlYEeo.cjs → extractor-DX36oYEv.cjs} +37 -64
  112. package/dist/src/{extractor-Hs7la_19.js → extractor-M67RUtg6.js} +23 -38
  113. package/dist/src/extractor-YMU_Gvt8.js +374 -0
  114. package/dist/src/fetch-4M3YRaqL.js +780 -0
  115. package/dist/src/{fetch-18MuNu9i.js → fetch-60Gzydls.js} +60 -46
  116. package/dist/src/{fetch-SRsE6Ctl.js → fetch-BMv0O527.js} +41 -35
  117. package/dist/src/{fetch-ZMn_oemb.cjs → fetch-BxUk8odA.cjs} +268 -279
  118. package/dist/src/fetch-KV5kNASw.js +5 -0
  119. package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
  120. package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
  121. package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
  122. package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
  123. package/dist/src/{formatDuration-Doo0xq-z.js → formatDuration-DgBVMN65.js} +2 -2
  124. package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
  125. package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
  126. package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
  127. package/dist/src/{genaiTracer-CQlpZkrp.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
  128. package/dist/src/graders-Bu0H9nXi.js +32 -0
  129. package/dist/src/{graders-BaMCwIKp.js → graders-CHO8EPM4.js} +385 -417
  130. package/dist/src/graders-Cfhkvx-e.js +34 -0
  131. package/dist/src/{graders-QsALpIdy.js → graders-CpdqD9PI.js} +385 -417
  132. package/dist/src/graders-DClJVpGP.cjs +32 -0
  133. package/dist/src/{graders-DzUUnUjC.cjs → graders-DOXycdlG.cjs} +721 -753
  134. package/dist/src/graders-DcnJsrMO.js +32 -0
  135. package/dist/src/graders-R9rYUM0d.js +13466 -0
  136. package/dist/src/{image-BiEVdpdP.js → image-BmEZqVmk.js} +57 -18
  137. package/dist/src/{image-mhAGP07h.js → image-CBBVXWuT.js} +57 -18
  138. package/dist/src/{image-D10zEe1f.cjs → image-CDLQOcqT.cjs} +6 -7
  139. package/dist/src/{image-COCWy5dX.js → image-DJEvKveK.js} +6 -5
  140. package/dist/src/{image-C3BjJUAU.cjs → image-DTedmQPg.cjs} +77 -32
  141. package/dist/src/{image-DB4sHxdJ.js → image-gvmivTEe.js} +7 -9
  142. package/dist/src/image-pAX56tPG.js +257 -0
  143. package/dist/src/{image-BXt_7u0v.js → image-tL5hIOFh.js} +6 -8
  144. package/dist/src/index.cjs +696 -693
  145. package/dist/src/index.d.cts +113 -10
  146. package/dist/src/index.d.ts +113 -6
  147. package/dist/src/index.js +657 -658
  148. package/dist/src/{interactiveCheck-DU-MAhp5.js → interactiveCheck-BgLZUIt3.js} +7 -8
  149. package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
  150. package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
  151. package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
  152. package/dist/src/invariant-vgHWClmd.js +25 -0
  153. package/dist/src/{knowledgeBase-DotRBzUE.js → knowledgeBase-CLJybhnF.js} +19 -34
  154. package/dist/src/{knowledgeBase-XJQ0Qyez.js → knowledgeBase-CoU-UQBg.js} +17 -41
  155. package/dist/src/{knowledgeBase-CMvMlLZR.js → knowledgeBase-DjWPVqSb.js} +17 -43
  156. package/dist/src/{knowledgeBase-Bnb00xKs.cjs → knowledgeBase-wkxuRFhA.cjs} +17 -40
  157. package/dist/src/{litellm-CHrRmPAe.js → litellm-B9Hysuri.js} +16 -18
  158. package/dist/src/{litellm-CrLJrPIm.js → litellm-CTfa0hqi.js} +15 -17
  159. package/dist/src/{litellm-BrnZhMcL.cjs → litellm-NYpQ8RQu.cjs} +15 -16
  160. package/dist/src/{litellm-BECdjOTx.js → litellm-ePxtr9F1.js} +15 -4
  161. package/dist/src/{logger-w8Ozp0Td.js → logger-CT3IKMKA.js} +24 -41
  162. package/dist/src/{logger-BdZ-IqBc.cjs → logger-Cp1GPUjj.cjs} +166 -192
  163. package/dist/src/logger-DLcq4dWf.js +713 -0
  164. package/dist/src/{logger-BotXmWKW.js → logger-KkObSCzq.js} +27 -43
  165. package/dist/src/{luma-ray-C0RkI3lt.cjs → luma-ray-B0GGNRc1.cjs} +20 -21
  166. package/dist/src/{luma-ray-C-w6EsJm.js → luma-ray-BE2mOt6N.js} +20 -13
  167. package/dist/src/{luma-ray-BOeX-h0M.js → luma-ray-BW9IRGIc.js} +22 -21
  168. package/dist/src/{luma-ray-DgKLS0BF.js → luma-ray-Cm1KZBhs.js} +20 -23
  169. package/dist/src/main.js +1985 -2055
  170. package/dist/src/{messages-DXV3Qh8_.cjs → messages-1JrJs91T.cjs} +35 -34
  171. package/dist/src/{messages-D61tPFQo.js → messages-1x9atZmP.js} +25 -24
  172. package/dist/src/{messages-CDZYGNlS.js → messages-BLbWdsyt.js} +25 -24
  173. package/dist/src/messages-D8EA0oDc.js +240 -0
  174. package/dist/src/{meteor-P2rUE-Uz.js → meteor-44VjEACX.js} +3 -4
  175. package/dist/src/{meteor-SLNTgmXm.js → meteor-D-SotUw9.js} +3 -4
  176. package/dist/src/{meteor-odmwVbyG.cjs → meteor-DLZZ3osF.cjs} +3 -4
  177. package/dist/src/{meteor-Dj8cTkU_.js → meteor-DUiCJRC-.js} +3 -4
  178. package/dist/src/modelslab-C1OLRmVX.js +166 -0
  179. package/dist/src/modelslab-CqXBy3U8.js +168 -0
  180. package/dist/src/modelslab-DcOSFwKh.cjs +166 -0
  181. package/dist/src/modelslab-X5-4LroM.js +166 -0
  182. package/dist/src/{nova-reel-C2LFfVTf.js → nova-reel-BgS1ZWuK.js} +20 -13
  183. package/dist/src/{nova-reel-DtCjbD5O.js → nova-reel-D2ZkOSyr.js} +22 -21
  184. package/dist/src/{nova-reel-D9FXq3Mt.cjs → nova-reel-D9xfaMBs.cjs} +20 -21
  185. package/dist/src/{nova-reel-Bk5npr2q.js → nova-reel-DihqLeol.js} +20 -23
  186. package/dist/src/{nova-sonic-BoRSY_U6.cjs → nova-sonic-DVu3mMIy.cjs} +30 -31
  187. package/dist/src/{nova-sonic-D_qERM-K.js → nova-sonic-DezhVUYT.js} +30 -26
  188. package/dist/src/{nova-sonic-CgaWLDM1.js → nova-sonic-P-CdUMlV.js} +30 -31
  189. package/dist/src/{nova-sonic-BXRfQyF-.js → nova-sonic-Q3BOJeig.js} +31 -32
  190. package/dist/src/{openai-Bigwjgo1.cjs → openai-Cuif0GEt.cjs} +8 -9
  191. package/dist/src/{openai-Dz3surb_.js → openai-DElQ-fPX.js} +3 -4
  192. package/dist/src/{openai-CT5fwbve.js → openai-DhbB7eWK.js} +3 -4
  193. package/dist/src/openai-j-sE2O7r.js +44 -0
  194. package/dist/src/{openclaw-dHLcXUWZ.js → openclaw-BiSZPL7J.js} +20 -14
  195. package/dist/src/{openclaw-CpPrXwf6.js → openclaw-Bv1DINsX.js} +20 -27
  196. package/dist/src/{openclaw-B6XY2kUf.js → openclaw-D1D_ej1z.js} +21 -28
  197. package/dist/src/{openclaw-DDSfq5fp.cjs → openclaw-DAfWQn-o.cjs} +33 -39
  198. package/dist/src/opencode-sdk-C7m-wRfI.js +560 -0
  199. package/dist/src/opencode-sdk-CfaLN8PY.cjs +564 -0
  200. package/dist/src/opencode-sdk-D95s6SnR.js +562 -0
  201. package/dist/src/opencode-sdk-DxUPkLT7.js +560 -0
  202. package/dist/src/{otlpReceiver-DmRb0NBj.js → otlpReceiver--AIRW_S4.js} +53 -51
  203. package/dist/src/{otlpReceiver-Dg817agV.js → otlpReceiver-Bn5wGB1v.js} +53 -55
  204. package/dist/src/{otlpReceiver-B6Xo4KZM.cjs → otlpReceiver-Diec4cln.cjs} +53 -55
  205. package/dist/src/{otlpReceiver-BO0rbDzh.js → otlpReceiver-g3ByGaXs.js} +53 -55
  206. package/dist/src/{providerRegistry-Xf0qdqGQ.js → providerRegistry-B0RUOLI_.js} +7 -8
  207. package/dist/src/{providerRegistry-wCWd7sKQ.js → providerRegistry-CD8MEar9.js} +7 -8
  208. package/dist/src/{providerRegistry-lc7a7utN.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
  209. package/dist/src/providerRegistry-DM8rZYol.js +45 -0
  210. package/dist/src/providers-B3HvufyI.js +33246 -0
  211. package/dist/src/{providers-BiNq_Iyc.js → providers-BKRJTjBz.js} +1743 -1795
  212. package/dist/src/providers-C1rOSHiR.js +32 -0
  213. package/dist/src/{providers-BlEhY5mi.js → providers-CFLy1_ji.js} +1750 -1802
  214. package/dist/src/{providers-BNKVY53V.cjs → providers-CFu-TZl-.cjs} +2111 -2163
  215. package/dist/src/providers-CxmDwEFf.cjs +31 -0
  216. package/dist/src/providers-Dodakqr0.js +30 -0
  217. package/dist/src/providers-GIQ2TcsA.js +30 -0
  218. package/dist/src/{pythonUtils-r1uBuA0n.js → pythonUtils-C3py6GC1.js} +18 -19
  219. package/dist/src/{pythonUtils-DZ6EbdY4.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
  220. package/dist/src/{pythonUtils-vMlk9Qp5.js → pythonUtils-D5nxkQ0P.js} +18 -19
  221. package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
  222. package/dist/src/quiverai-C2jVwbH1.js +213 -0
  223. package/dist/src/quiverai-CI6gYJVI.js +213 -0
  224. package/dist/src/quiverai-CLkWkyZc.cjs +213 -0
  225. package/dist/src/quiverai-MHSxbmmZ.js +215 -0
  226. package/dist/src/{render-CAZvKKkB.js → render-Drod8m7K.js} +4 -5
  227. package/dist/src/{responses-DLLjADw5.js → responses-BKqJmhhc.js} +34 -27
  228. package/dist/src/{responses-TsdODUpm.js → responses-CGw0DCzh.js} +34 -27
  229. package/dist/src/responses-jxdehPkC.js +660 -0
  230. package/dist/src/{responses-zOtKtnY_.cjs → responses-tD4Bd4dc.cjs} +49 -42
  231. package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
  232. package/dist/src/{rubyUtils-Cs35SDYa.js → rubyUtils-BUVePouc.js} +27 -20
  233. package/dist/src/rubyUtils-BcuGX77l.js +222 -0
  234. package/dist/src/{rubyUtils-BtjlqyXt.js → rubyUtils-Boc4HZzX.js} +18 -19
  235. package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
  236. package/dist/src/{rubyUtils-DCVaJ3mc.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
  237. package/dist/src/{sagemaker-Du4LIR97.js → sagemaker-BK4Zb993.js} +75 -70
  238. package/dist/src/{sagemaker-CLdUAv5z.js → sagemaker-BfiWTmvn.js} +77 -77
  239. package/dist/src/{sagemaker-DwNnEVYt.cjs → sagemaker-CcQHM1jV.cjs} +75 -76
  240. package/dist/src/{sagemaker-BcgLu0U4.js → sagemaker-D2Q1c-sD.js} +75 -79
  241. package/dist/src/{scanner-Dyw21Wg_.js → scanner-J8CA3LsV.js} +149 -122
  242. package/dist/src/server/index.js +5620 -67302
  243. package/dist/src/{server-CgUQ25qW.cjs → server-B0PPuDw-.cjs} +57 -67
  244. package/dist/src/server-B1vi21hA.js +7 -0
  245. package/dist/src/{server-CbMTRQkg.js → server-BC7XJFgr.js} +19 -24
  246. package/dist/src/server-Cm9Kai_h.cjs +5 -0
  247. package/dist/src/{server-DWmZLfCy.js → server-DbFphssR.js} +26 -29
  248. package/dist/src/server-OAs3nBRT.js +229 -0
  249. package/dist/src/{signal-Bl32q42d.js → signal-BOTbd53Z.js} +9 -11
  250. package/dist/src/{slack-BtMkB6xP.cjs → slack-BmVAVGaK.cjs} +7 -8
  251. package/dist/src/{slack-OZYxoVON.js → slack-DCUPTzS2.js} +8 -8
  252. package/dist/src/{slack-DPqj42Ts.js → slack-DOdy_kyv.js} +7 -8
  253. package/dist/src/{slack-BfdBx2tO.js → slack-DXMKtA-f.js} +7 -9
  254. package/dist/src/store-BNmZ1KAz.cjs +5 -0
  255. package/dist/src/{store-BqwfFEyF.cjs → store-BSc-TF2w.cjs} +44 -45
  256. package/dist/src/store-BltJg2cd.js +6 -0
  257. package/dist/src/{store-D4gdn9ih.js → store-D1tv90v3.js} +34 -35
  258. package/dist/src/{store-2ocbYY9D.js → store-DQLEjuEO.js} +40 -36
  259. package/dist/src/store-Ub2vaGJ1.js +228 -0
  260. package/dist/src/{tables-D-NSwNIb.js → tables-5EvT_Bwn.js} +23 -23
  261. package/dist/src/{tables-B9E1kRp-.cjs → tables-C7K-XKWp.cjs} +93 -93
  262. package/dist/src/{tables-C7TT2XVn.js → tables-D36WTqKX.js} +25 -25
  263. package/dist/src/tables-xKANLRBD.js +288 -0
  264. package/dist/src/telemetry-5BCRNBbe.cjs +5 -0
  265. package/dist/src/{telemetry-DZ_7PaVq.js → telemetry-C15ziL8u.js} +17 -14
  266. package/dist/src/{telemetry-BXyVqyAg.js → telemetry-C2YDkUQH.js} +11 -13
  267. package/dist/src/{telemetry-D0_yFdtU.cjs → telemetry-CbrnxHp_.cjs} +21 -24
  268. package/dist/src/telemetry-D4W5hboe.js +7 -0
  269. package/dist/src/telemetry-DMb2Mpfm.js +171 -0
  270. package/dist/src/{text-Dm78AVGG.js → text-B_UCRPp2.js} +2 -2
  271. package/dist/src/{text-DF2hMKdg.cjs → text-CW1cyrwj.cjs} +12 -13
  272. package/dist/src/{text-DgMr_tiM.js → text-Db-Wt2u2.js} +2 -2
  273. package/dist/src/text-TIv0QYnd.js +22 -0
  274. package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
  275. package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
  276. package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
  277. package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
  278. package/dist/src/{transcription-FNIz3YOe.cjs → transcription-CL78qbOU.cjs} +14 -15
  279. package/dist/src/{transcription-C-M81iDA.js → transcription-DAtxHhAM.js} +14 -7
  280. package/dist/src/{transcription-CYuY5sFO.js → transcription-LNZTNUUL.js} +14 -16
  281. package/dist/src/{transcription-Ch7S-LWw.js → transcription-QHh3AH6Z.js} +15 -17
  282. package/dist/src/{transform-CoP2bJ7P.js → transform-Cgi24fJ7.js} +94 -66
  283. package/dist/src/{transform-Kd6u-oNm.cjs → transform-CzK1Q0zl.cjs} +24 -26
  284. package/dist/src/{transform-D8dILpfZ.js → transform-DECvGmzp.js} +15 -13
  285. package/dist/src/{transform-DMaxQwDx.js → transform-DGLazrMm.js} +94 -66
  286. package/dist/src/transform-DGxXocjk.js +1506 -0
  287. package/dist/src/{transform-ivxEY4f7.cjs → transform-DOcQeLld.cjs} +234 -206
  288. package/dist/src/transform-DTGDnAzW.js +6 -0
  289. package/dist/src/{transform-CqTFr7KR.js → transform-DilY9wbS.js} +10 -12
  290. package/dist/src/transform-aa6tmVpZ.js +216 -0
  291. package/dist/src/transform-m3qNw4KP.cjs +5 -0
  292. package/dist/src/{transformersAvailability-DEU2naS1.js → transformersAvailability-CEVM2GNQ.js} +2 -2
  293. package/dist/src/{transformersAvailability-Bkep3ka7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
  294. package/dist/src/{transformersAvailability-DwmezkVe.js → transformersAvailability-D6c6ROpT.js} +2 -2
  295. package/dist/src/{types-t52w-XsS.js → types-CH3Ge2sE.js} +103 -92
  296. package/dist/src/{types-DMVjYLpx.js → types-CLKiCBW3.js} +98 -91
  297. package/dist/src/types-CN_TZ2GJ.js +3260 -0
  298. package/dist/src/{types-BIfttHrT.cjs → types-LJ0r3wbR.cjs} +573 -566
  299. package/dist/src/util-5cB-L7U3.js +1430 -0
  300. package/dist/src/util-6-GqIvzS.js +599 -0
  301. package/dist/src/{util-vjscpUzy.js → util-B7T3SiBS.js} +5 -6
  302. package/dist/src/{util-Cl0zfT3V.js → util-Betm42rL.js} +44 -17
  303. package/dist/src/{util-CUEt0Vum.js → util-C-PPYSMq.js} +44 -17
  304. package/dist/src/{util-DkFTvieG.cjs → util-CchiqXh_.cjs} +35 -36
  305. package/dist/src/{util-mJ58qbbw.js → util-DaWTWKBK.js} +5 -6
  306. package/dist/src/{util-C08Kns6-.cjs → util-Db0a0AFH.cjs} +89 -62
  307. package/dist/src/{util-DiCePfDu.js → util-Dlz_Wvgm.js} +102 -53
  308. package/dist/src/{util-BSh4a_Q8.js → util-YT5HPZaS.js} +102 -53
  309. package/dist/src/{util-DUYOvxAy.cjs → util-Yz-1aEhW.cjs} +274 -219
  310. package/dist/src/util-ZZH-3QZz.js +293 -0
  311. package/dist/src/{utils-DFaZa6Rf.cjs → utils-Cz9qXqII.cjs} +32 -35
  312. package/dist/src/{utils-CVzb4YiI.js → utils-XiOAgly5.js} +4 -7
  313. package/dist/src/utils-dLokC-eR.js +94 -0
  314. package/dist/src/{utils-JaY9veb5.js → utils-f2-Moju7.js} +4 -7
  315. package/dist/tsconfig.tsbuildinfo +1 -1
  316. package/package.json +59 -53
  317. package/dist/src/app/assets/index-BOgkICuY.css +0 -1
  318. package/dist/src/app/assets/index-CSgqn_Vd.js +0 -428
  319. package/dist/src/app/tsconfig.app.tsbuildinfo +0 -1
  320. package/dist/src/base-BaXmtXYp.js +0 -107
  321. package/dist/src/base-Dtp8b4_N.js +0 -106
  322. package/dist/src/base-f71xxWai.cjs +0 -111
  323. package/dist/src/cache-BUPcq0Ad.js +0 -6
  324. package/dist/src/cache-CVfRb-HD.cjs +0 -6
  325. package/dist/src/cache-O4EuX2JV.js +0 -8
  326. package/dist/src/chunk-DHDDz29n.js +0 -22
  327. package/dist/src/chunk-FhC4c-0y.js +0 -21
  328. package/dist/src/cloud-CZ4hytdm.js +0 -5
  329. package/dist/src/eval-CKHWqG9f.js +0 -16
  330. package/dist/src/evalResult-CxTP-LMm.cjs +0 -11
  331. package/dist/src/evalResult-CzLURDcP.js +0 -13
  332. package/dist/src/evalResult-DyttNQ_G.js +0 -11
  333. package/dist/src/evaluator-0PvfeBYh.js +0 -38
  334. package/dist/src/fetch-Bi0o-fdp.js +0 -4
  335. package/dist/src/fetch-CMptBDVg.cjs +0 -4
  336. package/dist/src/fetch-DAZkv3gV.js +0 -6
  337. package/dist/src/graders-BCytzXrb.js +0 -34
  338. package/dist/src/graders-CGZQShfJ.cjs +0 -33
  339. package/dist/src/graders-spkuVC-E.js +0 -36
  340. package/dist/src/opencode-sdk-CImWVqy9.js +0 -382
  341. package/dist/src/opencode-sdk-CuCztr4P.js +0 -380
  342. package/dist/src/opencode-sdk-DhcfRbBH.js +0 -376
  343. package/dist/src/opencode-sdk-mqF-Oj3f.cjs +0 -383
  344. package/dist/src/providers-BMZZmPBJ.cjs +0 -32
  345. package/dist/src/providers-CQQrNaJk.js +0 -32
  346. package/dist/src/providers-Ck8HyrC-.js +0 -34
  347. package/dist/src/quiverai-BNfIwKCO.cjs +0 -54
  348. package/dist/src/quiverai-BQigKdIH.js +0 -57
  349. package/dist/src/quiverai-Bfy2WnE2.js +0 -55
  350. package/dist/src/quiverai-CedIP0PJ.js +0 -43
  351. package/dist/src/rubyUtils-D7--T12C.js +0 -6
  352. package/dist/src/rubyUtils-DRRiMFV2.js +0 -5
  353. package/dist/src/rubyUtils-vb8OYFC-.cjs +0 -5
  354. package/dist/src/server-BUbS0Qfh.js +0 -6
  355. package/dist/src/server-XpGXFHkS.cjs +0 -6
  356. package/dist/src/server-gfOx5Zrk.js +0 -8
  357. package/dist/src/store-5u2yriTV.js +0 -7
  358. package/dist/src/store-D_lq_8oQ.js +0 -6
  359. package/dist/src/store-m5KT6Ly7.cjs +0 -6
  360. package/dist/src/telemetry-5RHFoCJh.js +0 -6
  361. package/dist/src/telemetry-Do8wMnA-.js +0 -8
  362. package/dist/src/telemetry-LojxPoFq.cjs +0 -6
  363. package/dist/src/transform-8eGmaH-7.js +0 -7
  364. package/dist/src/transform-BRVvWaG4.cjs +0 -6
  365. package/dist/src/transform-GybT0X0u.js +0 -8
  366. package/dist/src/transformersAvailability-DkAWaK5B.js +0 -35
@@ -0,0 +1,713 @@
1
+ import { t as invariant } from "./invariant-vgHWClmd.js";
2
+ import dotenv from "dotenv";
3
+ import fs from "fs";
4
+ import * as path$1 from "path";
5
+ import path from "path";
6
+ import chalk from "chalk";
7
+ import winston from "winston";
8
+ import * as os$1 from "os";
9
+ import Ajv from "ajv";
10
+ import addFormats from "ajv-formats";
11
+ import yaml from "js-yaml";
12
+ import safeStringify from "fast-safe-stringify";
13
+ //#region src/cliState.ts
14
+ const state = {};
15
+ //#endregion
16
+ //#region src/envars.ts
17
+ dotenv.config({ quiet: true });
18
+ function getEnvString(key, defaultValue) {
19
+ if (state.config?.env && typeof state.config.env === "object") {
20
+ const envValue = state.config.env[key];
21
+ if (envValue !== void 0) return String(envValue);
22
+ }
23
+ const value = process.env[key];
24
+ if (value === void 0) return defaultValue;
25
+ return value;
26
+ }
27
+ /**
28
+ * Get a boolean environment variable.
29
+ * @param key The name of the environment variable.
30
+ * @param defaultValue Optional default value if the environment variable is not set.
31
+ * @returns The boolean value of the environment variable, or the default value if provided.
32
+ */
33
+ function getEnvBool(key, defaultValue) {
34
+ const value = getEnvString(key) || defaultValue;
35
+ if (typeof value === "boolean") return value;
36
+ if (typeof value === "string") return [
37
+ "1",
38
+ "true",
39
+ "yes",
40
+ "yup",
41
+ "yeppers"
42
+ ].includes(value.toLowerCase());
43
+ return Boolean(defaultValue);
44
+ }
45
+ function getEnvInt(key, defaultValue) {
46
+ const value = getEnvString(key) || defaultValue;
47
+ if (typeof value === "number") return Math.floor(value);
48
+ if (typeof value === "string") {
49
+ const parsedValue = Number.parseInt(value, 10);
50
+ if (!Number.isNaN(parsedValue)) return parsedValue;
51
+ }
52
+ return defaultValue;
53
+ }
54
+ function getEnvFloat(key, defaultValue) {
55
+ const value = getEnvString(key) || defaultValue;
56
+ if (typeof value === "number") return value;
57
+ if (typeof value === "string") {
58
+ const parsedValue = Number.parseFloat(value);
59
+ if (!Number.isNaN(parsedValue)) return parsedValue;
60
+ }
61
+ return defaultValue;
62
+ }
63
+ /**
64
+ * Get the timeout in milliseconds for each individual test case/provider API call.
65
+ * When this timeout is reached, that specific test is marked as an error.
66
+ * @param defaultValue Optional default value if the environment variable is not set. Defaults to 0 (no timeout).
67
+ * @returns The timeout value in milliseconds, or the default value if not set.
68
+ */
69
+ function getEvalTimeoutMs(defaultValue = 0) {
70
+ return getEnvInt("PROMPTFOO_EVAL_TIMEOUT_MS", defaultValue);
71
+ }
72
+ /**
73
+ * Get the maximum total runtime in milliseconds for the entire evaluation process.
74
+ * When this timeout is reached, all remaining tests are marked as errors and the evaluation ends.
75
+ * @param defaultValue Optional default value if the environment variable is not set. Defaults to 0 (no limit).
76
+ * @returns The max duration in milliseconds, or the default value if not set.
77
+ */
78
+ function getMaxEvalTimeMs(defaultValue = 0) {
79
+ return getEnvInt("PROMPTFOO_MAX_EVAL_TIME_MS", defaultValue);
80
+ }
81
+ /**
82
+ * Check if the application is running in a CI environment.
83
+ * @returns True if running in a CI environment, false otherwise.
84
+ */
85
+ function isCI() {
86
+ return getEnvBool("CI") || getEnvBool("GITHUB_ACTIONS") || getEnvBool("TRAVIS") || getEnvBool("CIRCLECI") || getEnvBool("JENKINS") || getEnvBool("GITLAB_CI") || getEnvBool("APPVEYOR") || getEnvBool("CODEBUILD_BUILD_ID") || getEnvBool("TF_BUILD") || getEnvBool("BITBUCKET_COMMIT") || getEnvBool("BUDDY") || getEnvBool("BUILDKITE") || getEnvBool("TEAMCITY_VERSION");
87
+ }
88
+ //#endregion
89
+ //#region src/util/config/manage.ts
90
+ let configDirectoryPath = getEnvString("PROMPTFOO_CONFIG_DIR");
91
+ const isNodeEnvironment = typeof process !== "undefined" && process.versions && process.versions.node;
92
+ function getConfigDirectoryPath(createIfNotExists = false) {
93
+ const p = configDirectoryPath || path$1.join(os$1.homedir(), ".promptfoo");
94
+ if (createIfNotExists && isNodeEnvironment) try {
95
+ fs.mkdirSync(p, { recursive: true });
96
+ } catch {}
97
+ return p;
98
+ }
99
+ //#endregion
100
+ //#region src/util/json.ts
101
+ let ajvInstance = null;
102
+ function getAjv() {
103
+ if (!ajvInstance) {
104
+ ajvInstance = new Ajv({ strictSchema: !getEnvBool("PROMPTFOO_DISABLE_AJV_STRICT_MODE") });
105
+ addFormats(ajvInstance);
106
+ }
107
+ return ajvInstance;
108
+ }
109
+ function isValidJson(str) {
110
+ try {
111
+ JSON.parse(str);
112
+ return true;
113
+ } catch {
114
+ return false;
115
+ }
116
+ }
117
+ /**
118
+ * Creates a truncated version of an object for safe JSON stringification.
119
+ * Prevents memory issues by limiting string, array, and object sizes.
120
+ *
121
+ * @param value - The value to truncate and stringify
122
+ * @param prettyPrint - Whether to format the JSON with indentation
123
+ * @returns A JSON string representation of the truncated value
124
+ */
125
+ function safeJsonStringifyTruncated(value, prettyPrint = false) {
126
+ const cache = /* @__PURE__ */ new Set();
127
+ const space = prettyPrint ? 2 : void 0;
128
+ const truncateValue = (val) => {
129
+ if (typeof val === "string") return val.length > 1e3 ? val.substring(0, 1e3) + "...[truncated]" : val;
130
+ if (Array.isArray(val)) {
131
+ const truncated = val.slice(0, 10).map(truncateValue);
132
+ if (val.length > 10) truncated.push(`...[${val.length - 10} more items]`);
133
+ return truncated;
134
+ }
135
+ if (typeof val === "object" && val !== null) {
136
+ if (cache.has(val)) return "[Circular Reference]";
137
+ cache.add(val);
138
+ const truncated = {};
139
+ let count = 0;
140
+ for (const [k, v] of Object.entries(val)) {
141
+ if (count >= 20) {
142
+ truncated["...[truncated]"] = `${Object.keys(val).length - count} more keys`;
143
+ break;
144
+ }
145
+ truncated[k] = truncateValue(v);
146
+ count++;
147
+ }
148
+ return truncated;
149
+ }
150
+ return val;
151
+ };
152
+ try {
153
+ return JSON.stringify(truncateValue(value), null, space) || "{}";
154
+ } catch {
155
+ return `{"error": "Failed to stringify even truncated data", "type": "${typeof value}", "constructor": "${value?.constructor?.name || "unknown"}"}`;
156
+ }
157
+ }
158
+ /**
159
+ * Safely stringify a value to JSON, handling circular references and large objects.
160
+ *
161
+ * @param value - The value to stringify
162
+ * @param prettyPrint - Whether to format the JSON with indentation
163
+ * @returns JSON string representation, or undefined if serialization fails
164
+ */
165
+ function safeJsonStringify(value, prettyPrint = false) {
166
+ const cache = /* @__PURE__ */ new Set();
167
+ const space = prettyPrint ? 2 : void 0;
168
+ try {
169
+ return JSON.stringify(value, (_key, val) => {
170
+ if (typeof val === "object" && val !== null) {
171
+ if (cache.has(val)) return;
172
+ cache.add(val);
173
+ }
174
+ return val;
175
+ }, space) || void 0;
176
+ } catch (error) {
177
+ if (error instanceof RangeError && error.message.includes("Invalid string length")) return safeJsonStringifyTruncated(value, prettyPrint);
178
+ return;
179
+ }
180
+ }
181
+ function convertSlashCommentsToHash(str) {
182
+ return str.split("\n").map((line) => {
183
+ let state = "normal";
184
+ let result = "";
185
+ let i = 0;
186
+ while (i < line.length) {
187
+ const char = line[i];
188
+ const nextChar = line[i + 1];
189
+ const prevChar = i > 0 ? line[i - 1] : "";
190
+ switch (state) {
191
+ case "normal":
192
+ if (char === "'" && !/[a-zA-Z]/.test(prevChar)) {
193
+ state = "singleQuote";
194
+ result += char;
195
+ } else if (char === "\"") {
196
+ state = "doubleQuote";
197
+ result += char;
198
+ } else if (char === "/" && nextChar === "/") {
199
+ let tokenStart = 0;
200
+ for (let j = i - 1; j >= 0; j--) if (/\s/.test(line[j])) {
201
+ tokenStart = j + 1;
202
+ break;
203
+ }
204
+ if (line.slice(tokenStart, i + 2).includes("://")) {
205
+ result += char;
206
+ break;
207
+ }
208
+ let slashCount = 2;
209
+ while (i + slashCount < line.length && line[i + slashCount] === "/") slashCount++;
210
+ const hashes = "#".repeat(Math.floor(slashCount / 2));
211
+ return result + hashes + line.slice(i + slashCount);
212
+ } else result += char;
213
+ break;
214
+ case "singleQuote":
215
+ result += char;
216
+ if (char === "'" && prevChar !== "\\" && !/[a-zA-Z]/.test(nextChar)) state = "normal";
217
+ break;
218
+ case "doubleQuote":
219
+ result += char;
220
+ if (char === "\"" && prevChar !== "\\") state = "normal";
221
+ break;
222
+ }
223
+ i++;
224
+ }
225
+ return result;
226
+ }).join("\n");
227
+ }
228
+ function extractJsonObjects(str) {
229
+ const jsonObjects = [];
230
+ const maxJsonLength = 1e5;
231
+ for (let i = 0; i < str.length; i++) if (str[i] === "{") {
232
+ let openBraces = 1;
233
+ let closeBraces = 0;
234
+ let j = i + 1;
235
+ while (j < Math.min(i + maxJsonLength, str.length) && openBraces > closeBraces) {
236
+ if (str[j] === "{") openBraces++;
237
+ if (str[j] === "}") closeBraces++;
238
+ j++;
239
+ if (openBraces === closeBraces || j === str.length || j === i + maxJsonLength) try {
240
+ let potentialJson = str.slice(i, j);
241
+ if (openBraces > closeBraces) potentialJson += "}".repeat(openBraces - closeBraces);
242
+ const processedJson = convertSlashCommentsToHash(potentialJson);
243
+ const parsedObj = yaml.load(processedJson, { json: true });
244
+ if (typeof parsedObj === "object" && parsedObj !== null) {
245
+ jsonObjects.push(parsedObj);
246
+ i = j - 1;
247
+ break;
248
+ }
249
+ } catch {
250
+ if (openBraces === closeBraces) break;
251
+ }
252
+ }
253
+ }
254
+ return jsonObjects;
255
+ }
256
+ function extractFirstJsonObject(str) {
257
+ const jsonObjects = extractJsonObjects(str);
258
+ invariant(jsonObjects.length >= 1, `Expected a JSON object, but got ${JSON.stringify(str)}`);
259
+ return jsonObjects[0];
260
+ }
261
+ /**
262
+ * Reorders the keys of an object based on a specified order, preserving any unspecified keys.
263
+ * Symbol keys are preserved and added at the end.
264
+ *
265
+ * @param obj - The object whose keys need to be reordered.
266
+ * @param order - An array specifying the desired order of keys.
267
+ * @returns A new object with keys reordered according to the specified order.
268
+ *
269
+ * @example
270
+ * const obj = { c: 3, a: 1, b: 2 };
271
+ * const orderedObj = orderKeys(obj, ['a', 'b']);
272
+ * // Result: { a: 1, b: 2, c: 3 }
273
+ */
274
+ function orderKeys(obj, order) {
275
+ const result = {};
276
+ for (const key of order) if (key in obj && obj[key] !== void 0) result[key] = obj[key];
277
+ for (const key in obj) if (!(key in result) && obj[key] !== void 0) result[key] = obj[key];
278
+ const symbolKeys = Object.getOwnPropertySymbols(obj);
279
+ for (const sym of symbolKeys) if (obj[sym] !== void 0) result[sym] = obj[sym];
280
+ return result;
281
+ }
282
+ /**
283
+ * Creates a summary of an EvaluateResult for logging purposes, avoiding RangeError
284
+ * when stringifying large evaluation results.
285
+ *
286
+ * Extracts key information while truncating potentially large fields like response
287
+ * outputs and metadata values.
288
+ *
289
+ * @param result - The evaluation result to summarize
290
+ * @param maxOutputLength - Maximum length for response output before truncation. Default: 500
291
+ * @param includeMetadataKeys - Whether to include metadata keys in the summary. Default: true
292
+ * @returns A summarized version safe for JSON stringification
293
+ * @throws {TypeError} If result is null or undefined
294
+ */
295
+ function summarizeEvaluateResultForLogging(result, maxOutputLength = 500, includeMetadataKeys = true) {
296
+ if (!result) throw new TypeError("EvaluateResult cannot be null or undefined");
297
+ const summary = {
298
+ id: result.id,
299
+ testIdx: result.testIdx,
300
+ promptIdx: result.promptIdx,
301
+ success: result.success,
302
+ score: result.score,
303
+ error: result.error,
304
+ failureReason: result.failureReason
305
+ };
306
+ if (result.provider) summary.provider = {
307
+ id: result.provider.id || "",
308
+ label: result.provider.label
309
+ };
310
+ if (result.response) {
311
+ summary.response = {
312
+ error: result.response.error,
313
+ cached: result.response.cached,
314
+ cost: result.response.cost,
315
+ tokenUsage: result.response.tokenUsage
316
+ };
317
+ if (result.response.output != null) {
318
+ const output = String(result.response.output);
319
+ summary.response.output = output.length > maxOutputLength ? output.substring(0, maxOutputLength) + "...[truncated]" : output;
320
+ }
321
+ if (result.response.metadata && includeMetadataKeys) summary.response.metadata = {
322
+ keys: Object.keys(result.response.metadata),
323
+ keyCount: Object.keys(result.response.metadata).length
324
+ };
325
+ }
326
+ if (result.testCase) summary.testCase = {
327
+ description: result.testCase.description,
328
+ vars: result.testCase.vars ? Object.keys(result.testCase.vars) : void 0
329
+ };
330
+ return summary;
331
+ }
332
+ //#endregion
333
+ //#region src/util/sanitizer.ts
334
+ /**
335
+ * Generic utility functions for sanitizing objects to prevent logging of secrets and credentials
336
+ * Uses a custom recursive approach for reliable deep object sanitization.
337
+ */
338
+ const MAX_DEPTH = 4;
339
+ const DUMMY_BASE = "http://placeholder";
340
+ const REDACTED = "[REDACTED]";
341
+ /**
342
+ * Set of field names that should be redacted (case-insensitive, with hyphens/underscores normalized)
343
+ * Note: Keys are stored in their normalized form (lowercase, no hyphens/underscores)
344
+ */
345
+ const SECRET_FIELD_NAMES = new Set([
346
+ "password",
347
+ "passwd",
348
+ "pwd",
349
+ "secret",
350
+ "secrets",
351
+ "secretkey",
352
+ "credentials",
353
+ "apikey",
354
+ "apisecret",
355
+ "token",
356
+ "accesstoken",
357
+ "refreshtoken",
358
+ "idtoken",
359
+ "bearertoken",
360
+ "authtoken",
361
+ "clientsecret",
362
+ "webhooksecret",
363
+ "anthropicapikey",
364
+ "awsbearertokenbedrock",
365
+ "authorization",
366
+ "auth",
367
+ "bearer",
368
+ "apikeyenvar",
369
+ "xapikey",
370
+ "xauthtoken",
371
+ "xaccesstoken",
372
+ "xauth",
373
+ "xsecret",
374
+ "xcsrftoken",
375
+ "xsessiondata",
376
+ "csrftoken",
377
+ "sessionid",
378
+ "session",
379
+ "cookie",
380
+ "setcookie",
381
+ "certificatepassword",
382
+ "keystorepassword",
383
+ "pfxpassword",
384
+ "privatekey",
385
+ "certkey",
386
+ "encryptionkey",
387
+ "signingkey",
388
+ "signature",
389
+ "sig",
390
+ "passphrase",
391
+ "certificatecontent",
392
+ "keystorecontent",
393
+ "pfx",
394
+ "pfxcontent",
395
+ "keycontent",
396
+ "certcontent"
397
+ ]);
398
+ /**
399
+ * Normalize field names for comparison (lowercase, no hyphens/underscores)
400
+ */
401
+ function normalizeFieldName(fieldName) {
402
+ return fieldName.toLowerCase().replace(/[-_]/g, "");
403
+ }
404
+ /**
405
+ * Check if a field name should be redacted
406
+ */
407
+ function isSecretField(fieldName) {
408
+ return SECRET_FIELD_NAMES.has(normalizeFieldName(fieldName));
409
+ }
410
+ /**
411
+ * Check if a value looks like a secret based on common patterns.
412
+ * Detects API keys, tokens, and other credential patterns.
413
+ */
414
+ function looksLikeSecret(value) {
415
+ if (typeof value !== "string") return false;
416
+ if (/^sk-[a-zA-Z0-9-_]{20,}/.test(value)) return true;
417
+ if (/^sk-proj-[a-zA-Z0-9-_]{20,}/.test(value)) return true;
418
+ if (/^sk-ant-[a-zA-Z0-9-_]{20,}/.test(value)) return true;
419
+ if (/^key-[a-zA-Z0-9]{20,}/.test(value)) return true;
420
+ if (/^Bearer\s+.{20,}/i.test(value)) return true;
421
+ if (/^Basic\s+.{20,}/i.test(value)) return true;
422
+ if (/^[a-zA-Z0-9+/=_-]{64,}$/.test(value)) return true;
423
+ if (/^AKIA[A-Z0-9]{16}/.test(value)) return true;
424
+ if (/^AIza[a-zA-Z0-9_-]{35}/.test(value)) return true;
425
+ return false;
426
+ }
427
+ /**
428
+ * Detect class instances (objects with custom prototypes and methods)
429
+ */
430
+ function isClassInstance(obj) {
431
+ const proto = Object.getPrototypeOf(obj);
432
+ if (!proto || proto === Object.prototype) return false;
433
+ return Object.getOwnPropertyNames(proto).some((prop) => prop !== "constructor" && typeof proto[prop] === "function");
434
+ }
435
+ /**
436
+ * Parse and sanitize JSON strings, also check if the string looks like a secret
437
+ */
438
+ function sanitizeJsonString(str, depth, maxDepth) {
439
+ try {
440
+ const parsed = JSON.parse(str);
441
+ if (parsed && typeof parsed === "object") {
442
+ const sanitized = recursiveSanitize(parsed, depth, maxDepth);
443
+ return JSON.stringify(sanitized);
444
+ }
445
+ } catch {
446
+ if (looksLikeSecret(str)) return REDACTED;
447
+ }
448
+ return str;
449
+ }
450
+ /**
451
+ * Sanitize plain object fields
452
+ */
453
+ function sanitizePlainObject(obj, depth, maxDepth) {
454
+ const sanitized = {};
455
+ for (const [key, value] of Object.entries(obj)) if (key === "url" && typeof value === "string") sanitized[key] = sanitizeUrl(value);
456
+ else if (isSecretField(key)) sanitized[key] = REDACTED;
457
+ else if (typeof value === "string" && looksLikeSecret(value)) sanitized[key] = REDACTED;
458
+ else sanitized[key] = recursiveSanitize(value, depth + 1, maxDepth);
459
+ return sanitized;
460
+ }
461
+ /**
462
+ * Recursively sanitize an object, redacting secret fields at any depth
463
+ */
464
+ function recursiveSanitize(obj, depth = 0, maxDepth = MAX_DEPTH) {
465
+ if (typeof obj === "function") return `[Function] ${obj.name}`;
466
+ if (typeof obj === "string") return sanitizeJsonString(obj, depth, maxDepth);
467
+ if (obj === null || obj === void 0 || typeof obj !== "object") return obj;
468
+ if (depth > maxDepth) return "[...]";
469
+ if (Array.isArray(obj)) return obj.map((item) => recursiveSanitize(item, depth + 1, maxDepth));
470
+ if (isClassInstance(obj)) return `[${obj.constructor?.name || "Object"} Instance]`;
471
+ return sanitizePlainObject(obj, depth, maxDepth);
472
+ }
473
+ /**
474
+ * Generic function to sanitize any object by removing or redacting sensitive information
475
+ * @param obj - The object to sanitize
476
+ * @param options - Optional configuration
477
+ * @returns A sanitized copy of the object with secrets redacted
478
+ */
479
+ function sanitizeObject(obj, options = {}) {
480
+ const { context = "object", throwOnError = false, maxDepth = MAX_DEPTH } = options;
481
+ try {
482
+ if (obj === null || obj === void 0) return obj;
483
+ if (typeof obj === "string") return sanitizeJsonString(obj, 0, maxDepth);
484
+ if (typeof obj !== "object") return obj;
485
+ return recursiveSanitize(JSON.parse(safeStringify(obj, (_key, val) => {
486
+ if (val instanceof Error) return {
487
+ name: val.name,
488
+ message: val.message
489
+ };
490
+ return val;
491
+ }, void 0, {
492
+ depthLimit: Number.MAX_SAFE_INTEGER,
493
+ edgesLimit: Number.MAX_SAFE_INTEGER
494
+ })), 0, maxDepth);
495
+ } catch (error) {
496
+ if (throwOnError) throw error;
497
+ console.error(`Error sanitizing ${context}:`, error);
498
+ return obj;
499
+ }
500
+ }
501
+ function sanitizeUrl(url) {
502
+ try {
503
+ if (typeof url !== "string" || !url.trim()) return url;
504
+ if (url.includes("{{") && url.includes("}}")) return url;
505
+ const isPathOnly = url.startsWith("/") && !url.startsWith("//");
506
+ const parsedUrl = isPathOnly ? new URL(url, DUMMY_BASE) : new URL(url);
507
+ const sanitizedUrl = new URL(parsedUrl.href);
508
+ if (sanitizedUrl.username || sanitizedUrl.password) {
509
+ sanitizedUrl.username = "***";
510
+ sanitizedUrl.password = "***";
511
+ }
512
+ const sensitiveParams = /(api[_-]?key|token|password|secret|signature|sig|access[_-]?token|refresh[_-]?token|id[_-]?token|client[_-]?secret|authorization)/i;
513
+ try {
514
+ for (const key of Array.from(sanitizedUrl.searchParams.keys())) if (sensitiveParams.test(key)) sanitizedUrl.searchParams.set(key, "[REDACTED]");
515
+ } catch (paramError) {
516
+ console.warn(`Failed to sanitize URL parameters ${url}: ${paramError}`);
517
+ }
518
+ if (isPathOnly) return sanitizedUrl.pathname + sanitizedUrl.search + sanitizedUrl.hash;
519
+ return sanitizedUrl.toString();
520
+ } catch (error) {
521
+ console.warn(`Failed to sanitize URL ${url}: ${error}`);
522
+ return url;
523
+ }
524
+ }
525
+ //#endregion
526
+ //#region src/logger.ts
527
+ let globalLogCallback = null;
528
+ function setLogCallback(callback) {
529
+ globalLogCallback = callback;
530
+ }
531
+ const LOG_LEVELS = {
532
+ error: 0,
533
+ warn: 1,
534
+ info: 2,
535
+ debug: 3
536
+ };
537
+ let sourceMapSupportInitialized = false;
538
+ async function initializeSourceMapSupport() {
539
+ if (!sourceMapSupportInitialized) try {
540
+ (await import("source-map-support")).install();
541
+ sourceMapSupportInitialized = true;
542
+ } catch {}
543
+ }
544
+ /**
545
+ * Gets the caller location (filename and line number)
546
+ * @returns String with file location information
547
+ */
548
+ function getCallerLocation() {
549
+ try {
550
+ const callerLine = ((/* @__PURE__ */ new Error("stack trace capture")).stack?.split("\n") || [])[3];
551
+ if (callerLine) {
552
+ const matchParens = callerLine.match(/at (?:.*) \((.+):(\d+):(\d+)\)/);
553
+ const matchNormal = callerLine.match(/at (.+):(\d+):(\d+)/);
554
+ const match = matchParens || matchNormal;
555
+ if (match) {
556
+ const filePath = match[1];
557
+ const line = match[2];
558
+ return `[${path.basename(filePath)}:${line}]`;
559
+ }
560
+ }
561
+ } catch {}
562
+ return "";
563
+ }
564
+ /**
565
+ * Extracts the actual message string from potentially nested info objects
566
+ */
567
+ function extractMessage(info) {
568
+ if (typeof info.message === "object" && info.message !== null && "message" in info.message) return typeof info.message.message === "string" ? info.message.message : String(info.message.message);
569
+ return typeof info.message === "string" ? info.message : JSON.stringify(info.message);
570
+ }
571
+ const consoleFormatter = winston.format.printf((info) => {
572
+ const message = extractMessage(info);
573
+ if (globalLogCallback) globalLogCallback(message);
574
+ const location = info.location ? `${info.location} ` : "";
575
+ if (info.level === "error") return chalk.red(`${location}${message}`);
576
+ else if (info.level === "warn") return chalk.yellow(`${location}${message}`);
577
+ else if (info.level === "info") return `${location}${message}`;
578
+ else if (info.level === "debug") return `${chalk.cyan(location)}${message}`;
579
+ throw new Error(`Invalid log level: ${info.level}`);
580
+ });
581
+ winston.format.printf((info) => {
582
+ const timestamp = (/* @__PURE__ */ new Date()).toISOString();
583
+ const location = info.location ? ` ${info.location}` : "";
584
+ const message = extractMessage(info);
585
+ return `${timestamp} [${info.level.toUpperCase()}]${location}: ${message}`;
586
+ });
587
+ const winstonLogger = winston.createLogger({
588
+ levels: LOG_LEVELS,
589
+ transports: [new winston.transports.Console({
590
+ level: getEnvString("LOG_LEVEL", "info"),
591
+ format: winston.format.combine(winston.format.simple(), consoleFormatter)
592
+ })]
593
+ });
594
+ function getLogLevel() {
595
+ return winstonLogger.transports[0].level;
596
+ }
597
+ function setLogLevel(level) {
598
+ if (level in LOG_LEVELS) {
599
+ winstonLogger.transports[0].level = level;
600
+ if (level === "debug") initializeSourceMapSupport();
601
+ } else throw new Error(`Invalid log level: ${level}`);
602
+ }
603
+ function isDebugEnabled() {
604
+ return getLogLevel() === "debug";
605
+ }
606
+ /**
607
+ * Creates a logger method for the specified log level.
608
+ * Accepts either a string message or a structured object with a message field.
609
+ */
610
+ function createLogMethod(level) {
611
+ return (input) => {
612
+ const location = level === "debug" ? getCallerLocation() : isDebugEnabled() ? getCallerLocation() : "";
613
+ if (level === "debug") initializeSourceMapSupport();
614
+ const message = typeof input === "string" ? input : input.message;
615
+ return winstonLogger[level]({
616
+ message,
617
+ location
618
+ });
619
+ };
620
+ }
621
+ let internalLogger = Object.assign({}, winstonLogger, {
622
+ error: createLogMethod("error"),
623
+ warn: createLogMethod("warn"),
624
+ info: createLogMethod("info"),
625
+ debug: createLogMethod("debug"),
626
+ add: winstonLogger.add.bind(winstonLogger),
627
+ remove: winstonLogger.remove.bind(winstonLogger),
628
+ transports: winstonLogger.transports
629
+ });
630
+ /**
631
+ * Sanitizes context object for logging using generic sanitization
632
+ */
633
+ function sanitizeContext(context) {
634
+ const contextWithSanitizedUrls = {};
635
+ for (const [key, value] of Object.entries(context)) if (key === "url" && typeof value === "string") contextWithSanitizedUrls[key] = sanitizeUrl(value);
636
+ else contextWithSanitizedUrls[key] = value;
637
+ return sanitizeObject(contextWithSanitizedUrls, { context: "log context" });
638
+ }
639
+ /**
640
+ * Creates a log method that accepts an optional context parameter.
641
+ * If context is provided, it will be sanitized and formatted.
642
+ *
643
+ * When structured logging is enabled (via setStructuredLogging(true)):
644
+ * - Passes { message, ...context } object to the logger
645
+ * - Ideal for cloud logging integrations that expect structured data
646
+ *
647
+ * When structured logging is disabled (default):
648
+ * - Formats context as JSON string appended to message
649
+ * - Suitable for CLI/console output
650
+ */
651
+ function createLogMethodWithContext(level) {
652
+ return (message, context) => {
653
+ if (!context) {
654
+ internalLogger[level](message);
655
+ return;
656
+ }
657
+ const sanitized = sanitizeContext(context);
658
+ {
659
+ const contextStr = safeJsonStringify(sanitized, true);
660
+ internalLogger[level](`${message}\n${contextStr}`);
661
+ }
662
+ };
663
+ }
664
+ const logger = {
665
+ error: createLogMethodWithContext("error"),
666
+ warn: createLogMethodWithContext("warn"),
667
+ info: createLogMethodWithContext("info"),
668
+ debug: createLogMethodWithContext("debug"),
669
+ add: (transport) => internalLogger.add ? internalLogger.add(transport) : void 0,
670
+ remove: (transport) => internalLogger.remove ? internalLogger.remove(transport) : void 0,
671
+ get transports() {
672
+ return internalLogger.transports || [];
673
+ },
674
+ get level() {
675
+ return internalLogger.transports?.[0]?.level || "info";
676
+ },
677
+ set level(newLevel) {
678
+ if (internalLogger.transports?.[0]) internalLogger.transports[0].level = newLevel;
679
+ }
680
+ };
681
+ /**
682
+ * Logs request/response details in a formatted way
683
+ * @param url - Request URL
684
+ * @param requestBody - Request body object
685
+ * @param response - Response object (optional)
686
+ * @param error - Whether to log as error (true) or debug (false)
687
+ */
688
+ async function logRequestResponse(options) {
689
+ const { url, requestBody, requestMethod, response, error } = options;
690
+ const logMethod = error ? logger.error : logger.debug;
691
+ let responseText = "";
692
+ if (response) try {
693
+ responseText = await response.clone().text();
694
+ } catch {
695
+ responseText = "Unable to read response";
696
+ }
697
+ logMethod("Api Request", {
698
+ message: "API request",
699
+ url: sanitizeUrl(url),
700
+ method: requestMethod,
701
+ requestBody: sanitizeObject(requestBody, { context: "request body" }),
702
+ ...response && {
703
+ status: response.status,
704
+ statusText: response.statusText
705
+ },
706
+ ...responseText && { response: responseText }
707
+ });
708
+ }
709
+ if (getEnvString("LOG_LEVEL", "info") === "debug") initializeSourceMapSupport();
710
+ //#endregion
711
+ export { isCI as C, getMaxEvalTimeMs as S, getEnvBool as _, setLogCallback as a, getEnvString as b, sanitizeUrl as c, getAjv as d, isValidJson as f, getConfigDirectoryPath as g, summarizeEvaluateResultForLogging as h, logger as i, extractFirstJsonObject as l, safeJsonStringify as m, isDebugEnabled as n, setLogLevel as o, orderKeys as p, logRequestResponse as r, sanitizeObject as s, getLogLevel as t, extractJsonObjects as u, getEnvFloat as v, state as w, getEvalTimeoutMs as x, getEnvInt as y };
712
+
713
+ //# sourceMappingURL=logger-DLcq4dWf.js.map