promptfoo 0.120.26 → 0.121.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/drizzle/0023_wooden_mandrill.sql +2 -0
- package/dist/drizzle/meta/0023_snapshot.json +1496 -0
- package/dist/drizzle/meta/_journal.json +7 -0
- package/dist/src/{ListApp-D3DG0F8h.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-BgNJDBE6.js +206 -0
- package/dist/src/{accounts-BzEY8H3v.cjs → accounts-Bx-x3bmW.cjs} +99 -80
- package/dist/src/{accounts-DHHiXsy6.js → accounts-CMqkzrVf.js} +61 -36
- package/dist/src/{accounts-R3gfCR_g.js → accounts-xrUGFA6n.js} +60 -35
- package/dist/src/{agentic-utils-D6_gzOUF.js → agentic-utils-BKIN5PKu.js} +9 -10
- package/dist/src/{agents-CwM7re15.cjs → agents-B0f4HICh.cjs} +37 -40
- package/dist/src/{agents-Cnph5GLD.js → agents-C-dDThPK.js} +37 -37
- package/dist/src/{agents-C7BiinFI.cjs → agents-CErsqg5U.cjs} +19 -27
- package/dist/src/{agents-v4cW_ZgC.js → agents-CVIn-Utx.js} +19 -22
- package/dist/src/{agents-GiUianme.js → agents-CXknwsFX.js} +37 -40
- package/dist/src/{agents-DETIQHqF.js → agents-DeH4Gu94.js} +21 -28
- package/dist/src/{agents-DYIT-hQy.js → agents-Dy2YpZpa.js} +38 -41
- package/dist/src/{agents-Cao4i7AX.js → agents-aF4-T121.js} +19 -30
- package/dist/src/{aimlapi-DMF6v_vb.js → aimlapi-BAGZDo5G.js} +16 -18
- package/dist/src/{aimlapi-CMJpKK-B.js → aimlapi-BNfTBexL.js} +15 -17
- package/dist/src/{aimlapi-DtSf1ykJ.js → aimlapi-DHRKlBEA.js} +15 -4
- package/dist/src/{aimlapi-DoGLcQW_.cjs → aimlapi-tg0Gkcvr.cjs} +15 -16
- package/dist/src/app/assets/index-BFCZg7hQ.js +439 -0
- package/dist/src/app/assets/index-NCn4eVBv.css +1 -0
- package/dist/src/app/assets/{vendor-charts-CYyo8R8v.js → vendor-charts-CCl15Imd.js} +1 -1
- package/dist/src/app/assets/{vendor-markdown-DSmzq4Jh.js → vendor-markdown-0tekx3KX.js} +1 -1
- package/dist/src/app/index.html +4 -34
- package/dist/src/{audio-DUH4q0Xq.js → audio-BRODU0UK.js} +7 -9
- package/dist/src/{audio-BWjyvHn9.cjs → audio-BWeaWovU.cjs} +6 -7
- package/dist/src/{audio-U580w8jM.js → audio-CHQ4r-RV.js} +6 -5
- package/dist/src/{audio-BrJBFN2b.js → audio-tf_NBjlC.js} +6 -8
- package/dist/src/base-B0tcrnq_.js +193 -0
- package/dist/src/base-B4QJRyFS.js +194 -0
- package/dist/src/base-DBtwl2FR.cjs +222 -0
- package/dist/src/base-fEDN28WM.js +193 -0
- package/dist/src/{blobs-kt8v3UyH.js → blobs-BAU-dXan.js} +9 -12
- package/dist/src/{blobs-C9J2mVgo.js → blobs-Bpg5rH6i.js} +9 -12
- package/dist/src/{blobs-673H0jCl.cjs → blobs-DvS-O6be.cjs} +34 -37
- package/dist/src/blobs-qTYm-1PY.js +236 -0
- package/dist/src/{cache-BLLayYEN.js → cache-8XhNqPKW.js} +64 -67
- package/dist/src/cache-Bbn1Nyrd.cjs +5 -0
- package/dist/src/cache-BwsMSda7.js +6 -0
- package/dist/src/{cache-mIszOnuz.js → cache-CG0SlR1d.js} +64 -66
- package/dist/src/{cache-7xULbvt3.cjs → cache-COish3-W.cjs} +114 -117
- package/dist/src/cache-D3eqDYGU.js +739 -0
- package/dist/src/{chat-Fl6TZJRS.cjs → chat-2K608PeQ.cjs} +20 -21
- package/dist/src/chat-BKm79wib.js +764 -0
- package/dist/src/{chat-XPN9YHhr.js → chat-CM_kyI8B.js} +20 -9
- package/dist/src/{chat-DIywASPG.js → chat-CRWNNq73.js} +49 -49
- package/dist/src/{chat-C8Ei4f87.js → chat-CznLWr_D.js} +49 -49
- package/dist/src/{chat-CgyGj2hC.js → chat-DHMH-N64.js} +20 -22
- package/dist/src/{chat-C4zqjObh.cjs → chat-DaqekjFr.cjs} +69 -69
- package/dist/src/{chat-Cpz3O-Xl.js → chat-DxysjBvt.js} +21 -23
- package/dist/src/{chatkit-Dpxrq4eD.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DIrJX8xk.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-DEls11hE.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-e25Ziu17.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-6-xTaLwM.js → claude-agent-sdk-BLTu0WBO.js} +45 -31
- package/dist/src/{claude-agent-sdk-BzxF6NIJ.cjs → claude-agent-sdk-CJH22shf.cjs} +44 -29
- package/dist/src/{claude-agent-sdk-CmkTnKGH.js → claude-agent-sdk-D6_k9FKA.js} +45 -33
- package/dist/src/{claude-agent-sdk-rXCBLK_o.js → claude-agent-sdk-Dy5lT-Tx.js} +46 -21
- package/dist/src/{cloud-BMbRVJFw.js → cloud-Bc9526yV.js} +32 -12
- package/dist/src/cloud-DmE0EwsY.js +4 -0
- package/dist/src/{cloudflare-ai-CUg4BTcj.js → cloudflare-ai-C9r2sRhw.js} +16 -18
- package/dist/src/{cloudflare-ai-Z9X219gp.js → cloudflare-ai-CWWJCRim.js} +16 -4
- package/dist/src/{cloudflare-ai-BAQ0u_dg.cjs → cloudflare-ai-ClWSdor4.cjs} +16 -17
- package/dist/src/{cloudflare-ai-CobxMTR4.js → cloudflare-ai-ICsOuD-z.js} +17 -19
- package/dist/src/{cloudflare-gateway-C0sgfr_z.cjs → cloudflare-gateway-C2_-KG5o.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-_itGuXry.js → cloudflare-gateway-D6O7AlYb.js} +23 -23
- package/dist/src/{cloudflare-gateway-D2_yi-Fh.js → cloudflare-gateway-D6xFc5pa.js} +21 -25
- package/dist/src/{cloudflare-gateway-Djf3F3_H.js → cloudflare-gateway-pXGHxJ47.js} +26 -14
- package/dist/src/{codex-sdk-ibXwdglL.js → codex-sdk-C6UMlxwV.js} +49 -32
- package/dist/src/{codex-sdk-BASDNkIl.js → codex-sdk-DUwKWezN.js} +49 -30
- package/dist/src/{codex-sdk-dSnGdgIp.js → codex-sdk-GGAw0qbD.js} +49 -32
- package/dist/src/{codex-sdk-wTEpMM_X.cjs → codex-sdk-fAO0c3yA.cjs} +49 -32
- package/dist/src/{cometapi-B01btbfb.js → cometapi-BasUi7-_.js} +17 -19
- package/dist/src/{cometapi-DHUAH6nK.js → cometapi-Bbjp5V4x.js} +16 -4
- package/dist/src/{cometapi-ChAaRjg5.js → cometapi-DkXrKi5z.js} +21 -24
- package/dist/src/{cometapi-JbvOJSCO.cjs → cometapi-vY6aDZgo.cjs} +21 -22
- package/dist/src/{completion-D9_MDlnd.js → completion-6Mx_iXxK.js} +11 -13
- package/dist/src/{completion-BBJ6zmG3.js → completion-C5rtR_9P.js} +11 -13
- package/dist/src/{completion-DDyL3Cb2.cjs → completion-CDOouNzq.cjs} +21 -23
- package/dist/src/completion-C_P3ypkJ.js +120 -0
- package/dist/src/createHash-CTQmL3G2.js +15 -0
- package/dist/src/createHash-CfZSc0b4.cjs +27 -0
- package/dist/src/createHash-Da8fMwqB.js +16 -0
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-JAAubMw3.js → docker-5KcG-_86.js} +18 -20
- package/dist/src/{docker-Ckw-j7Rr.cjs → docker-BwsKwxFs.cjs} +18 -19
- package/dist/src/{docker-vnOg96gi.js → docker-CZnqU1XV.js} +18 -7
- package/dist/src/{docker-BuButc4D.js → docker-DzxyDPIj.js} +19 -21
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-CYhseqj4.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-rDtG_2rg.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-BQkx5roy.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-17JizQIv.js +15 -0
- package/dist/src/{eval-CYrbG57o.js → eval-DmFyWU7i.js} +49 -55
- package/dist/src/{evalResult-COsVttMA.js → evalResult-CDQiuUuf.js} +16 -12
- package/dist/src/{evalResult-6JaUIStC.js → evalResult-CTG2AHOS.js} +10 -11
- package/dist/src/evalResult-Cqj8pldJ.js +12 -0
- package/dist/src/{evalResult-DlRfu_Rq.cjs → evalResult-Dap2CekP.cjs} +20 -21
- package/dist/src/evalResult-DvcJAWJU.cjs +10 -0
- package/dist/src/evalResult-Hftn-S_i.js +10 -0
- package/dist/src/evaluator-B2CFNt-P.js +36 -0
- package/dist/src/{evaluator-3EJCMTs0.js → evaluator-DPFRbFIL.js} +210 -232
- package/dist/src/{extractor-LSYjrhK0.js → extractor-CFG6bcWJ.js} +23 -38
- package/dist/src/{extractor-DbhlYEeo.cjs → extractor-DX36oYEv.cjs} +37 -64
- package/dist/src/{extractor-Hs7la_19.js → extractor-M67RUtg6.js} +23 -38
- package/dist/src/extractor-YMU_Gvt8.js +374 -0
- package/dist/src/fetch-4M3YRaqL.js +780 -0
- package/dist/src/{fetch-18MuNu9i.js → fetch-60Gzydls.js} +60 -46
- package/dist/src/{fetch-SRsE6Ctl.js → fetch-BMv0O527.js} +41 -35
- package/dist/src/{fetch-ZMn_oemb.cjs → fetch-BxUk8odA.cjs} +268 -279
- package/dist/src/fetch-KV5kNASw.js +5 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-Doo0xq-z.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-CQlpZkrp.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/graders-Bu0H9nXi.js +32 -0
- package/dist/src/{graders-BaMCwIKp.js → graders-CHO8EPM4.js} +385 -417
- package/dist/src/graders-Cfhkvx-e.js +34 -0
- package/dist/src/{graders-QsALpIdy.js → graders-CpdqD9PI.js} +385 -417
- package/dist/src/graders-DClJVpGP.cjs +32 -0
- package/dist/src/{graders-DzUUnUjC.cjs → graders-DOXycdlG.cjs} +721 -753
- package/dist/src/graders-DcnJsrMO.js +32 -0
- package/dist/src/graders-R9rYUM0d.js +13466 -0
- package/dist/src/{image-BiEVdpdP.js → image-BmEZqVmk.js} +57 -18
- package/dist/src/{image-mhAGP07h.js → image-CBBVXWuT.js} +57 -18
- package/dist/src/{image-D10zEe1f.cjs → image-CDLQOcqT.cjs} +6 -7
- package/dist/src/{image-COCWy5dX.js → image-DJEvKveK.js} +6 -5
- package/dist/src/{image-C3BjJUAU.cjs → image-DTedmQPg.cjs} +77 -32
- package/dist/src/{image-DB4sHxdJ.js → image-gvmivTEe.js} +7 -9
- package/dist/src/image-pAX56tPG.js +257 -0
- package/dist/src/{image-BXt_7u0v.js → image-tL5hIOFh.js} +6 -8
- package/dist/src/index.cjs +696 -693
- package/dist/src/index.d.cts +113 -10
- package/dist/src/index.d.ts +113 -6
- package/dist/src/index.js +657 -658
- package/dist/src/{interactiveCheck-DU-MAhp5.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-DotRBzUE.js → knowledgeBase-CLJybhnF.js} +19 -34
- package/dist/src/{knowledgeBase-XJQ0Qyez.js → knowledgeBase-CoU-UQBg.js} +17 -41
- package/dist/src/{knowledgeBase-CMvMlLZR.js → knowledgeBase-DjWPVqSb.js} +17 -43
- package/dist/src/{knowledgeBase-Bnb00xKs.cjs → knowledgeBase-wkxuRFhA.cjs} +17 -40
- package/dist/src/{litellm-CHrRmPAe.js → litellm-B9Hysuri.js} +16 -18
- package/dist/src/{litellm-CrLJrPIm.js → litellm-CTfa0hqi.js} +15 -17
- package/dist/src/{litellm-BrnZhMcL.cjs → litellm-NYpQ8RQu.cjs} +15 -16
- package/dist/src/{litellm-BECdjOTx.js → litellm-ePxtr9F1.js} +15 -4
- package/dist/src/{logger-w8Ozp0Td.js → logger-CT3IKMKA.js} +24 -41
- package/dist/src/{logger-BdZ-IqBc.cjs → logger-Cp1GPUjj.cjs} +166 -192
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-BotXmWKW.js → logger-KkObSCzq.js} +27 -43
- package/dist/src/{luma-ray-C0RkI3lt.cjs → luma-ray-B0GGNRc1.cjs} +20 -21
- package/dist/src/{luma-ray-C-w6EsJm.js → luma-ray-BE2mOt6N.js} +20 -13
- package/dist/src/{luma-ray-BOeX-h0M.js → luma-ray-BW9IRGIc.js} +22 -21
- package/dist/src/{luma-ray-DgKLS0BF.js → luma-ray-Cm1KZBhs.js} +20 -23
- package/dist/src/main.js +1985 -2055
- package/dist/src/{messages-DXV3Qh8_.cjs → messages-1JrJs91T.cjs} +35 -34
- package/dist/src/{messages-D61tPFQo.js → messages-1x9atZmP.js} +25 -24
- package/dist/src/{messages-CDZYGNlS.js → messages-BLbWdsyt.js} +25 -24
- package/dist/src/messages-D8EA0oDc.js +240 -0
- package/dist/src/{meteor-P2rUE-Uz.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-SLNTgmXm.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-odmwVbyG.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-Dj8cTkU_.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/modelslab-C1OLRmVX.js +166 -0
- package/dist/src/modelslab-CqXBy3U8.js +168 -0
- package/dist/src/modelslab-DcOSFwKh.cjs +166 -0
- package/dist/src/modelslab-X5-4LroM.js +166 -0
- package/dist/src/{nova-reel-C2LFfVTf.js → nova-reel-BgS1ZWuK.js} +20 -13
- package/dist/src/{nova-reel-DtCjbD5O.js → nova-reel-D2ZkOSyr.js} +22 -21
- package/dist/src/{nova-reel-D9FXq3Mt.cjs → nova-reel-D9xfaMBs.cjs} +20 -21
- package/dist/src/{nova-reel-Bk5npr2q.js → nova-reel-DihqLeol.js} +20 -23
- package/dist/src/{nova-sonic-BoRSY_U6.cjs → nova-sonic-DVu3mMIy.cjs} +30 -31
- package/dist/src/{nova-sonic-D_qERM-K.js → nova-sonic-DezhVUYT.js} +30 -26
- package/dist/src/{nova-sonic-CgaWLDM1.js → nova-sonic-P-CdUMlV.js} +30 -31
- package/dist/src/{nova-sonic-BXRfQyF-.js → nova-sonic-Q3BOJeig.js} +31 -32
- package/dist/src/{openai-Bigwjgo1.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-Dz3surb_.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CT5fwbve.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-dHLcXUWZ.js → openclaw-BiSZPL7J.js} +20 -14
- package/dist/src/{openclaw-CpPrXwf6.js → openclaw-Bv1DINsX.js} +20 -27
- package/dist/src/{openclaw-B6XY2kUf.js → openclaw-D1D_ej1z.js} +21 -28
- package/dist/src/{openclaw-DDSfq5fp.cjs → openclaw-DAfWQn-o.cjs} +33 -39
- package/dist/src/opencode-sdk-C7m-wRfI.js +560 -0
- package/dist/src/opencode-sdk-CfaLN8PY.cjs +564 -0
- package/dist/src/opencode-sdk-D95s6SnR.js +562 -0
- package/dist/src/opencode-sdk-DxUPkLT7.js +560 -0
- package/dist/src/{otlpReceiver-DmRb0NBj.js → otlpReceiver--AIRW_S4.js} +53 -51
- package/dist/src/{otlpReceiver-Dg817agV.js → otlpReceiver-Bn5wGB1v.js} +53 -55
- package/dist/src/{otlpReceiver-B6Xo4KZM.cjs → otlpReceiver-Diec4cln.cjs} +53 -55
- package/dist/src/{otlpReceiver-BO0rbDzh.js → otlpReceiver-g3ByGaXs.js} +53 -55
- package/dist/src/{providerRegistry-Xf0qdqGQ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-wCWd7sKQ.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-lc7a7utN.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B3HvufyI.js +33246 -0
- package/dist/src/{providers-BiNq_Iyc.js → providers-BKRJTjBz.js} +1743 -1795
- package/dist/src/providers-C1rOSHiR.js +32 -0
- package/dist/src/{providers-BlEhY5mi.js → providers-CFLy1_ji.js} +1750 -1802
- package/dist/src/{providers-BNKVY53V.cjs → providers-CFu-TZl-.cjs} +2111 -2163
- package/dist/src/providers-CxmDwEFf.cjs +31 -0
- package/dist/src/providers-Dodakqr0.js +30 -0
- package/dist/src/providers-GIQ2TcsA.js +30 -0
- package/dist/src/{pythonUtils-r1uBuA0n.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DZ6EbdY4.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-vMlk9Qp5.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/quiverai-C2jVwbH1.js +213 -0
- package/dist/src/quiverai-CI6gYJVI.js +213 -0
- package/dist/src/quiverai-CLkWkyZc.cjs +213 -0
- package/dist/src/quiverai-MHSxbmmZ.js +215 -0
- package/dist/src/{render-CAZvKKkB.js → render-Drod8m7K.js} +4 -5
- package/dist/src/{responses-DLLjADw5.js → responses-BKqJmhhc.js} +34 -27
- package/dist/src/{responses-TsdODUpm.js → responses-CGw0DCzh.js} +34 -27
- package/dist/src/responses-jxdehPkC.js +660 -0
- package/dist/src/{responses-zOtKtnY_.cjs → responses-tD4Bd4dc.cjs} +49 -42
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-Cs35SDYa.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-BtjlqyXt.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-DCVaJ3mc.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-Du4LIR97.js → sagemaker-BK4Zb993.js} +75 -70
- package/dist/src/{sagemaker-CLdUAv5z.js → sagemaker-BfiWTmvn.js} +77 -77
- package/dist/src/{sagemaker-DwNnEVYt.cjs → sagemaker-CcQHM1jV.cjs} +75 -76
- package/dist/src/{sagemaker-BcgLu0U4.js → sagemaker-D2Q1c-sD.js} +75 -79
- package/dist/src/{scanner-Dyw21Wg_.js → scanner-J8CA3LsV.js} +149 -122
- package/dist/src/server/index.js +5620 -67302
- package/dist/src/{server-CgUQ25qW.cjs → server-B0PPuDw-.cjs} +57 -67
- package/dist/src/server-B1vi21hA.js +7 -0
- package/dist/src/{server-CbMTRQkg.js → server-BC7XJFgr.js} +19 -24
- package/dist/src/server-Cm9Kai_h.cjs +5 -0
- package/dist/src/{server-DWmZLfCy.js → server-DbFphssR.js} +26 -29
- package/dist/src/server-OAs3nBRT.js +229 -0
- package/dist/src/{signal-Bl32q42d.js → signal-BOTbd53Z.js} +9 -11
- package/dist/src/{slack-BtMkB6xP.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-OZYxoVON.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-DPqj42Ts.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-BfdBx2tO.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-BNmZ1KAz.cjs +5 -0
- package/dist/src/{store-BqwfFEyF.cjs → store-BSc-TF2w.cjs} +44 -45
- package/dist/src/store-BltJg2cd.js +6 -0
- package/dist/src/{store-D4gdn9ih.js → store-D1tv90v3.js} +34 -35
- package/dist/src/{store-2ocbYY9D.js → store-DQLEjuEO.js} +40 -36
- package/dist/src/store-Ub2vaGJ1.js +228 -0
- package/dist/src/{tables-D-NSwNIb.js → tables-5EvT_Bwn.js} +23 -23
- package/dist/src/{tables-B9E1kRp-.cjs → tables-C7K-XKWp.cjs} +93 -93
- package/dist/src/{tables-C7TT2XVn.js → tables-D36WTqKX.js} +25 -25
- package/dist/src/tables-xKANLRBD.js +288 -0
- package/dist/src/telemetry-5BCRNBbe.cjs +5 -0
- package/dist/src/{telemetry-DZ_7PaVq.js → telemetry-C15ziL8u.js} +17 -14
- package/dist/src/{telemetry-BXyVqyAg.js → telemetry-C2YDkUQH.js} +11 -13
- package/dist/src/{telemetry-D0_yFdtU.cjs → telemetry-CbrnxHp_.cjs} +21 -24
- package/dist/src/telemetry-D4W5hboe.js +7 -0
- package/dist/src/telemetry-DMb2Mpfm.js +171 -0
- package/dist/src/{text-Dm78AVGG.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-DF2hMKdg.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-DgMr_tiM.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-FNIz3YOe.cjs → transcription-CL78qbOU.cjs} +14 -15
- package/dist/src/{transcription-C-M81iDA.js → transcription-DAtxHhAM.js} +14 -7
- package/dist/src/{transcription-CYuY5sFO.js → transcription-LNZTNUUL.js} +14 -16
- package/dist/src/{transcription-Ch7S-LWw.js → transcription-QHh3AH6Z.js} +15 -17
- package/dist/src/{transform-CoP2bJ7P.js → transform-Cgi24fJ7.js} +94 -66
- package/dist/src/{transform-Kd6u-oNm.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-D8dILpfZ.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/{transform-DMaxQwDx.js → transform-DGLazrMm.js} +94 -66
- package/dist/src/transform-DGxXocjk.js +1506 -0
- package/dist/src/{transform-ivxEY4f7.cjs → transform-DOcQeLld.cjs} +234 -206
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-CqTFr7KR.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/{transformersAvailability-DEU2naS1.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-Bkep3ka7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-DwmezkVe.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-t52w-XsS.js → types-CH3Ge2sE.js} +103 -92
- package/dist/src/{types-DMVjYLpx.js → types-CLKiCBW3.js} +98 -91
- package/dist/src/types-CN_TZ2GJ.js +3260 -0
- package/dist/src/{types-BIfttHrT.cjs → types-LJ0r3wbR.cjs} +573 -566
- package/dist/src/util-5cB-L7U3.js +1430 -0
- package/dist/src/util-6-GqIvzS.js +599 -0
- package/dist/src/{util-vjscpUzy.js → util-B7T3SiBS.js} +5 -6
- package/dist/src/{util-Cl0zfT3V.js → util-Betm42rL.js} +44 -17
- package/dist/src/{util-CUEt0Vum.js → util-C-PPYSMq.js} +44 -17
- package/dist/src/{util-DkFTvieG.cjs → util-CchiqXh_.cjs} +35 -36
- package/dist/src/{util-mJ58qbbw.js → util-DaWTWKBK.js} +5 -6
- package/dist/src/{util-C08Kns6-.cjs → util-Db0a0AFH.cjs} +89 -62
- package/dist/src/{util-DiCePfDu.js → util-Dlz_Wvgm.js} +102 -53
- package/dist/src/{util-BSh4a_Q8.js → util-YT5HPZaS.js} +102 -53
- package/dist/src/{util-DUYOvxAy.cjs → util-Yz-1aEhW.cjs} +274 -219
- package/dist/src/util-ZZH-3QZz.js +293 -0
- package/dist/src/{utils-DFaZa6Rf.cjs → utils-Cz9qXqII.cjs} +32 -35
- package/dist/src/{utils-CVzb4YiI.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-JaY9veb5.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +59 -53
- package/dist/src/app/assets/index-BOgkICuY.css +0 -1
- package/dist/src/app/assets/index-CSgqn_Vd.js +0 -428
- package/dist/src/app/tsconfig.app.tsbuildinfo +0 -1
- package/dist/src/base-BaXmtXYp.js +0 -107
- package/dist/src/base-Dtp8b4_N.js +0 -106
- package/dist/src/base-f71xxWai.cjs +0 -111
- package/dist/src/cache-BUPcq0Ad.js +0 -6
- package/dist/src/cache-CVfRb-HD.cjs +0 -6
- package/dist/src/cache-O4EuX2JV.js +0 -8
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-CZ4hytdm.js +0 -5
- package/dist/src/eval-CKHWqG9f.js +0 -16
- package/dist/src/evalResult-CxTP-LMm.cjs +0 -11
- package/dist/src/evalResult-CzLURDcP.js +0 -13
- package/dist/src/evalResult-DyttNQ_G.js +0 -11
- package/dist/src/evaluator-0PvfeBYh.js +0 -38
- package/dist/src/fetch-Bi0o-fdp.js +0 -4
- package/dist/src/fetch-CMptBDVg.cjs +0 -4
- package/dist/src/fetch-DAZkv3gV.js +0 -6
- package/dist/src/graders-BCytzXrb.js +0 -34
- package/dist/src/graders-CGZQShfJ.cjs +0 -33
- package/dist/src/graders-spkuVC-E.js +0 -36
- package/dist/src/opencode-sdk-CImWVqy9.js +0 -382
- package/dist/src/opencode-sdk-CuCztr4P.js +0 -380
- package/dist/src/opencode-sdk-DhcfRbBH.js +0 -376
- package/dist/src/opencode-sdk-mqF-Oj3f.cjs +0 -383
- package/dist/src/providers-BMZZmPBJ.cjs +0 -32
- package/dist/src/providers-CQQrNaJk.js +0 -32
- package/dist/src/providers-Ck8HyrC-.js +0 -34
- package/dist/src/quiverai-BNfIwKCO.cjs +0 -54
- package/dist/src/quiverai-BQigKdIH.js +0 -57
- package/dist/src/quiverai-Bfy2WnE2.js +0 -55
- package/dist/src/quiverai-CedIP0PJ.js +0 -43
- package/dist/src/rubyUtils-D7--T12C.js +0 -6
- package/dist/src/rubyUtils-DRRiMFV2.js +0 -5
- package/dist/src/rubyUtils-vb8OYFC-.cjs +0 -5
- package/dist/src/server-BUbS0Qfh.js +0 -6
- package/dist/src/server-XpGXFHkS.cjs +0 -6
- package/dist/src/server-gfOx5Zrk.js +0 -8
- package/dist/src/store-5u2yriTV.js +0 -7
- package/dist/src/store-D_lq_8oQ.js +0 -6
- package/dist/src/store-m5KT6Ly7.cjs +0 -6
- package/dist/src/telemetry-5RHFoCJh.js +0 -6
- package/dist/src/telemetry-Do8wMnA-.js +0 -8
- package/dist/src/telemetry-LojxPoFq.cjs +0 -6
- package/dist/src/transform-8eGmaH-7.js +0 -7
- package/dist/src/transform-BRVvWaG4.cjs +0 -6
- package/dist/src/transform-GybT0X0u.js +0 -8
- package/dist/src/transformersAvailability-DkAWaK5B.js +0 -35
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-
|
|
1
|
+
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DnqA1y9x.js";
|
|
2
2
|
import dedent from "dedent";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
-
|
|
5
4
|
//#region src/types/env.ts
|
|
6
5
|
const ProviderEnvOverridesSchema = z.object({
|
|
7
6
|
AI21_API_BASE_URL: z.string().optional(),
|
|
@@ -65,6 +64,7 @@ const ProviderEnvOverridesSchema = z.object({
|
|
|
65
64
|
MISTRAL_API_BASE_URL: z.string().optional(),
|
|
66
65
|
MISTRAL_API_HOST: z.string().optional(),
|
|
67
66
|
MISTRAL_API_KEY: z.string().optional(),
|
|
67
|
+
MODELSLAB_API_KEY: z.string().optional(),
|
|
68
68
|
NSCALE_SERVICE_TOKEN: z.string().optional(),
|
|
69
69
|
NSCALE_API_KEY: z.string().optional(),
|
|
70
70
|
OLLAMA_API_KEY: z.string().optional(),
|
|
@@ -121,7 +121,6 @@ const ProviderEnvOverridesSchema = z.object({
|
|
|
121
121
|
AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
|
|
122
122
|
PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
|
|
123
123
|
});
|
|
124
|
-
|
|
125
124
|
//#endregion
|
|
126
125
|
//#region src/types/shared.ts
|
|
127
126
|
const CompletionTokenDetailsSchema = z.object({
|
|
@@ -149,7 +148,6 @@ const BaseTokenUsageSchema = z.object({
|
|
|
149
148
|
}).optional()
|
|
150
149
|
});
|
|
151
150
|
const InputsSchema = z.record(z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" }), z.string().min(1, { error: "Input descriptions must be non-empty strings" }));
|
|
152
|
-
|
|
153
151
|
//#endregion
|
|
154
152
|
//#region src/validators/prompts.ts
|
|
155
153
|
const PromptConfigSchema = z.object({
|
|
@@ -169,10 +167,8 @@ function assert$1() {}
|
|
|
169
167
|
assert$1();
|
|
170
168
|
assert$1();
|
|
171
169
|
assert$1();
|
|
172
|
-
|
|
173
170
|
//#endregion
|
|
174
171
|
//#region src/redteam/constants/plugins.ts
|
|
175
|
-
const DEFAULT_NUM_TESTS_PER_PLUGIN = 5;
|
|
176
172
|
const MULTI_INPUT_VAR = "__prompt";
|
|
177
173
|
const LLAMA_GUARD_REPLICATE_PROVIDER = "replicate:moderation:meta/llama-guard-4-12b";
|
|
178
174
|
const LLAMA_GUARD_ENABLED_CATEGORIES = [
|
|
@@ -369,6 +365,7 @@ const PHARMACY_PLUGINS = [
|
|
|
369
365
|
];
|
|
370
366
|
const INSURANCE_PLUGINS = [
|
|
371
367
|
"insurance:coverage-discrimination",
|
|
368
|
+
"insurance:data-disclosure",
|
|
372
369
|
"insurance:network-misinformation",
|
|
373
370
|
"insurance:phi-disclosure"
|
|
374
371
|
];
|
|
@@ -454,6 +451,7 @@ const ADDITIONAL_PLUGINS = [
|
|
|
454
451
|
"ecommerce:price-manipulation",
|
|
455
452
|
"goal-misalignment",
|
|
456
453
|
"insurance:coverage-discrimination",
|
|
454
|
+
"insurance:data-disclosure",
|
|
457
455
|
"insurance:network-misinformation",
|
|
458
456
|
"insurance:phi-disclosure",
|
|
459
457
|
"off-topic",
|
|
@@ -531,7 +529,7 @@ const DEFAULT_PLUGINS = new Set([...[
|
|
|
531
529
|
...PII_PLUGINS,
|
|
532
530
|
...BIAS_PLUGINS
|
|
533
531
|
].sort()]);
|
|
534
|
-
|
|
532
|
+
new Set([
|
|
535
533
|
...DEFAULT_PLUGINS,
|
|
536
534
|
"bola",
|
|
537
535
|
"bfla",
|
|
@@ -591,12 +589,11 @@ const REMOTE_ONLY_PLUGIN_IDS = [
|
|
|
591
589
|
...TELECOM_PLUGINS,
|
|
592
590
|
...REALESTATE_PLUGINS
|
|
593
591
|
];
|
|
594
|
-
|
|
592
|
+
[
|
|
595
593
|
...Object.keys(UNALIGNED_PROVIDER_HARM_PLUGINS),
|
|
596
594
|
...BIAS_PLUGINS,
|
|
597
595
|
...REMOTE_ONLY_PLUGIN_IDS
|
|
598
596
|
];
|
|
599
|
-
|
|
600
597
|
//#endregion
|
|
601
598
|
//#region src/redteam/constants/frameworks.ts
|
|
602
599
|
const OWASP_LLM_TOP_10_MAPPING = {
|
|
@@ -1435,7 +1432,73 @@ const GDPR_MAPPING = {
|
|
|
1435
1432
|
strategies: []
|
|
1436
1433
|
}
|
|
1437
1434
|
};
|
|
1435
|
+
/**
|
|
1436
|
+
* U.S. Department of Defense (DoD) AI ethical principles.
|
|
1437
|
+
*
|
|
1438
|
+
* Source:
|
|
1439
|
+
* * https://www.defense.gov/News/News-Stories/Article/Article/2094085/dod-adopts-5-principles-of-artificial-intelligence-ethics/
|
|
1440
|
+
*/
|
|
1441
|
+
const DOD_AI_ETHICS_MAPPING = {
|
|
1442
|
+
"dod:ai:ethics:01": {
|
|
1443
|
+
plugins: [
|
|
1444
|
+
"excessive-agency",
|
|
1445
|
+
"goal-misalignment",
|
|
1446
|
+
"overreliance",
|
|
1447
|
+
"hijacking"
|
|
1448
|
+
],
|
|
1449
|
+
strategies: ["jailbreak", "jailbreak-templates"]
|
|
1450
|
+
},
|
|
1451
|
+
"dod:ai:ethics:02": {
|
|
1452
|
+
plugins: [
|
|
1453
|
+
"bias:age",
|
|
1454
|
+
"bias:disability",
|
|
1455
|
+
"bias:gender",
|
|
1456
|
+
"bias:race",
|
|
1457
|
+
"harmful:hate"
|
|
1458
|
+
],
|
|
1459
|
+
strategies: []
|
|
1460
|
+
},
|
|
1461
|
+
"dod:ai:ethics:03": {
|
|
1462
|
+
plugins: [
|
|
1463
|
+
"hallucination",
|
|
1464
|
+
"harmful:misinformation-disinformation",
|
|
1465
|
+
"rag-source-attribution",
|
|
1466
|
+
"unverifiable-claims"
|
|
1467
|
+
],
|
|
1468
|
+
strategies: []
|
|
1469
|
+
},
|
|
1470
|
+
"dod:ai:ethics:04": {
|
|
1471
|
+
plugins: [
|
|
1472
|
+
"harmful:misinformation-disinformation",
|
|
1473
|
+
"harmful:unsafe-practices",
|
|
1474
|
+
"shell-injection",
|
|
1475
|
+
"sql-injection",
|
|
1476
|
+
"ssrf",
|
|
1477
|
+
"debug-access",
|
|
1478
|
+
"reasoning-dos"
|
|
1479
|
+
],
|
|
1480
|
+
strategies: ["jailbreak", "jailbreak-templates"]
|
|
1481
|
+
},
|
|
1482
|
+
"dod:ai:ethics:05": {
|
|
1483
|
+
plugins: [
|
|
1484
|
+
"excessive-agency",
|
|
1485
|
+
"hijacking",
|
|
1486
|
+
"indirect-prompt-injection",
|
|
1487
|
+
"system-prompt-override",
|
|
1488
|
+
"rbac",
|
|
1489
|
+
"bfla",
|
|
1490
|
+
"bola",
|
|
1491
|
+
"tool-discovery"
|
|
1492
|
+
],
|
|
1493
|
+
strategies: [
|
|
1494
|
+
"jailbreak",
|
|
1495
|
+
"jailbreak-templates",
|
|
1496
|
+
"jailbreak:composite"
|
|
1497
|
+
]
|
|
1498
|
+
}
|
|
1499
|
+
};
|
|
1438
1500
|
const ALIASED_PLUGINS = [
|
|
1501
|
+
"dod:ai:ethics",
|
|
1439
1502
|
"mitre:atlas",
|
|
1440
1503
|
"nist:ai",
|
|
1441
1504
|
"nist:ai:measure",
|
|
@@ -1462,9 +1525,11 @@ const ALIASED_PLUGINS = [
|
|
|
1462
1525
|
...Object.keys(OWASP_AGENTIC_TOP_10_MAPPING),
|
|
1463
1526
|
...Object.keys(EU_AI_ACT_MAPPING),
|
|
1464
1527
|
...Object.keys(ISO_42001_MAPPING),
|
|
1465
|
-
...Object.keys(GDPR_MAPPING)
|
|
1528
|
+
...Object.keys(GDPR_MAPPING),
|
|
1529
|
+
...Object.keys(DOD_AI_ETHICS_MAPPING)
|
|
1466
1530
|
];
|
|
1467
1531
|
const ALIASED_PLUGIN_MAPPINGS = {
|
|
1532
|
+
"dod:ai:ethics": DOD_AI_ETHICS_MAPPING,
|
|
1468
1533
|
"mitre:atlas": MITRE_ATLAS_MAPPING,
|
|
1469
1534
|
"nist:ai:measure": NIST_AI_RMF_MAPPING,
|
|
1470
1535
|
"owasp:api": OWASP_API_TOP_10_MAPPING,
|
|
@@ -1525,7 +1590,6 @@ const ALIASED_PLUGIN_MAPPINGS = {
|
|
|
1525
1590
|
strategies: []
|
|
1526
1591
|
} }
|
|
1527
1592
|
};
|
|
1528
|
-
|
|
1529
1593
|
//#endregion
|
|
1530
1594
|
//#region src/redteam/constants/metadata.ts
|
|
1531
1595
|
const Severity = {
|
|
@@ -1542,20 +1606,8 @@ const SeveritySchema = z.enum([
|
|
|
1542
1606
|
"low",
|
|
1543
1607
|
"informational"
|
|
1544
1608
|
]);
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
[Severity.High]: "High",
|
|
1548
|
-
[Severity.Medium]: "Medium",
|
|
1549
|
-
[Severity.Low]: "Low",
|
|
1550
|
-
[Severity.Informational]: "Informational"
|
|
1551
|
-
};
|
|
1552
|
-
const severityRiskScores = {
|
|
1553
|
-
[Severity.Critical]: 9,
|
|
1554
|
-
[Severity.High]: 7,
|
|
1555
|
-
[Severity.Medium]: 4,
|
|
1556
|
-
[Severity.Low]: 0,
|
|
1557
|
-
[Severity.Informational]: 0
|
|
1558
|
-
};
|
|
1609
|
+
Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
|
|
1610
|
+
Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
|
|
1559
1611
|
const riskCategorySeverityMap = {
|
|
1560
1612
|
["agentic:memory-poisoning"]: Severity.High,
|
|
1561
1613
|
aegis: Severity.Medium,
|
|
@@ -1644,6 +1696,7 @@ const riskCategorySeverityMap = {
|
|
|
1644
1696
|
imitation: Severity.Low,
|
|
1645
1697
|
"indirect-prompt-injection": Severity.High,
|
|
1646
1698
|
"insurance:coverage-discrimination": Severity.Critical,
|
|
1699
|
+
"insurance:data-disclosure": Severity.Critical,
|
|
1647
1700
|
"insurance:network-misinformation": Severity.High,
|
|
1648
1701
|
"insurance:phi-disclosure": Severity.Critical,
|
|
1649
1702
|
"ecommerce:pci-dss": Severity.Critical,
|
|
@@ -1704,7 +1757,7 @@ const riskCategorySeverityMap = {
|
|
|
1704
1757
|
wordplay: Severity.Low,
|
|
1705
1758
|
xstest: Severity.Low
|
|
1706
1759
|
};
|
|
1707
|
-
|
|
1760
|
+
Object.entries({
|
|
1708
1761
|
"Security & Access Control": [
|
|
1709
1762
|
"ascii-smuggling",
|
|
1710
1763
|
"bfla",
|
|
@@ -1727,6 +1780,7 @@ const riskCategories = {
|
|
|
1727
1780
|
"data-exfil",
|
|
1728
1781
|
"divergent-repetition",
|
|
1729
1782
|
"harmful:privacy",
|
|
1783
|
+
"insurance:data-disclosure",
|
|
1730
1784
|
"insurance:phi-disclosure",
|
|
1731
1785
|
"pii:api-db",
|
|
1732
1786
|
"pii:direct",
|
|
@@ -1852,14 +1906,12 @@ const riskCategories = {
|
|
|
1852
1906
|
"vlsu",
|
|
1853
1907
|
"xstest"
|
|
1854
1908
|
]
|
|
1855
|
-
}
|
|
1856
|
-
const categoryMapReverse = Object.entries(riskCategories).reduce((acc, [category, harms]) => {
|
|
1909
|
+
}).reduce((acc, [category, harms]) => {
|
|
1857
1910
|
harms.forEach((harm) => {
|
|
1858
1911
|
acc[harm] = category;
|
|
1859
1912
|
});
|
|
1860
1913
|
return acc;
|
|
1861
1914
|
}, {});
|
|
1862
|
-
const categoryLabels = Object.keys(categoryMapReverse);
|
|
1863
1915
|
const categoryAliases = {
|
|
1864
1916
|
["agentic:memory-poisoning"]: "AgenticMemoryPoisoning",
|
|
1865
1917
|
aegis: "Aegis",
|
|
@@ -1977,6 +2029,7 @@ const categoryAliases = {
|
|
|
1977
2029
|
imitation: "Imitation",
|
|
1978
2030
|
"indirect-prompt-injection": "Indirect Prompt Injection",
|
|
1979
2031
|
"insurance:coverage-discrimination": "InsuranceCoverageDiscrimination",
|
|
2032
|
+
"insurance:data-disclosure": "InsuranceDataDisclosure",
|
|
1980
2033
|
"insurance:network-misinformation": "InsuranceNetworkMisinformation",
|
|
1981
2034
|
"insurance:phi-disclosure": "InsurancePhiDisclosure",
|
|
1982
2035
|
intent: "Intent",
|
|
@@ -2008,7 +2061,7 @@ const categoryAliases = {
|
|
|
2008
2061
|
wordplay: "Wordplay",
|
|
2009
2062
|
xstest: "XSTest"
|
|
2010
2063
|
};
|
|
2011
|
-
|
|
2064
|
+
Object.entries(categoryAliases).reduce((acc, [key, value]) => {
|
|
2012
2065
|
acc[value] = key;
|
|
2013
2066
|
return acc;
|
|
2014
2067
|
}, {});
|
|
@@ -2046,7 +2099,7 @@ const pluginDescriptions = {
|
|
|
2046
2099
|
bias: "Unified bias detection across protected characteristics including age, disability, gender, and race",
|
|
2047
2100
|
medical: "Comprehensive medical AI safety testing for healthcare-specific vulnerabilities including hallucination, anchoring bias, incorrect knowledge, off-label use, prioritization errors, and sycophancy",
|
|
2048
2101
|
pharmacy: "Comprehensive pharmacy AI safety testing for pharmaceutical vulnerabilities including drug interactions, dosage calculations, and controlled substance compliance",
|
|
2049
|
-
insurance: "Comprehensive insurance AI safety testing
|
|
2102
|
+
insurance: "Comprehensive insurance AI safety testing across all insurance types including coverage discrimination, network misinformation, PHI disclosure, and policyholder data disclosure",
|
|
2050
2103
|
financial: "Comprehensive financial AI safety testing for financial services vulnerabilities including calculation errors, compliance violations, hallucination, and data leakage",
|
|
2051
2104
|
ecommerce: "Comprehensive e-commerce AI safety testing for payment security and transaction integrity vulnerabilities including PCI DSS compliance, fraud detection, and price manipulation",
|
|
2052
2105
|
telecom: "Comprehensive telecommunications AI safety testing across CPNI protection, account security, regulatory compliance (FCC, TCPA, CALEA), and telecom-specific fraud prevention",
|
|
@@ -2099,8 +2152,9 @@ const pluginDescriptions = {
|
|
|
2099
2152
|
"harmful:weapons:ied": "Tests handling of Improvised Explosive Devices",
|
|
2100
2153
|
hijacking: "Assesses protection against unauthorized resource usage and purpose deviation",
|
|
2101
2154
|
imitation: "Tests safeguards against unauthorized entity impersonation attempts",
|
|
2102
|
-
"insurance:coverage-discrimination": "Tests for discriminatory coverage or
|
|
2103
|
-
"insurance:
|
|
2155
|
+
"insurance:coverage-discrimination": "Tests for discriminatory coverage, underwriting, or claims determinations based on protected characteristics (age, disability, race, genetic information, sex) in violation of federal civil rights laws including ADA, Section 1557, GINA, Fair Housing Act, ECOA, and state unfair trade practices acts",
|
|
2156
|
+
"insurance:data-disclosure": "Tests for unauthorized disclosure of protected policyholder data including claims history, property details, driving records, beneficiary information, and business data in violation of GLBA, FCRA, DPPA, and state insurance privacy laws",
|
|
2157
|
+
"insurance:network-misinformation": "Tests for inaccurate provider or vendor network information including wrong network status, terminated contracts, ghost networks, and outdated provider data that could expose policyholders to unexpected costs, voided warranties, or inadequate service",
|
|
2104
2158
|
"insurance:phi-disclosure": "Tests for unauthorized disclosure of Protected Health Information in violation of HIPAA privacy regulations including cross-member PHI leakage, inadequate authentication, and disclosure to unauthorized third parties",
|
|
2105
2159
|
"ecommerce:pci-dss": "Tests for exposure of payment card data including credit card numbers, CVVs, expiration dates, cardholder names, bank account details, and payment tokens in violation of PCI DSS requirements for secure payment processing",
|
|
2106
2160
|
"ecommerce:compliance-bypass": "Tests for attempts to bypass security rules, terms of service, age restrictions, geographic limitations, or compliance policies in e-commerce systems",
|
|
@@ -2160,7 +2214,6 @@ const pluginDescriptions = {
|
|
|
2160
2214
|
xstest: "Tests how models handle ambiguous terms related to potentially harmful topics like violence and drugs",
|
|
2161
2215
|
"guardrails-eval": "Evaluate guardrail effectiveness against common risks"
|
|
2162
2216
|
};
|
|
2163
|
-
|
|
2164
2217
|
//#endregion
|
|
2165
2218
|
//#region src/redteam/constants/strategies.ts
|
|
2166
2219
|
const FRAMEWORK_COMPLIANCE_IDS = [
|
|
@@ -2171,14 +2224,14 @@ const FRAMEWORK_COMPLIANCE_IDS = [
|
|
|
2171
2224
|
"owasp:agentic",
|
|
2172
2225
|
"eu:ai-act",
|
|
2173
2226
|
"iso:42001",
|
|
2174
|
-
"gdpr"
|
|
2227
|
+
"gdpr",
|
|
2228
|
+
"dod:ai:ethics"
|
|
2175
2229
|
];
|
|
2176
2230
|
const DEFAULT_STRATEGIES = [
|
|
2177
2231
|
"basic",
|
|
2178
2232
|
"jailbreak:meta",
|
|
2179
2233
|
"jailbreak:composite"
|
|
2180
2234
|
];
|
|
2181
|
-
const DEFAULT_STRATEGIES_SET = new Set(DEFAULT_STRATEGIES);
|
|
2182
2235
|
const MULTI_TURN_STRATEGIES = [
|
|
2183
2236
|
"crescendo",
|
|
2184
2237
|
"goat",
|
|
@@ -2186,16 +2239,9 @@ const MULTI_TURN_STRATEGIES = [
|
|
|
2186
2239
|
"custom",
|
|
2187
2240
|
"mischievous-user"
|
|
2188
2241
|
];
|
|
2189
|
-
const MULTI_TURN_STRATEGY_SET = new Set(MULTI_TURN_STRATEGIES);
|
|
2190
2242
|
const isCustomStrategy = (strategyId) => {
|
|
2191
2243
|
return strategyId === "custom" || strategyId.startsWith("custom:");
|
|
2192
2244
|
};
|
|
2193
|
-
const MULTI_MODAL_STRATEGIES = [
|
|
2194
|
-
"audio",
|
|
2195
|
-
"image",
|
|
2196
|
-
"video"
|
|
2197
|
-
];
|
|
2198
|
-
const MULTI_MODAL_STRATEGIES_SET = new Set(MULTI_MODAL_STRATEGIES);
|
|
2199
2245
|
const AGENTIC_STRATEGIES = [
|
|
2200
2246
|
"crescendo",
|
|
2201
2247
|
"goat",
|
|
@@ -2207,7 +2253,6 @@ const AGENTIC_STRATEGIES = [
|
|
|
2207
2253
|
"jailbreak:tree",
|
|
2208
2254
|
"mischievous-user"
|
|
2209
2255
|
];
|
|
2210
|
-
const AGENTIC_STRATEGIES_SET = new Set(AGENTIC_STRATEGIES);
|
|
2211
2256
|
const DATASET_PLUGINS = [
|
|
2212
2257
|
"beavertails",
|
|
2213
2258
|
"cyberseceval",
|
|
@@ -2269,22 +2314,6 @@ const _ALL_STRATEGIES = [
|
|
|
2269
2314
|
...AGENTIC_STRATEGIES
|
|
2270
2315
|
];
|
|
2271
2316
|
const ALL_STRATEGIES = Array.from(new Set(_ALL_STRATEGIES)).sort();
|
|
2272
|
-
const CONFIGURABLE_STRATEGIES = [
|
|
2273
|
-
"layer",
|
|
2274
|
-
"best-of-n",
|
|
2275
|
-
"goat",
|
|
2276
|
-
"crescendo",
|
|
2277
|
-
"indirect-web-pwn",
|
|
2278
|
-
"jailbreak",
|
|
2279
|
-
"jailbreak:hydra",
|
|
2280
|
-
"jailbreak:meta",
|
|
2281
|
-
"jailbreak:tree",
|
|
2282
|
-
"gcg",
|
|
2283
|
-
"citation",
|
|
2284
|
-
"custom",
|
|
2285
|
-
"mischievous-user"
|
|
2286
|
-
];
|
|
2287
|
-
const CONFIGURABLE_STRATEGIES_SET = new Set(CONFIGURABLE_STRATEGIES);
|
|
2288
2317
|
/**
|
|
2289
2318
|
* Default 'n' fan out for strategies that can add additional test cases during generation
|
|
2290
2319
|
*/
|
|
@@ -2299,19 +2328,6 @@ function getDefaultNFanout(strategyId) {
|
|
|
2299
2328
|
function isFanoutStrategy(strategyId) {
|
|
2300
2329
|
return strategyId in DEFAULT_N_FAN_OUT_BY_STRATEGY;
|
|
2301
2330
|
}
|
|
2302
|
-
const STRATEGIES_REQUIRING_REMOTE = [
|
|
2303
|
-
"audio",
|
|
2304
|
-
"citation",
|
|
2305
|
-
"gcg",
|
|
2306
|
-
"goat",
|
|
2307
|
-
"indirect-web-pwn",
|
|
2308
|
-
"jailbreak:composite",
|
|
2309
|
-
"jailbreak:hydra",
|
|
2310
|
-
"jailbreak:likert",
|
|
2311
|
-
"jailbreak:meta"
|
|
2312
|
-
];
|
|
2313
|
-
const STRATEGIES_REQUIRING_REMOTE_SET = new Set(STRATEGIES_REQUIRING_REMOTE);
|
|
2314
|
-
|
|
2315
2331
|
//#endregion
|
|
2316
2332
|
//#region src/util/uuid.ts
|
|
2317
2333
|
/**
|
|
@@ -2328,7 +2344,6 @@ const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-
|
|
|
2328
2344
|
function isUuid(value) {
|
|
2329
2345
|
return UUID_REGEX.test(value);
|
|
2330
2346
|
}
|
|
2331
|
-
|
|
2332
2347
|
//#endregion
|
|
2333
2348
|
//#region src/redteam/plugins/policy/validators.ts
|
|
2334
2349
|
/**
|
|
@@ -2364,7 +2379,6 @@ function isValidInlinePolicyId(id) {
|
|
|
2364
2379
|
function isValidPolicyId(id) {
|
|
2365
2380
|
return isValidReusablePolicyId(id) || isValidInlinePolicyId(id);
|
|
2366
2381
|
}
|
|
2367
|
-
|
|
2368
2382
|
//#endregion
|
|
2369
2383
|
//#region src/redteam/types.ts
|
|
2370
2384
|
const PolicyObjectSchema = z.object({
|
|
@@ -2399,6 +2413,7 @@ const PluginConfigSchema = z.object({
|
|
|
2399
2413
|
name: z.string().optional(),
|
|
2400
2414
|
multilingual: z.boolean().optional(),
|
|
2401
2415
|
indirectInjectionVar: z.string().optional(),
|
|
2416
|
+
intendedResults: z.array(z.string()).optional(),
|
|
2402
2417
|
intent: z.union([z.string(), z.array(z.union([z.string(), z.array(z.string())]))]).optional(),
|
|
2403
2418
|
policy: z.union([z.string(), PolicyObjectSchema]).optional(),
|
|
2404
2419
|
systemPrompt: z.string().optional(),
|
|
@@ -2430,7 +2445,6 @@ var PartialGenerationError = class extends Error {
|
|
|
2430
2445
|
this.failedPlugins = failedPlugins;
|
|
2431
2446
|
}
|
|
2432
2447
|
};
|
|
2433
|
-
|
|
2434
2448
|
//#endregion
|
|
2435
2449
|
//#region src/validators/providers.ts
|
|
2436
2450
|
const ProviderOptionsSchema = z.object({
|
|
@@ -2455,7 +2469,7 @@ const ApiProviderSchema = z.object({
|
|
|
2455
2469
|
config: z.any().optional(),
|
|
2456
2470
|
inputs: InputsSchema.optional()
|
|
2457
2471
|
});
|
|
2458
|
-
|
|
2472
|
+
z.object({
|
|
2459
2473
|
cached: z.boolean().optional(),
|
|
2460
2474
|
cost: z.number().optional(),
|
|
2461
2475
|
error: z.string().optional(),
|
|
@@ -2464,17 +2478,17 @@ const ProviderResponseSchema = z.object({
|
|
|
2464
2478
|
output: z.union([z.string(), z.any()]).optional(),
|
|
2465
2479
|
tokenUsage: BaseTokenUsageSchema.optional()
|
|
2466
2480
|
});
|
|
2467
|
-
|
|
2481
|
+
z.object({
|
|
2468
2482
|
error: z.string().optional(),
|
|
2469
2483
|
embedding: z.array(z.number()).optional(),
|
|
2470
2484
|
tokenUsage: BaseTokenUsageSchema.partial().optional()
|
|
2471
2485
|
});
|
|
2472
|
-
|
|
2486
|
+
z.object({
|
|
2473
2487
|
error: z.string().optional(),
|
|
2474
2488
|
similarity: z.number().optional(),
|
|
2475
2489
|
tokenUsage: BaseTokenUsageSchema.partial().optional()
|
|
2476
2490
|
});
|
|
2477
|
-
|
|
2491
|
+
z.object({
|
|
2478
2492
|
error: z.string().optional(),
|
|
2479
2493
|
classification: z.record(z.string(), z.number()).optional()
|
|
2480
2494
|
});
|
|
@@ -2493,7 +2507,6 @@ const ProviderSchema = z.union([
|
|
|
2493
2507
|
ApiProviderSchema,
|
|
2494
2508
|
ProviderOptionsSchema
|
|
2495
2509
|
]);
|
|
2496
|
-
|
|
2497
2510
|
//#endregion
|
|
2498
2511
|
//#region src/validators/redteam.ts
|
|
2499
2512
|
const TracingConfigSchema = z.lazy(() => z.object({
|
|
@@ -2538,7 +2551,7 @@ const RedteamPluginObjectSchema = z.object({
|
|
|
2538
2551
|
message: `Invalid plugin id "${val}". Custom plugins must start with file:// or use a built-in plugin. See https://www.promptfoo.dev/docs/red-team/plugins for available plugins.`
|
|
2539
2552
|
});
|
|
2540
2553
|
})]).describe("Name of the plugin"),
|
|
2541
|
-
numTests: z.int().positive().prefault(
|
|
2554
|
+
numTests: z.int().positive().prefault(5).describe("Number of tests to generate for this plugin"),
|
|
2542
2555
|
config: z.record(z.string(), z.unknown()).optional().describe("Plugin-specific configuration"),
|
|
2543
2556
|
severity: SeveritySchema.optional().describe("Severity level for this plugin")
|
|
2544
2557
|
});
|
|
@@ -2579,10 +2592,7 @@ const RedteamStrategySchema = z.union([strategyIdSchema, z.object({
|
|
|
2579
2592
|
id: strategyIdSchema,
|
|
2580
2593
|
config: z.record(z.string(), z.unknown()).optional().describe("Strategy-specific configuration")
|
|
2581
2594
|
})]);
|
|
2582
|
-
|
|
2583
|
-
* Schema for `promptfoo redteam generate` command options
|
|
2584
|
-
*/
|
|
2585
|
-
const RedteamGenerateOptionsSchema = z.object({
|
|
2595
|
+
z.object({
|
|
2586
2596
|
addPlugins: z.array(z.enum(ADDITIONAL_PLUGINS)).optional().describe("Additional plugins to include"),
|
|
2587
2597
|
addStrategies: z.array(z.enum(ADDITIONAL_STRATEGIES)).optional().describe("Additional strategies to include"),
|
|
2588
2598
|
cache: z.boolean().describe("Whether to use caching"),
|
|
@@ -2760,11 +2770,9 @@ const RedteamConfigSchema = z.object({
|
|
|
2760
2770
|
});
|
|
2761
2771
|
function assert() {}
|
|
2762
2772
|
assert();
|
|
2763
|
-
|
|
2764
2773
|
//#endregion
|
|
2765
2774
|
//#region src/validators/shared.ts
|
|
2766
2775
|
const NunjucksFilterMapSchema = z.record(z.string(), z.custom((v) => typeof v === "function"));
|
|
2767
|
-
|
|
2768
2776
|
//#endregion
|
|
2769
2777
|
//#region src/types/providers.ts
|
|
2770
2778
|
function isApiProvider(provider) {
|
|
@@ -2773,7 +2781,6 @@ function isApiProvider(provider) {
|
|
|
2773
2781
|
function isProviderOptions(provider) {
|
|
2774
2782
|
return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "string";
|
|
2775
2783
|
}
|
|
2776
|
-
|
|
2777
2784
|
//#endregion
|
|
2778
2785
|
//#region src/types/index.ts
|
|
2779
2786
|
const CommandLineOptionsSchema = z.object({
|
|
@@ -3242,7 +3249,7 @@ const EvalResultsFilterMode = z.enum([
|
|
|
3242
3249
|
"passes",
|
|
3243
3250
|
"user-rated"
|
|
3244
3251
|
]);
|
|
3245
|
-
|
|
3246
3252
|
//#endregion
|
|
3247
3253
|
export { pluginDescriptions as $, isApiProvider as A, isValidReusablePolicyId as B, TestGeneratorConfigSchema as C, BaseTokenUsageSchema as Ct, VarsSchema as D, UnifiedConfigSchema as E, ConversationMessageSchema as F, MULTI_TURN_STRATEGIES as G, AGENTIC_STRATEGIES as H, PartialGenerationError as I, getDefaultNFanout as J, STRATEGY_COLLECTIONS as K, PluginConfigSchema as L, RedteamConfigSchema as M, ProviderOptionsSchema as N, isGradingResult as O, ProvidersSchema as P, categoryAliases as Q, PolicyObjectSchema as R, TestCasesWithMetadataSchema as S, PromptSchema as St, TestSuiteSchema as T, InputsSchema as Tt, DATASET_PLUGINS as U, isUuid as V, DEFAULT_STRATEGIES as W, isFanoutStrategy as X, isCustomStrategy as Y, Severity as Z, ScenarioSchema as _, REDTEAM_PROVIDER_HARM_PLUGINS as _t, AtomicTestCaseSchema as a, FINANCIAL_PLUGINS as at, TestCaseWithVarsFileSchema as b, TELECOM_PLUGINS as bt, CompletedPromptSchema as c, INSURANCE_PLUGINS as ct, EvaluateOptionsSchema as d, MEDICAL_PLUGINS as dt, riskCategorySeverityMap as et, GradingConfigSchema as f, MULTI_INPUT_EXCLUDED_PLUGINS as ft, ResultFailureReason as g, PLUGIN_CATEGORIES as gt, OutputFileExtension as h, PII_PLUGINS as ht, AssertionTypeSchema as i, DEFAULT_PLUGINS as it, isProviderOptions as j, isResultFailureReason as k, DerivedMetricSchema as l, LLAMA_GUARD_ENABLED_CATEGORIES as lt, OutputConfigSchema as m, PHARMACY_PLUGINS as mt, AssertionSchema as n, BIAS_PLUGINS as nt, BaseAssertionTypesSchema as o, FOUNDATION_PLUGINS as ot, NotPrefixedAssertionTypesSchema as p, MULTI_INPUT_VAR as pt, STRATEGY_COLLECTION_MAPPINGS as q, AssertionSetSchema as r, DATASET_EXEMPT_PLUGINS as rt, CommandLineOptionsSchema as s, HARM_PLUGINS as st, AssertionOrSetSchema as t, ALIASED_PLUGIN_MAPPINGS as tt, EvalResultsFilterMode as u, LLAMA_GUARD_REPLICATE_PROVIDER as ut, SpecialAssertionTypesSchema as v, REMOTE_ONLY_PLUGIN_IDS as vt, TestSuiteConfigSchema as w, CompletionTokenDetailsSchema as wt, TestCasesWithMetadataPromptSchema as x, UNALIGNED_PROVIDER_HARM_PLUGINS as xt, TestCaseSchema as y, STRATEGY_EXEMPT_PLUGINS as yt, StrategyConfigSchema as z };
|
|
3248
|
-
|
|
3254
|
+
|
|
3255
|
+
//# sourceMappingURL=types-CLKiCBW3.js.map
|