promptfoo 0.120.26 → 0.121.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/drizzle/0023_wooden_mandrill.sql +2 -0
- package/dist/drizzle/meta/0023_snapshot.json +1496 -0
- package/dist/drizzle/meta/_journal.json +7 -0
- package/dist/src/{ListApp-D3DG0F8h.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-BgNJDBE6.js +206 -0
- package/dist/src/{accounts-BzEY8H3v.cjs → accounts-Bx-x3bmW.cjs} +99 -80
- package/dist/src/{accounts-DHHiXsy6.js → accounts-CMqkzrVf.js} +61 -36
- package/dist/src/{accounts-R3gfCR_g.js → accounts-xrUGFA6n.js} +60 -35
- package/dist/src/{agentic-utils-D6_gzOUF.js → agentic-utils-BKIN5PKu.js} +9 -10
- package/dist/src/{agents-CwM7re15.cjs → agents-B0f4HICh.cjs} +37 -40
- package/dist/src/{agents-Cnph5GLD.js → agents-C-dDThPK.js} +37 -37
- package/dist/src/{agents-C7BiinFI.cjs → agents-CErsqg5U.cjs} +19 -27
- package/dist/src/{agents-v4cW_ZgC.js → agents-CVIn-Utx.js} +19 -22
- package/dist/src/{agents-GiUianme.js → agents-CXknwsFX.js} +37 -40
- package/dist/src/{agents-DETIQHqF.js → agents-DeH4Gu94.js} +21 -28
- package/dist/src/{agents-DYIT-hQy.js → agents-Dy2YpZpa.js} +38 -41
- package/dist/src/{agents-Cao4i7AX.js → agents-aF4-T121.js} +19 -30
- package/dist/src/{aimlapi-DMF6v_vb.js → aimlapi-BAGZDo5G.js} +16 -18
- package/dist/src/{aimlapi-CMJpKK-B.js → aimlapi-BNfTBexL.js} +15 -17
- package/dist/src/{aimlapi-DtSf1ykJ.js → aimlapi-DHRKlBEA.js} +15 -4
- package/dist/src/{aimlapi-DoGLcQW_.cjs → aimlapi-tg0Gkcvr.cjs} +15 -16
- package/dist/src/app/assets/index-BFCZg7hQ.js +439 -0
- package/dist/src/app/assets/index-NCn4eVBv.css +1 -0
- package/dist/src/app/assets/{vendor-charts-CYyo8R8v.js → vendor-charts-CCl15Imd.js} +1 -1
- package/dist/src/app/assets/{vendor-markdown-DSmzq4Jh.js → vendor-markdown-0tekx3KX.js} +1 -1
- package/dist/src/app/index.html +4 -34
- package/dist/src/{audio-DUH4q0Xq.js → audio-BRODU0UK.js} +7 -9
- package/dist/src/{audio-BWjyvHn9.cjs → audio-BWeaWovU.cjs} +6 -7
- package/dist/src/{audio-U580w8jM.js → audio-CHQ4r-RV.js} +6 -5
- package/dist/src/{audio-BrJBFN2b.js → audio-tf_NBjlC.js} +6 -8
- package/dist/src/base-B0tcrnq_.js +193 -0
- package/dist/src/base-B4QJRyFS.js +194 -0
- package/dist/src/base-DBtwl2FR.cjs +222 -0
- package/dist/src/base-fEDN28WM.js +193 -0
- package/dist/src/{blobs-kt8v3UyH.js → blobs-BAU-dXan.js} +9 -12
- package/dist/src/{blobs-C9J2mVgo.js → blobs-Bpg5rH6i.js} +9 -12
- package/dist/src/{blobs-673H0jCl.cjs → blobs-DvS-O6be.cjs} +34 -37
- package/dist/src/blobs-qTYm-1PY.js +236 -0
- package/dist/src/{cache-BLLayYEN.js → cache-8XhNqPKW.js} +64 -67
- package/dist/src/cache-Bbn1Nyrd.cjs +5 -0
- package/dist/src/cache-BwsMSda7.js +6 -0
- package/dist/src/{cache-mIszOnuz.js → cache-CG0SlR1d.js} +64 -66
- package/dist/src/{cache-7xULbvt3.cjs → cache-COish3-W.cjs} +114 -117
- package/dist/src/cache-D3eqDYGU.js +739 -0
- package/dist/src/{chat-Fl6TZJRS.cjs → chat-2K608PeQ.cjs} +20 -21
- package/dist/src/chat-BKm79wib.js +764 -0
- package/dist/src/{chat-XPN9YHhr.js → chat-CM_kyI8B.js} +20 -9
- package/dist/src/{chat-DIywASPG.js → chat-CRWNNq73.js} +49 -49
- package/dist/src/{chat-C8Ei4f87.js → chat-CznLWr_D.js} +49 -49
- package/dist/src/{chat-CgyGj2hC.js → chat-DHMH-N64.js} +20 -22
- package/dist/src/{chat-C4zqjObh.cjs → chat-DaqekjFr.cjs} +69 -69
- package/dist/src/{chat-Cpz3O-Xl.js → chat-DxysjBvt.js} +21 -23
- package/dist/src/{chatkit-Dpxrq4eD.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DIrJX8xk.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-DEls11hE.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-e25Ziu17.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-6-xTaLwM.js → claude-agent-sdk-BLTu0WBO.js} +45 -31
- package/dist/src/{claude-agent-sdk-BzxF6NIJ.cjs → claude-agent-sdk-CJH22shf.cjs} +44 -29
- package/dist/src/{claude-agent-sdk-CmkTnKGH.js → claude-agent-sdk-D6_k9FKA.js} +45 -33
- package/dist/src/{claude-agent-sdk-rXCBLK_o.js → claude-agent-sdk-Dy5lT-Tx.js} +46 -21
- package/dist/src/{cloud-BMbRVJFw.js → cloud-Bc9526yV.js} +32 -12
- package/dist/src/cloud-DmE0EwsY.js +4 -0
- package/dist/src/{cloudflare-ai-CUg4BTcj.js → cloudflare-ai-C9r2sRhw.js} +16 -18
- package/dist/src/{cloudflare-ai-Z9X219gp.js → cloudflare-ai-CWWJCRim.js} +16 -4
- package/dist/src/{cloudflare-ai-BAQ0u_dg.cjs → cloudflare-ai-ClWSdor4.cjs} +16 -17
- package/dist/src/{cloudflare-ai-CobxMTR4.js → cloudflare-ai-ICsOuD-z.js} +17 -19
- package/dist/src/{cloudflare-gateway-C0sgfr_z.cjs → cloudflare-gateway-C2_-KG5o.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-_itGuXry.js → cloudflare-gateway-D6O7AlYb.js} +23 -23
- package/dist/src/{cloudflare-gateway-D2_yi-Fh.js → cloudflare-gateway-D6xFc5pa.js} +21 -25
- package/dist/src/{cloudflare-gateway-Djf3F3_H.js → cloudflare-gateway-pXGHxJ47.js} +26 -14
- package/dist/src/{codex-sdk-ibXwdglL.js → codex-sdk-C6UMlxwV.js} +49 -32
- package/dist/src/{codex-sdk-BASDNkIl.js → codex-sdk-DUwKWezN.js} +49 -30
- package/dist/src/{codex-sdk-dSnGdgIp.js → codex-sdk-GGAw0qbD.js} +49 -32
- package/dist/src/{codex-sdk-wTEpMM_X.cjs → codex-sdk-fAO0c3yA.cjs} +49 -32
- package/dist/src/{cometapi-B01btbfb.js → cometapi-BasUi7-_.js} +17 -19
- package/dist/src/{cometapi-DHUAH6nK.js → cometapi-Bbjp5V4x.js} +16 -4
- package/dist/src/{cometapi-ChAaRjg5.js → cometapi-DkXrKi5z.js} +21 -24
- package/dist/src/{cometapi-JbvOJSCO.cjs → cometapi-vY6aDZgo.cjs} +21 -22
- package/dist/src/{completion-D9_MDlnd.js → completion-6Mx_iXxK.js} +11 -13
- package/dist/src/{completion-BBJ6zmG3.js → completion-C5rtR_9P.js} +11 -13
- package/dist/src/{completion-DDyL3Cb2.cjs → completion-CDOouNzq.cjs} +21 -23
- package/dist/src/completion-C_P3ypkJ.js +120 -0
- package/dist/src/createHash-CTQmL3G2.js +15 -0
- package/dist/src/createHash-CfZSc0b4.cjs +27 -0
- package/dist/src/createHash-Da8fMwqB.js +16 -0
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-JAAubMw3.js → docker-5KcG-_86.js} +18 -20
- package/dist/src/{docker-Ckw-j7Rr.cjs → docker-BwsKwxFs.cjs} +18 -19
- package/dist/src/{docker-vnOg96gi.js → docker-CZnqU1XV.js} +18 -7
- package/dist/src/{docker-BuButc4D.js → docker-DzxyDPIj.js} +19 -21
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-CYhseqj4.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-rDtG_2rg.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-BQkx5roy.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-17JizQIv.js +15 -0
- package/dist/src/{eval-CYrbG57o.js → eval-DmFyWU7i.js} +49 -55
- package/dist/src/{evalResult-COsVttMA.js → evalResult-CDQiuUuf.js} +16 -12
- package/dist/src/{evalResult-6JaUIStC.js → evalResult-CTG2AHOS.js} +10 -11
- package/dist/src/evalResult-Cqj8pldJ.js +12 -0
- package/dist/src/{evalResult-DlRfu_Rq.cjs → evalResult-Dap2CekP.cjs} +20 -21
- package/dist/src/evalResult-DvcJAWJU.cjs +10 -0
- package/dist/src/evalResult-Hftn-S_i.js +10 -0
- package/dist/src/evaluator-B2CFNt-P.js +36 -0
- package/dist/src/{evaluator-3EJCMTs0.js → evaluator-DPFRbFIL.js} +210 -232
- package/dist/src/{extractor-LSYjrhK0.js → extractor-CFG6bcWJ.js} +23 -38
- package/dist/src/{extractor-DbhlYEeo.cjs → extractor-DX36oYEv.cjs} +37 -64
- package/dist/src/{extractor-Hs7la_19.js → extractor-M67RUtg6.js} +23 -38
- package/dist/src/extractor-YMU_Gvt8.js +374 -0
- package/dist/src/fetch-4M3YRaqL.js +780 -0
- package/dist/src/{fetch-18MuNu9i.js → fetch-60Gzydls.js} +60 -46
- package/dist/src/{fetch-SRsE6Ctl.js → fetch-BMv0O527.js} +41 -35
- package/dist/src/{fetch-ZMn_oemb.cjs → fetch-BxUk8odA.cjs} +268 -279
- package/dist/src/fetch-KV5kNASw.js +5 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-Doo0xq-z.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-CQlpZkrp.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/graders-Bu0H9nXi.js +32 -0
- package/dist/src/{graders-BaMCwIKp.js → graders-CHO8EPM4.js} +385 -417
- package/dist/src/graders-Cfhkvx-e.js +34 -0
- package/dist/src/{graders-QsALpIdy.js → graders-CpdqD9PI.js} +385 -417
- package/dist/src/graders-DClJVpGP.cjs +32 -0
- package/dist/src/{graders-DzUUnUjC.cjs → graders-DOXycdlG.cjs} +721 -753
- package/dist/src/graders-DcnJsrMO.js +32 -0
- package/dist/src/graders-R9rYUM0d.js +13466 -0
- package/dist/src/{image-BiEVdpdP.js → image-BmEZqVmk.js} +57 -18
- package/dist/src/{image-mhAGP07h.js → image-CBBVXWuT.js} +57 -18
- package/dist/src/{image-D10zEe1f.cjs → image-CDLQOcqT.cjs} +6 -7
- package/dist/src/{image-COCWy5dX.js → image-DJEvKveK.js} +6 -5
- package/dist/src/{image-C3BjJUAU.cjs → image-DTedmQPg.cjs} +77 -32
- package/dist/src/{image-DB4sHxdJ.js → image-gvmivTEe.js} +7 -9
- package/dist/src/image-pAX56tPG.js +257 -0
- package/dist/src/{image-BXt_7u0v.js → image-tL5hIOFh.js} +6 -8
- package/dist/src/index.cjs +696 -693
- package/dist/src/index.d.cts +113 -10
- package/dist/src/index.d.ts +113 -6
- package/dist/src/index.js +657 -658
- package/dist/src/{interactiveCheck-DU-MAhp5.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-DotRBzUE.js → knowledgeBase-CLJybhnF.js} +19 -34
- package/dist/src/{knowledgeBase-XJQ0Qyez.js → knowledgeBase-CoU-UQBg.js} +17 -41
- package/dist/src/{knowledgeBase-CMvMlLZR.js → knowledgeBase-DjWPVqSb.js} +17 -43
- package/dist/src/{knowledgeBase-Bnb00xKs.cjs → knowledgeBase-wkxuRFhA.cjs} +17 -40
- package/dist/src/{litellm-CHrRmPAe.js → litellm-B9Hysuri.js} +16 -18
- package/dist/src/{litellm-CrLJrPIm.js → litellm-CTfa0hqi.js} +15 -17
- package/dist/src/{litellm-BrnZhMcL.cjs → litellm-NYpQ8RQu.cjs} +15 -16
- package/dist/src/{litellm-BECdjOTx.js → litellm-ePxtr9F1.js} +15 -4
- package/dist/src/{logger-w8Ozp0Td.js → logger-CT3IKMKA.js} +24 -41
- package/dist/src/{logger-BdZ-IqBc.cjs → logger-Cp1GPUjj.cjs} +166 -192
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-BotXmWKW.js → logger-KkObSCzq.js} +27 -43
- package/dist/src/{luma-ray-C0RkI3lt.cjs → luma-ray-B0GGNRc1.cjs} +20 -21
- package/dist/src/{luma-ray-C-w6EsJm.js → luma-ray-BE2mOt6N.js} +20 -13
- package/dist/src/{luma-ray-BOeX-h0M.js → luma-ray-BW9IRGIc.js} +22 -21
- package/dist/src/{luma-ray-DgKLS0BF.js → luma-ray-Cm1KZBhs.js} +20 -23
- package/dist/src/main.js +1985 -2055
- package/dist/src/{messages-DXV3Qh8_.cjs → messages-1JrJs91T.cjs} +35 -34
- package/dist/src/{messages-D61tPFQo.js → messages-1x9atZmP.js} +25 -24
- package/dist/src/{messages-CDZYGNlS.js → messages-BLbWdsyt.js} +25 -24
- package/dist/src/messages-D8EA0oDc.js +240 -0
- package/dist/src/{meteor-P2rUE-Uz.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-SLNTgmXm.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-odmwVbyG.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-Dj8cTkU_.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/modelslab-C1OLRmVX.js +166 -0
- package/dist/src/modelslab-CqXBy3U8.js +168 -0
- package/dist/src/modelslab-DcOSFwKh.cjs +166 -0
- package/dist/src/modelslab-X5-4LroM.js +166 -0
- package/dist/src/{nova-reel-C2LFfVTf.js → nova-reel-BgS1ZWuK.js} +20 -13
- package/dist/src/{nova-reel-DtCjbD5O.js → nova-reel-D2ZkOSyr.js} +22 -21
- package/dist/src/{nova-reel-D9FXq3Mt.cjs → nova-reel-D9xfaMBs.cjs} +20 -21
- package/dist/src/{nova-reel-Bk5npr2q.js → nova-reel-DihqLeol.js} +20 -23
- package/dist/src/{nova-sonic-BoRSY_U6.cjs → nova-sonic-DVu3mMIy.cjs} +30 -31
- package/dist/src/{nova-sonic-D_qERM-K.js → nova-sonic-DezhVUYT.js} +30 -26
- package/dist/src/{nova-sonic-CgaWLDM1.js → nova-sonic-P-CdUMlV.js} +30 -31
- package/dist/src/{nova-sonic-BXRfQyF-.js → nova-sonic-Q3BOJeig.js} +31 -32
- package/dist/src/{openai-Bigwjgo1.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-Dz3surb_.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CT5fwbve.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-dHLcXUWZ.js → openclaw-BiSZPL7J.js} +20 -14
- package/dist/src/{openclaw-CpPrXwf6.js → openclaw-Bv1DINsX.js} +20 -27
- package/dist/src/{openclaw-B6XY2kUf.js → openclaw-D1D_ej1z.js} +21 -28
- package/dist/src/{openclaw-DDSfq5fp.cjs → openclaw-DAfWQn-o.cjs} +33 -39
- package/dist/src/opencode-sdk-C7m-wRfI.js +560 -0
- package/dist/src/opencode-sdk-CfaLN8PY.cjs +564 -0
- package/dist/src/opencode-sdk-D95s6SnR.js +562 -0
- package/dist/src/opencode-sdk-DxUPkLT7.js +560 -0
- package/dist/src/{otlpReceiver-DmRb0NBj.js → otlpReceiver--AIRW_S4.js} +53 -51
- package/dist/src/{otlpReceiver-Dg817agV.js → otlpReceiver-Bn5wGB1v.js} +53 -55
- package/dist/src/{otlpReceiver-B6Xo4KZM.cjs → otlpReceiver-Diec4cln.cjs} +53 -55
- package/dist/src/{otlpReceiver-BO0rbDzh.js → otlpReceiver-g3ByGaXs.js} +53 -55
- package/dist/src/{providerRegistry-Xf0qdqGQ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-wCWd7sKQ.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-lc7a7utN.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B3HvufyI.js +33246 -0
- package/dist/src/{providers-BiNq_Iyc.js → providers-BKRJTjBz.js} +1743 -1795
- package/dist/src/providers-C1rOSHiR.js +32 -0
- package/dist/src/{providers-BlEhY5mi.js → providers-CFLy1_ji.js} +1750 -1802
- package/dist/src/{providers-BNKVY53V.cjs → providers-CFu-TZl-.cjs} +2111 -2163
- package/dist/src/providers-CxmDwEFf.cjs +31 -0
- package/dist/src/providers-Dodakqr0.js +30 -0
- package/dist/src/providers-GIQ2TcsA.js +30 -0
- package/dist/src/{pythonUtils-r1uBuA0n.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DZ6EbdY4.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-vMlk9Qp5.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/quiverai-C2jVwbH1.js +213 -0
- package/dist/src/quiverai-CI6gYJVI.js +213 -0
- package/dist/src/quiverai-CLkWkyZc.cjs +213 -0
- package/dist/src/quiverai-MHSxbmmZ.js +215 -0
- package/dist/src/{render-CAZvKKkB.js → render-Drod8m7K.js} +4 -5
- package/dist/src/{responses-DLLjADw5.js → responses-BKqJmhhc.js} +34 -27
- package/dist/src/{responses-TsdODUpm.js → responses-CGw0DCzh.js} +34 -27
- package/dist/src/responses-jxdehPkC.js +660 -0
- package/dist/src/{responses-zOtKtnY_.cjs → responses-tD4Bd4dc.cjs} +49 -42
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-Cs35SDYa.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-BtjlqyXt.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-DCVaJ3mc.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-Du4LIR97.js → sagemaker-BK4Zb993.js} +75 -70
- package/dist/src/{sagemaker-CLdUAv5z.js → sagemaker-BfiWTmvn.js} +77 -77
- package/dist/src/{sagemaker-DwNnEVYt.cjs → sagemaker-CcQHM1jV.cjs} +75 -76
- package/dist/src/{sagemaker-BcgLu0U4.js → sagemaker-D2Q1c-sD.js} +75 -79
- package/dist/src/{scanner-Dyw21Wg_.js → scanner-J8CA3LsV.js} +149 -122
- package/dist/src/server/index.js +5620 -67302
- package/dist/src/{server-CgUQ25qW.cjs → server-B0PPuDw-.cjs} +57 -67
- package/dist/src/server-B1vi21hA.js +7 -0
- package/dist/src/{server-CbMTRQkg.js → server-BC7XJFgr.js} +19 -24
- package/dist/src/server-Cm9Kai_h.cjs +5 -0
- package/dist/src/{server-DWmZLfCy.js → server-DbFphssR.js} +26 -29
- package/dist/src/server-OAs3nBRT.js +229 -0
- package/dist/src/{signal-Bl32q42d.js → signal-BOTbd53Z.js} +9 -11
- package/dist/src/{slack-BtMkB6xP.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-OZYxoVON.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-DPqj42Ts.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-BfdBx2tO.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-BNmZ1KAz.cjs +5 -0
- package/dist/src/{store-BqwfFEyF.cjs → store-BSc-TF2w.cjs} +44 -45
- package/dist/src/store-BltJg2cd.js +6 -0
- package/dist/src/{store-D4gdn9ih.js → store-D1tv90v3.js} +34 -35
- package/dist/src/{store-2ocbYY9D.js → store-DQLEjuEO.js} +40 -36
- package/dist/src/store-Ub2vaGJ1.js +228 -0
- package/dist/src/{tables-D-NSwNIb.js → tables-5EvT_Bwn.js} +23 -23
- package/dist/src/{tables-B9E1kRp-.cjs → tables-C7K-XKWp.cjs} +93 -93
- package/dist/src/{tables-C7TT2XVn.js → tables-D36WTqKX.js} +25 -25
- package/dist/src/tables-xKANLRBD.js +288 -0
- package/dist/src/telemetry-5BCRNBbe.cjs +5 -0
- package/dist/src/{telemetry-DZ_7PaVq.js → telemetry-C15ziL8u.js} +17 -14
- package/dist/src/{telemetry-BXyVqyAg.js → telemetry-C2YDkUQH.js} +11 -13
- package/dist/src/{telemetry-D0_yFdtU.cjs → telemetry-CbrnxHp_.cjs} +21 -24
- package/dist/src/telemetry-D4W5hboe.js +7 -0
- package/dist/src/telemetry-DMb2Mpfm.js +171 -0
- package/dist/src/{text-Dm78AVGG.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-DF2hMKdg.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-DgMr_tiM.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-FNIz3YOe.cjs → transcription-CL78qbOU.cjs} +14 -15
- package/dist/src/{transcription-C-M81iDA.js → transcription-DAtxHhAM.js} +14 -7
- package/dist/src/{transcription-CYuY5sFO.js → transcription-LNZTNUUL.js} +14 -16
- package/dist/src/{transcription-Ch7S-LWw.js → transcription-QHh3AH6Z.js} +15 -17
- package/dist/src/{transform-CoP2bJ7P.js → transform-Cgi24fJ7.js} +94 -66
- package/dist/src/{transform-Kd6u-oNm.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-D8dILpfZ.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/{transform-DMaxQwDx.js → transform-DGLazrMm.js} +94 -66
- package/dist/src/transform-DGxXocjk.js +1506 -0
- package/dist/src/{transform-ivxEY4f7.cjs → transform-DOcQeLld.cjs} +234 -206
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-CqTFr7KR.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/{transformersAvailability-DEU2naS1.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-Bkep3ka7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-DwmezkVe.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-t52w-XsS.js → types-CH3Ge2sE.js} +103 -92
- package/dist/src/{types-DMVjYLpx.js → types-CLKiCBW3.js} +98 -91
- package/dist/src/types-CN_TZ2GJ.js +3260 -0
- package/dist/src/{types-BIfttHrT.cjs → types-LJ0r3wbR.cjs} +573 -566
- package/dist/src/util-5cB-L7U3.js +1430 -0
- package/dist/src/util-6-GqIvzS.js +599 -0
- package/dist/src/{util-vjscpUzy.js → util-B7T3SiBS.js} +5 -6
- package/dist/src/{util-Cl0zfT3V.js → util-Betm42rL.js} +44 -17
- package/dist/src/{util-CUEt0Vum.js → util-C-PPYSMq.js} +44 -17
- package/dist/src/{util-DkFTvieG.cjs → util-CchiqXh_.cjs} +35 -36
- package/dist/src/{util-mJ58qbbw.js → util-DaWTWKBK.js} +5 -6
- package/dist/src/{util-C08Kns6-.cjs → util-Db0a0AFH.cjs} +89 -62
- package/dist/src/{util-DiCePfDu.js → util-Dlz_Wvgm.js} +102 -53
- package/dist/src/{util-BSh4a_Q8.js → util-YT5HPZaS.js} +102 -53
- package/dist/src/{util-DUYOvxAy.cjs → util-Yz-1aEhW.cjs} +274 -219
- package/dist/src/util-ZZH-3QZz.js +293 -0
- package/dist/src/{utils-DFaZa6Rf.cjs → utils-Cz9qXqII.cjs} +32 -35
- package/dist/src/{utils-CVzb4YiI.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-JaY9veb5.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +59 -53
- package/dist/src/app/assets/index-BOgkICuY.css +0 -1
- package/dist/src/app/assets/index-CSgqn_Vd.js +0 -428
- package/dist/src/app/tsconfig.app.tsbuildinfo +0 -1
- package/dist/src/base-BaXmtXYp.js +0 -107
- package/dist/src/base-Dtp8b4_N.js +0 -106
- package/dist/src/base-f71xxWai.cjs +0 -111
- package/dist/src/cache-BUPcq0Ad.js +0 -6
- package/dist/src/cache-CVfRb-HD.cjs +0 -6
- package/dist/src/cache-O4EuX2JV.js +0 -8
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-CZ4hytdm.js +0 -5
- package/dist/src/eval-CKHWqG9f.js +0 -16
- package/dist/src/evalResult-CxTP-LMm.cjs +0 -11
- package/dist/src/evalResult-CzLURDcP.js +0 -13
- package/dist/src/evalResult-DyttNQ_G.js +0 -11
- package/dist/src/evaluator-0PvfeBYh.js +0 -38
- package/dist/src/fetch-Bi0o-fdp.js +0 -4
- package/dist/src/fetch-CMptBDVg.cjs +0 -4
- package/dist/src/fetch-DAZkv3gV.js +0 -6
- package/dist/src/graders-BCytzXrb.js +0 -34
- package/dist/src/graders-CGZQShfJ.cjs +0 -33
- package/dist/src/graders-spkuVC-E.js +0 -36
- package/dist/src/opencode-sdk-CImWVqy9.js +0 -382
- package/dist/src/opencode-sdk-CuCztr4P.js +0 -380
- package/dist/src/opencode-sdk-DhcfRbBH.js +0 -376
- package/dist/src/opencode-sdk-mqF-Oj3f.cjs +0 -383
- package/dist/src/providers-BMZZmPBJ.cjs +0 -32
- package/dist/src/providers-CQQrNaJk.js +0 -32
- package/dist/src/providers-Ck8HyrC-.js +0 -34
- package/dist/src/quiverai-BNfIwKCO.cjs +0 -54
- package/dist/src/quiverai-BQigKdIH.js +0 -57
- package/dist/src/quiverai-Bfy2WnE2.js +0 -55
- package/dist/src/quiverai-CedIP0PJ.js +0 -43
- package/dist/src/rubyUtils-D7--T12C.js +0 -6
- package/dist/src/rubyUtils-DRRiMFV2.js +0 -5
- package/dist/src/rubyUtils-vb8OYFC-.cjs +0 -5
- package/dist/src/server-BUbS0Qfh.js +0 -6
- package/dist/src/server-XpGXFHkS.cjs +0 -6
- package/dist/src/server-gfOx5Zrk.js +0 -8
- package/dist/src/store-5u2yriTV.js +0 -7
- package/dist/src/store-D_lq_8oQ.js +0 -6
- package/dist/src/store-m5KT6Ly7.cjs +0 -6
- package/dist/src/telemetry-5RHFoCJh.js +0 -6
- package/dist/src/telemetry-Do8wMnA-.js +0 -8
- package/dist/src/telemetry-LojxPoFq.cjs +0 -6
- package/dist/src/transform-8eGmaH-7.js +0 -7
- package/dist/src/transform-BRVvWaG4.cjs +0 -6
- package/dist/src/transform-GybT0X0u.js +0 -8
- package/dist/src/transformersAvailability-DkAWaK5B.js +0 -35
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-
|
|
2
|
+
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-Ds-foDzt.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import dedent from "dedent";
|
|
5
|
-
|
|
6
5
|
//#region src/types/shared.ts
|
|
7
6
|
const CompletionTokenDetailsSchema = z.object({
|
|
8
7
|
reasoning: z.number().optional(),
|
|
@@ -29,10 +28,8 @@ const BaseTokenUsageSchema = z.object({
|
|
|
29
28
|
}).optional()
|
|
30
29
|
});
|
|
31
30
|
const InputsSchema = z.record(z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" }), z.string().min(1, { error: "Input descriptions must be non-empty strings" }));
|
|
32
|
-
|
|
33
31
|
//#endregion
|
|
34
32
|
//#region src/redteam/constants/plugins.ts
|
|
35
|
-
const DEFAULT_NUM_TESTS_PER_PLUGIN = 5;
|
|
36
33
|
const MULTI_INPUT_VAR = "__prompt";
|
|
37
34
|
const REDTEAM_MODEL = "openai:chat:gpt-5-2025-08-07";
|
|
38
35
|
const LLAMA_GUARD_REPLICATE_PROVIDER = "replicate:moderation:meta/llama-guard-4-12b";
|
|
@@ -230,6 +227,7 @@ const PHARMACY_PLUGINS = [
|
|
|
230
227
|
];
|
|
231
228
|
const INSURANCE_PLUGINS = [
|
|
232
229
|
"insurance:coverage-discrimination",
|
|
230
|
+
"insurance:data-disclosure",
|
|
233
231
|
"insurance:network-misinformation",
|
|
234
232
|
"insurance:phi-disclosure"
|
|
235
233
|
];
|
|
@@ -315,6 +313,7 @@ const ADDITIONAL_PLUGINS = [
|
|
|
315
313
|
"ecommerce:price-manipulation",
|
|
316
314
|
"goal-misalignment",
|
|
317
315
|
"insurance:coverage-discrimination",
|
|
316
|
+
"insurance:data-disclosure",
|
|
318
317
|
"insurance:network-misinformation",
|
|
319
318
|
"insurance:phi-disclosure",
|
|
320
319
|
"off-topic",
|
|
@@ -392,7 +391,7 @@ const DEFAULT_PLUGINS = new Set([...[
|
|
|
392
391
|
...PII_PLUGINS,
|
|
393
392
|
...BIAS_PLUGINS
|
|
394
393
|
].sort()]);
|
|
395
|
-
|
|
394
|
+
new Set([
|
|
396
395
|
...DEFAULT_PLUGINS,
|
|
397
396
|
"bola",
|
|
398
397
|
"bfla",
|
|
@@ -452,12 +451,11 @@ const REMOTE_ONLY_PLUGIN_IDS = [
|
|
|
452
451
|
...TELECOM_PLUGINS,
|
|
453
452
|
...REALESTATE_PLUGINS
|
|
454
453
|
];
|
|
455
|
-
|
|
454
|
+
[
|
|
456
455
|
...Object.keys(UNALIGNED_PROVIDER_HARM_PLUGINS),
|
|
457
456
|
...BIAS_PLUGINS,
|
|
458
457
|
...REMOTE_ONLY_PLUGIN_IDS
|
|
459
458
|
];
|
|
460
|
-
|
|
461
459
|
//#endregion
|
|
462
460
|
//#region src/redteam/constants/frameworks.ts
|
|
463
461
|
const OWASP_LLM_TOP_10_MAPPING = {
|
|
@@ -1296,7 +1294,73 @@ const GDPR_MAPPING = {
|
|
|
1296
1294
|
strategies: []
|
|
1297
1295
|
}
|
|
1298
1296
|
};
|
|
1297
|
+
/**
|
|
1298
|
+
* U.S. Department of Defense (DoD) AI ethical principles.
|
|
1299
|
+
*
|
|
1300
|
+
* Source:
|
|
1301
|
+
* * https://www.defense.gov/News/News-Stories/Article/Article/2094085/dod-adopts-5-principles-of-artificial-intelligence-ethics/
|
|
1302
|
+
*/
|
|
1303
|
+
const DOD_AI_ETHICS_MAPPING = {
|
|
1304
|
+
"dod:ai:ethics:01": {
|
|
1305
|
+
plugins: [
|
|
1306
|
+
"excessive-agency",
|
|
1307
|
+
"goal-misalignment",
|
|
1308
|
+
"overreliance",
|
|
1309
|
+
"hijacking"
|
|
1310
|
+
],
|
|
1311
|
+
strategies: ["jailbreak", "jailbreak-templates"]
|
|
1312
|
+
},
|
|
1313
|
+
"dod:ai:ethics:02": {
|
|
1314
|
+
plugins: [
|
|
1315
|
+
"bias:age",
|
|
1316
|
+
"bias:disability",
|
|
1317
|
+
"bias:gender",
|
|
1318
|
+
"bias:race",
|
|
1319
|
+
"harmful:hate"
|
|
1320
|
+
],
|
|
1321
|
+
strategies: []
|
|
1322
|
+
},
|
|
1323
|
+
"dod:ai:ethics:03": {
|
|
1324
|
+
plugins: [
|
|
1325
|
+
"hallucination",
|
|
1326
|
+
"harmful:misinformation-disinformation",
|
|
1327
|
+
"rag-source-attribution",
|
|
1328
|
+
"unverifiable-claims"
|
|
1329
|
+
],
|
|
1330
|
+
strategies: []
|
|
1331
|
+
},
|
|
1332
|
+
"dod:ai:ethics:04": {
|
|
1333
|
+
plugins: [
|
|
1334
|
+
"harmful:misinformation-disinformation",
|
|
1335
|
+
"harmful:unsafe-practices",
|
|
1336
|
+
"shell-injection",
|
|
1337
|
+
"sql-injection",
|
|
1338
|
+
"ssrf",
|
|
1339
|
+
"debug-access",
|
|
1340
|
+
"reasoning-dos"
|
|
1341
|
+
],
|
|
1342
|
+
strategies: ["jailbreak", "jailbreak-templates"]
|
|
1343
|
+
},
|
|
1344
|
+
"dod:ai:ethics:05": {
|
|
1345
|
+
plugins: [
|
|
1346
|
+
"excessive-agency",
|
|
1347
|
+
"hijacking",
|
|
1348
|
+
"indirect-prompt-injection",
|
|
1349
|
+
"system-prompt-override",
|
|
1350
|
+
"rbac",
|
|
1351
|
+
"bfla",
|
|
1352
|
+
"bola",
|
|
1353
|
+
"tool-discovery"
|
|
1354
|
+
],
|
|
1355
|
+
strategies: [
|
|
1356
|
+
"jailbreak",
|
|
1357
|
+
"jailbreak-templates",
|
|
1358
|
+
"jailbreak:composite"
|
|
1359
|
+
]
|
|
1360
|
+
}
|
|
1361
|
+
};
|
|
1299
1362
|
const ALIASED_PLUGINS = [
|
|
1363
|
+
"dod:ai:ethics",
|
|
1300
1364
|
"mitre:atlas",
|
|
1301
1365
|
"nist:ai",
|
|
1302
1366
|
"nist:ai:measure",
|
|
@@ -1323,9 +1387,11 @@ const ALIASED_PLUGINS = [
|
|
|
1323
1387
|
...Object.keys(OWASP_AGENTIC_TOP_10_MAPPING),
|
|
1324
1388
|
...Object.keys(EU_AI_ACT_MAPPING),
|
|
1325
1389
|
...Object.keys(ISO_42001_MAPPING),
|
|
1326
|
-
...Object.keys(GDPR_MAPPING)
|
|
1390
|
+
...Object.keys(GDPR_MAPPING),
|
|
1391
|
+
...Object.keys(DOD_AI_ETHICS_MAPPING)
|
|
1327
1392
|
];
|
|
1328
1393
|
const ALIASED_PLUGIN_MAPPINGS = {
|
|
1394
|
+
"dod:ai:ethics": DOD_AI_ETHICS_MAPPING,
|
|
1329
1395
|
"mitre:atlas": MITRE_ATLAS_MAPPING,
|
|
1330
1396
|
"nist:ai:measure": NIST_AI_RMF_MAPPING,
|
|
1331
1397
|
"owasp:api": OWASP_API_TOP_10_MAPPING,
|
|
@@ -1386,7 +1452,6 @@ const ALIASED_PLUGIN_MAPPINGS = {
|
|
|
1386
1452
|
strategies: []
|
|
1387
1453
|
} }
|
|
1388
1454
|
};
|
|
1389
|
-
|
|
1390
1455
|
//#endregion
|
|
1391
1456
|
//#region src/redteam/constants/metadata.ts
|
|
1392
1457
|
const subCategoryDescriptions = {
|
|
@@ -1464,6 +1529,7 @@ const subCategoryDescriptions = {
|
|
|
1464
1529
|
imitation: "Tests handling of unauthorized impersonation of entities",
|
|
1465
1530
|
"indirect-prompt-injection": "Tests for injection vulnerabilities via untrusted variables",
|
|
1466
1531
|
"insurance:coverage-discrimination": "Tests for discriminatory coverage decisions based on protected characteristics (age, disability, race, etc.)",
|
|
1532
|
+
"insurance:data-disclosure": "Tests for unauthorized disclosure of protected policyholder data including claims history, property details, driving records, and financial information",
|
|
1467
1533
|
"insurance:network-misinformation": "Tests for inaccurate provider network information",
|
|
1468
1534
|
"insurance:phi-disclosure": "Tests for unauthorized disclosure of Protected Health Information in violation of HIPAA privacy regulations",
|
|
1469
1535
|
"ecommerce:pci-dss": "Tests for exposure of payment card data including credit card numbers, CVVs, and bank account details in violation of PCI DSS requirements",
|
|
@@ -1546,7 +1612,7 @@ const subCategoryDescriptions = {
|
|
|
1546
1612
|
bias: "Bias detection across protected characteristics",
|
|
1547
1613
|
medical: "Medical AI safety testing across healthcare vulnerabilities",
|
|
1548
1614
|
pharmacy: "Pharmacy AI safety testing across pharmaceutical vulnerabilities",
|
|
1549
|
-
insurance: "Insurance AI safety testing across
|
|
1615
|
+
insurance: "Insurance AI safety testing across all insurance types",
|
|
1550
1616
|
financial: "Financial AI safety testing across financial services vulnerabilities",
|
|
1551
1617
|
ecommerce: "E-commerce AI safety testing across payment security and transaction integrity vulnerabilities",
|
|
1552
1618
|
telecom: "Telecommunications AI safety testing across CPNI protection, account security, and regulatory compliance vulnerabilities",
|
|
@@ -1586,20 +1652,8 @@ const SeveritySchema = z.enum([
|
|
|
1586
1652
|
"low",
|
|
1587
1653
|
"informational"
|
|
1588
1654
|
]);
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
[Severity.High]: "High",
|
|
1592
|
-
[Severity.Medium]: "Medium",
|
|
1593
|
-
[Severity.Low]: "Low",
|
|
1594
|
-
[Severity.Informational]: "Informational"
|
|
1595
|
-
};
|
|
1596
|
-
const severityRiskScores = {
|
|
1597
|
-
[Severity.Critical]: 9,
|
|
1598
|
-
[Severity.High]: 7,
|
|
1599
|
-
[Severity.Medium]: 4,
|
|
1600
|
-
[Severity.Low]: 0,
|
|
1601
|
-
[Severity.Informational]: 0
|
|
1602
|
-
};
|
|
1655
|
+
Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
|
|
1656
|
+
Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
|
|
1603
1657
|
const riskCategorySeverityMap = {
|
|
1604
1658
|
["agentic:memory-poisoning"]: Severity.High,
|
|
1605
1659
|
aegis: Severity.Medium,
|
|
@@ -1688,6 +1742,7 @@ const riskCategorySeverityMap = {
|
|
|
1688
1742
|
imitation: Severity.Low,
|
|
1689
1743
|
"indirect-prompt-injection": Severity.High,
|
|
1690
1744
|
"insurance:coverage-discrimination": Severity.Critical,
|
|
1745
|
+
"insurance:data-disclosure": Severity.Critical,
|
|
1691
1746
|
"insurance:network-misinformation": Severity.High,
|
|
1692
1747
|
"insurance:phi-disclosure": Severity.Critical,
|
|
1693
1748
|
"ecommerce:pci-dss": Severity.Critical,
|
|
@@ -1748,7 +1803,7 @@ const riskCategorySeverityMap = {
|
|
|
1748
1803
|
wordplay: Severity.Low,
|
|
1749
1804
|
xstest: Severity.Low
|
|
1750
1805
|
};
|
|
1751
|
-
|
|
1806
|
+
Object.entries({
|
|
1752
1807
|
"Security & Access Control": [
|
|
1753
1808
|
"ascii-smuggling",
|
|
1754
1809
|
"bfla",
|
|
@@ -1771,6 +1826,7 @@ const riskCategories = {
|
|
|
1771
1826
|
"data-exfil",
|
|
1772
1827
|
"divergent-repetition",
|
|
1773
1828
|
"harmful:privacy",
|
|
1829
|
+
"insurance:data-disclosure",
|
|
1774
1830
|
"insurance:phi-disclosure",
|
|
1775
1831
|
"pii:api-db",
|
|
1776
1832
|
"pii:direct",
|
|
@@ -1896,14 +1952,12 @@ const riskCategories = {
|
|
|
1896
1952
|
"vlsu",
|
|
1897
1953
|
"xstest"
|
|
1898
1954
|
]
|
|
1899
|
-
}
|
|
1900
|
-
const categoryMapReverse = Object.entries(riskCategories).reduce((acc, [category, harms]) => {
|
|
1955
|
+
}).reduce((acc, [category, harms]) => {
|
|
1901
1956
|
harms.forEach((harm) => {
|
|
1902
1957
|
acc[harm] = category;
|
|
1903
1958
|
});
|
|
1904
1959
|
return acc;
|
|
1905
1960
|
}, {});
|
|
1906
|
-
const categoryLabels = Object.keys(categoryMapReverse);
|
|
1907
1961
|
const categoryAliases = {
|
|
1908
1962
|
["agentic:memory-poisoning"]: "AgenticMemoryPoisoning",
|
|
1909
1963
|
aegis: "Aegis",
|
|
@@ -2021,6 +2075,7 @@ const categoryAliases = {
|
|
|
2021
2075
|
imitation: "Imitation",
|
|
2022
2076
|
"indirect-prompt-injection": "Indirect Prompt Injection",
|
|
2023
2077
|
"insurance:coverage-discrimination": "InsuranceCoverageDiscrimination",
|
|
2078
|
+
"insurance:data-disclosure": "InsuranceDataDisclosure",
|
|
2024
2079
|
"insurance:network-misinformation": "InsuranceNetworkMisinformation",
|
|
2025
2080
|
"insurance:phi-disclosure": "InsurancePhiDisclosure",
|
|
2026
2081
|
intent: "Intent",
|
|
@@ -2052,7 +2107,7 @@ const categoryAliases = {
|
|
|
2052
2107
|
wordplay: "Wordplay",
|
|
2053
2108
|
xstest: "XSTest"
|
|
2054
2109
|
};
|
|
2055
|
-
|
|
2110
|
+
Object.entries(categoryAliases).reduce((acc, [key, value]) => {
|
|
2056
2111
|
acc[value] = key;
|
|
2057
2112
|
return acc;
|
|
2058
2113
|
}, {});
|
|
@@ -2090,7 +2145,7 @@ const pluginDescriptions = {
|
|
|
2090
2145
|
bias: "Unified bias detection across protected characteristics including age, disability, gender, and race",
|
|
2091
2146
|
medical: "Comprehensive medical AI safety testing for healthcare-specific vulnerabilities including hallucination, anchoring bias, incorrect knowledge, off-label use, prioritization errors, and sycophancy",
|
|
2092
2147
|
pharmacy: "Comprehensive pharmacy AI safety testing for pharmaceutical vulnerabilities including drug interactions, dosage calculations, and controlled substance compliance",
|
|
2093
|
-
insurance: "Comprehensive insurance AI safety testing
|
|
2148
|
+
insurance: "Comprehensive insurance AI safety testing across all insurance types including coverage discrimination, network misinformation, PHI disclosure, and policyholder data disclosure",
|
|
2094
2149
|
financial: "Comprehensive financial AI safety testing for financial services vulnerabilities including calculation errors, compliance violations, hallucination, and data leakage",
|
|
2095
2150
|
ecommerce: "Comprehensive e-commerce AI safety testing for payment security and transaction integrity vulnerabilities including PCI DSS compliance, fraud detection, and price manipulation",
|
|
2096
2151
|
telecom: "Comprehensive telecommunications AI safety testing across CPNI protection, account security, regulatory compliance (FCC, TCPA, CALEA), and telecom-specific fraud prevention",
|
|
@@ -2143,8 +2198,9 @@ const pluginDescriptions = {
|
|
|
2143
2198
|
"harmful:weapons:ied": "Tests handling of Improvised Explosive Devices",
|
|
2144
2199
|
hijacking: "Assesses protection against unauthorized resource usage and purpose deviation",
|
|
2145
2200
|
imitation: "Tests safeguards against unauthorized entity impersonation attempts",
|
|
2146
|
-
"insurance:coverage-discrimination": "Tests for discriminatory coverage or
|
|
2147
|
-
"insurance:
|
|
2201
|
+
"insurance:coverage-discrimination": "Tests for discriminatory coverage, underwriting, or claims determinations based on protected characteristics (age, disability, race, genetic information, sex) in violation of federal civil rights laws including ADA, Section 1557, GINA, Fair Housing Act, ECOA, and state unfair trade practices acts",
|
|
2202
|
+
"insurance:data-disclosure": "Tests for unauthorized disclosure of protected policyholder data including claims history, property details, driving records, beneficiary information, and business data in violation of GLBA, FCRA, DPPA, and state insurance privacy laws",
|
|
2203
|
+
"insurance:network-misinformation": "Tests for inaccurate provider or vendor network information including wrong network status, terminated contracts, ghost networks, and outdated provider data that could expose policyholders to unexpected costs, voided warranties, or inadequate service",
|
|
2148
2204
|
"insurance:phi-disclosure": "Tests for unauthorized disclosure of Protected Health Information in violation of HIPAA privacy regulations including cross-member PHI leakage, inadequate authentication, and disclosure to unauthorized third parties",
|
|
2149
2205
|
"ecommerce:pci-dss": "Tests for exposure of payment card data including credit card numbers, CVVs, expiration dates, cardholder names, bank account details, and payment tokens in violation of PCI DSS requirements for secure payment processing",
|
|
2150
2206
|
"ecommerce:compliance-bypass": "Tests for attempts to bypass security rules, terms of service, age restrictions, geographic limitations, or compliance policies in e-commerce systems",
|
|
@@ -2204,7 +2260,6 @@ const pluginDescriptions = {
|
|
|
2204
2260
|
xstest: "Tests how models handle ambiguous terms related to potentially harmful topics like violence and drugs",
|
|
2205
2261
|
"guardrails-eval": "Evaluate guardrail effectiveness against common risks"
|
|
2206
2262
|
};
|
|
2207
|
-
|
|
2208
2263
|
//#endregion
|
|
2209
2264
|
//#region src/redteam/constants/strategies.ts
|
|
2210
2265
|
const FRAMEWORK_COMPLIANCE_IDS = [
|
|
@@ -2215,15 +2270,14 @@ const FRAMEWORK_COMPLIANCE_IDS = [
|
|
|
2215
2270
|
"owasp:agentic",
|
|
2216
2271
|
"eu:ai-act",
|
|
2217
2272
|
"iso:42001",
|
|
2218
|
-
"gdpr"
|
|
2273
|
+
"gdpr",
|
|
2274
|
+
"dod:ai:ethics"
|
|
2219
2275
|
];
|
|
2220
2276
|
const DEFAULT_STRATEGIES = [
|
|
2221
2277
|
"basic",
|
|
2222
2278
|
"jailbreak:meta",
|
|
2223
2279
|
"jailbreak:composite"
|
|
2224
2280
|
];
|
|
2225
|
-
const DEFAULT_STRATEGIES_SET = new Set(DEFAULT_STRATEGIES);
|
|
2226
|
-
const DEFAULT_MULTI_TURN_MAX_TURNS = 5;
|
|
2227
2281
|
const MULTI_TURN_STRATEGIES = [
|
|
2228
2282
|
"crescendo",
|
|
2229
2283
|
"goat",
|
|
@@ -2238,12 +2292,6 @@ const isMultiTurnStrategy = (strategyId) => {
|
|
|
2238
2292
|
const isCustomStrategy = (strategyId) => {
|
|
2239
2293
|
return strategyId === "custom" || strategyId.startsWith("custom:");
|
|
2240
2294
|
};
|
|
2241
|
-
const MULTI_MODAL_STRATEGIES = [
|
|
2242
|
-
"audio",
|
|
2243
|
-
"image",
|
|
2244
|
-
"video"
|
|
2245
|
-
];
|
|
2246
|
-
const MULTI_MODAL_STRATEGIES_SET = new Set(MULTI_MODAL_STRATEGIES);
|
|
2247
2295
|
const AGENTIC_STRATEGIES = [
|
|
2248
2296
|
"crescendo",
|
|
2249
2297
|
"goat",
|
|
@@ -2255,7 +2303,6 @@ const AGENTIC_STRATEGIES = [
|
|
|
2255
2303
|
"jailbreak:tree",
|
|
2256
2304
|
"mischievous-user"
|
|
2257
2305
|
];
|
|
2258
|
-
const AGENTIC_STRATEGIES_SET = new Set(AGENTIC_STRATEGIES);
|
|
2259
2306
|
const DATASET_PLUGINS = [
|
|
2260
2307
|
"beavertails",
|
|
2261
2308
|
"cyberseceval",
|
|
@@ -2317,22 +2364,6 @@ const _ALL_STRATEGIES = [
|
|
|
2317
2364
|
...AGENTIC_STRATEGIES
|
|
2318
2365
|
];
|
|
2319
2366
|
const ALL_STRATEGIES = Array.from(new Set(_ALL_STRATEGIES)).sort();
|
|
2320
|
-
const CONFIGURABLE_STRATEGIES = [
|
|
2321
|
-
"layer",
|
|
2322
|
-
"best-of-n",
|
|
2323
|
-
"goat",
|
|
2324
|
-
"crescendo",
|
|
2325
|
-
"indirect-web-pwn",
|
|
2326
|
-
"jailbreak",
|
|
2327
|
-
"jailbreak:hydra",
|
|
2328
|
-
"jailbreak:meta",
|
|
2329
|
-
"jailbreak:tree",
|
|
2330
|
-
"gcg",
|
|
2331
|
-
"citation",
|
|
2332
|
-
"custom",
|
|
2333
|
-
"mischievous-user"
|
|
2334
|
-
];
|
|
2335
|
-
const CONFIGURABLE_STRATEGIES_SET = new Set(CONFIGURABLE_STRATEGIES);
|
|
2336
2367
|
/**
|
|
2337
2368
|
* Default 'n' fan out for strategies that can add additional test cases during generation
|
|
2338
2369
|
*/
|
|
@@ -2347,19 +2378,6 @@ function getDefaultNFanout(strategyId) {
|
|
|
2347
2378
|
function isFanoutStrategy(strategyId) {
|
|
2348
2379
|
return strategyId in DEFAULT_N_FAN_OUT_BY_STRATEGY;
|
|
2349
2380
|
}
|
|
2350
|
-
const STRATEGIES_REQUIRING_REMOTE = [
|
|
2351
|
-
"audio",
|
|
2352
|
-
"citation",
|
|
2353
|
-
"gcg",
|
|
2354
|
-
"goat",
|
|
2355
|
-
"indirect-web-pwn",
|
|
2356
|
-
"jailbreak:composite",
|
|
2357
|
-
"jailbreak:hydra",
|
|
2358
|
-
"jailbreak:likert",
|
|
2359
|
-
"jailbreak:meta"
|
|
2360
|
-
];
|
|
2361
|
-
const STRATEGIES_REQUIRING_REMOTE_SET = new Set(STRATEGIES_REQUIRING_REMOTE);
|
|
2362
|
-
|
|
2363
2381
|
//#endregion
|
|
2364
2382
|
//#region src/util/uuid.ts
|
|
2365
2383
|
/**
|
|
@@ -2376,7 +2394,6 @@ const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-
|
|
|
2376
2394
|
function isUuid(value) {
|
|
2377
2395
|
return UUID_REGEX.test(value);
|
|
2378
2396
|
}
|
|
2379
|
-
|
|
2380
2397
|
//#endregion
|
|
2381
2398
|
//#region src/redteam/plugins/policy/validators.ts
|
|
2382
2399
|
/**
|
|
@@ -2412,7 +2429,6 @@ function isValidInlinePolicyId(id) {
|
|
|
2412
2429
|
function isValidPolicyId(id) {
|
|
2413
2430
|
return isValidReusablePolicyId(id) || isValidInlinePolicyId(id);
|
|
2414
2431
|
}
|
|
2415
|
-
|
|
2416
2432
|
//#endregion
|
|
2417
2433
|
//#region src/redteam/types.ts
|
|
2418
2434
|
const PolicyObjectSchema = z.object({
|
|
@@ -2447,6 +2463,7 @@ const PluginConfigSchema = z.object({
|
|
|
2447
2463
|
name: z.string().optional(),
|
|
2448
2464
|
multilingual: z.boolean().optional(),
|
|
2449
2465
|
indirectInjectionVar: z.string().optional(),
|
|
2466
|
+
intendedResults: z.array(z.string()).optional(),
|
|
2450
2467
|
intent: z.union([z.string(), z.array(z.union([z.string(), z.array(z.string())]))]).optional(),
|
|
2451
2468
|
policy: z.union([z.string(), PolicyObjectSchema]).optional(),
|
|
2452
2469
|
systemPrompt: z.string().optional(),
|
|
@@ -2478,7 +2495,6 @@ var PartialGenerationError = class extends Error {
|
|
|
2478
2495
|
this.failedPlugins = failedPlugins;
|
|
2479
2496
|
}
|
|
2480
2497
|
};
|
|
2481
|
-
|
|
2482
2498
|
//#endregion
|
|
2483
2499
|
//#region src/types/env.ts
|
|
2484
2500
|
const ProviderEnvOverridesSchema = z.object({
|
|
@@ -2543,6 +2559,7 @@ const ProviderEnvOverridesSchema = z.object({
|
|
|
2543
2559
|
MISTRAL_API_BASE_URL: z.string().optional(),
|
|
2544
2560
|
MISTRAL_API_HOST: z.string().optional(),
|
|
2545
2561
|
MISTRAL_API_KEY: z.string().optional(),
|
|
2562
|
+
MODELSLAB_API_KEY: z.string().optional(),
|
|
2546
2563
|
NSCALE_SERVICE_TOKEN: z.string().optional(),
|
|
2547
2564
|
NSCALE_API_KEY: z.string().optional(),
|
|
2548
2565
|
OLLAMA_API_KEY: z.string().optional(),
|
|
@@ -2599,7 +2616,6 @@ const ProviderEnvOverridesSchema = z.object({
|
|
|
2599
2616
|
AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
|
|
2600
2617
|
PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
|
|
2601
2618
|
});
|
|
2602
|
-
|
|
2603
2619
|
//#endregion
|
|
2604
2620
|
//#region src/validators/providers.ts
|
|
2605
2621
|
const ProviderOptionsSchema = z.object({
|
|
@@ -2624,7 +2640,7 @@ const ApiProviderSchema = z.object({
|
|
|
2624
2640
|
config: z.any().optional(),
|
|
2625
2641
|
inputs: InputsSchema.optional()
|
|
2626
2642
|
});
|
|
2627
|
-
|
|
2643
|
+
z.object({
|
|
2628
2644
|
cached: z.boolean().optional(),
|
|
2629
2645
|
cost: z.number().optional(),
|
|
2630
2646
|
error: z.string().optional(),
|
|
@@ -2633,17 +2649,17 @@ const ProviderResponseSchema = z.object({
|
|
|
2633
2649
|
output: z.union([z.string(), z.any()]).optional(),
|
|
2634
2650
|
tokenUsage: BaseTokenUsageSchema.optional()
|
|
2635
2651
|
});
|
|
2636
|
-
|
|
2652
|
+
z.object({
|
|
2637
2653
|
error: z.string().optional(),
|
|
2638
2654
|
embedding: z.array(z.number()).optional(),
|
|
2639
2655
|
tokenUsage: BaseTokenUsageSchema.partial().optional()
|
|
2640
2656
|
});
|
|
2641
|
-
|
|
2657
|
+
z.object({
|
|
2642
2658
|
error: z.string().optional(),
|
|
2643
2659
|
similarity: z.number().optional(),
|
|
2644
2660
|
tokenUsage: BaseTokenUsageSchema.partial().optional()
|
|
2645
2661
|
});
|
|
2646
|
-
|
|
2662
|
+
z.object({
|
|
2647
2663
|
error: z.string().optional(),
|
|
2648
2664
|
classification: z.record(z.string(), z.number()).optional()
|
|
2649
2665
|
});
|
|
@@ -2662,7 +2678,6 @@ const ProviderSchema = z.union([
|
|
|
2662
2678
|
ApiProviderSchema,
|
|
2663
2679
|
ProviderOptionsSchema
|
|
2664
2680
|
]);
|
|
2665
|
-
|
|
2666
2681
|
//#endregion
|
|
2667
2682
|
//#region src/validators/prompts.ts
|
|
2668
2683
|
const PromptConfigSchema = z.object({
|
|
@@ -2682,7 +2697,6 @@ function assert$1() {}
|
|
|
2682
2697
|
assert$1();
|
|
2683
2698
|
assert$1();
|
|
2684
2699
|
assert$1();
|
|
2685
|
-
|
|
2686
2700
|
//#endregion
|
|
2687
2701
|
//#region src/validators/redteam.ts
|
|
2688
2702
|
const TracingConfigSchema = z.lazy(() => z.object({
|
|
@@ -2727,7 +2741,7 @@ const RedteamPluginObjectSchema = z.object({
|
|
|
2727
2741
|
message: `Invalid plugin id "${val}". Custom plugins must start with file:// or use a built-in plugin. See https://www.promptfoo.dev/docs/red-team/plugins for available plugins.`
|
|
2728
2742
|
});
|
|
2729
2743
|
})]).describe("Name of the plugin"),
|
|
2730
|
-
numTests: z.int().positive().prefault(
|
|
2744
|
+
numTests: z.int().positive().prefault(5).describe("Number of tests to generate for this plugin"),
|
|
2731
2745
|
config: z.record(z.string(), z.unknown()).optional().describe("Plugin-specific configuration"),
|
|
2732
2746
|
severity: SeveritySchema.optional().describe("Severity level for this plugin")
|
|
2733
2747
|
});
|
|
@@ -2949,11 +2963,9 @@ const RedteamConfigSchema = z.object({
|
|
|
2949
2963
|
});
|
|
2950
2964
|
function assert() {}
|
|
2951
2965
|
assert();
|
|
2952
|
-
|
|
2953
2966
|
//#endregion
|
|
2954
2967
|
//#region src/validators/shared.ts
|
|
2955
2968
|
const NunjucksFilterMapSchema = z.record(z.string(), z.custom((v) => typeof v === "function"));
|
|
2956
|
-
|
|
2957
2969
|
//#endregion
|
|
2958
2970
|
//#region src/types/providers.ts
|
|
2959
2971
|
function isApiProvider(provider) {
|
|
@@ -2962,7 +2974,6 @@ function isApiProvider(provider) {
|
|
|
2962
2974
|
function isProviderOptions(provider) {
|
|
2963
2975
|
return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "string";
|
|
2964
2976
|
}
|
|
2965
|
-
|
|
2966
2977
|
//#endregion
|
|
2967
2978
|
//#region src/types/index.ts
|
|
2968
2979
|
const CommandLineOptionsSchema = z.object({
|
|
@@ -3227,12 +3238,12 @@ const TestCaseSchema = z.object({
|
|
|
3227
3238
|
strategyConfig: z.custom().optional()
|
|
3228
3239
|
}).catchall(z.any()).optional()
|
|
3229
3240
|
});
|
|
3230
|
-
|
|
3241
|
+
TestCaseSchema.extend({ vars: z.union([
|
|
3231
3242
|
VarsSchema,
|
|
3232
3243
|
z.string(),
|
|
3233
3244
|
z.array(z.string())
|
|
3234
3245
|
]).optional() });
|
|
3235
|
-
|
|
3246
|
+
z.object({
|
|
3236
3247
|
id: z.string(),
|
|
3237
3248
|
testCases: z.union([z.string(), z.array(z.union([z.string(), TestCaseSchema]))]),
|
|
3238
3249
|
recentEvalDate: z.date(),
|
|
@@ -3245,7 +3256,7 @@ const ScenarioSchema = z.object({
|
|
|
3245
3256
|
config: z.array(TestCaseSchema.partial()),
|
|
3246
3257
|
tests: z.array(TestCaseSchema)
|
|
3247
3258
|
});
|
|
3248
|
-
|
|
3259
|
+
TestCaseSchema.extend({ vars: VarsSchema.optional() }).strict();
|
|
3249
3260
|
/**
|
|
3250
3261
|
* Configuration schema for test generators that accept parameters
|
|
3251
3262
|
*
|
|
@@ -3431,7 +3442,7 @@ const EvalResultsFilterMode = z.enum([
|
|
|
3431
3442
|
"passes",
|
|
3432
3443
|
"user-rated"
|
|
3433
3444
|
]);
|
|
3434
|
-
|
|
3435
3445
|
//#endregion
|
|
3436
|
-
export {
|
|
3437
|
-
|
|
3446
|
+
export { INSURANCE_PLUGINS as $, DATASET_PLUGINS as A, categoryAliases as B, PolicyObjectSchema as C, ADDITIONAL_STRATEGIES as D, isUuid as E, getDefaultNFanout as F, ADDITIONAL_PLUGINS as G, riskCategorySeverityMap as H, isCustomStrategy as I, DATASET_EXEMPT_PLUGINS as J, ALL_PLUGINS as K, isFanoutStrategy as L, MULTI_TURN_STRATEGIES as M, STRATEGY_COLLECTIONS as N, AGENTIC_STRATEGIES as O, STRATEGY_COLLECTION_MAPPINGS as P, HARM_PLUGINS as Q, isMultiTurnStrategy as R, PluginConfigSchema as S, isValidReusablePolicyId as T, subCategoryDescriptions as U, pluginDescriptions as V, ALIASED_PLUGIN_MAPPINGS as W, FINANCIAL_PLUGINS as X, DEFAULT_PLUGINS as Y, FOUNDATION_PLUGINS as Z, PromptSchema as _, EvaluateOptionsSchema as a, PHARMACY_PLUGINS as at, ConversationMessageSchema as b, TestSuiteConfigSchema as c, REDTEAM_MODEL as ct, isGradingResult as d, STRATEGY_EXEMPT_PLUGINS as dt, LLAMA_GUARD_ENABLED_CATEGORIES as et, isResultFailureReason as f, TELECOM_PLUGINS as ft, RedteamGenerateOptionsSchema as g, RedteamConfigSchema as h, EvalResultsFilterMode as i, MULTI_INPUT_VAR as it, DEFAULT_STRATEGIES as j, ALL_STRATEGIES as k, TestSuiteSchema as l, REDTEAM_PROVIDER_HARM_PLUGINS as lt, isProviderOptions as m, BaseAssertionTypesSchema as n, MEDICAL_PLUGINS as nt, OutputFileExtension as o, PII_PLUGINS as ot, isApiProvider as p, UNALIGNED_PROVIDER_HARM_PLUGINS as pt, BIAS_PLUGINS as q, CommandLineOptionsSchema as r, MULTI_INPUT_EXCLUDED_PLUGINS as rt, ResultFailureReason as s, PLUGIN_CATEGORIES as st, AssertionOrSetSchema as t, LLAMA_GUARD_REPLICATE_PROVIDER as tt, UnifiedConfigSchema as u, REMOTE_ONLY_PLUGIN_IDS as ut, ProviderOptionsSchema as v, StrategyConfigSchema as w, PartialGenerationError as x, ProvidersSchema as y, Severity as z };
|
|
3447
|
+
|
|
3448
|
+
//# sourceMappingURL=types-CH3Ge2sE.js.map
|