promptfoo 0.120.27 → 0.121.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/{ListApp-8WOe2nT6.js → ListApp-Du7YVwj5.js} +2 -4
- package/dist/src/accounts-B0pgC1oV.js +206 -0
- package/dist/src/{accounts-DVINui-2.js → accounts-Bm2D8Db9.js} +39 -34
- package/dist/src/{accounts-CPDRAMND.js → accounts-CiBLOnA7.js} +38 -33
- package/dist/src/{accounts-Fl2J3_Fu.cjs → accounts-gtkH-5KX.cjs} +77 -78
- package/dist/src/{agentic-utils-D922n6mm.js → agentic-utils-DS1g3GLF.js} +9 -10
- package/dist/src/{agents-BcsN_BgB.js → agents-9qiOy0ho.js} +16 -12
- package/dist/src/{agents-BXLmVsxR.js → agents-CBr9A01V.js} +37 -37
- package/dist/src/{agents-pMfppv9Z.js → agents-CmvBq8LV.js} +16 -18
- package/dist/src/{agents-hqgSV-3o.js → agents-D__IdAlg.js} +39 -40
- package/dist/src/{agents-BO2n8Z0d.cjs → agents-DbRtpYxR.cjs} +37 -40
- package/dist/src/{agents-BdUTAwi-.js → agents-DgF2zDag.js} +37 -42
- package/dist/src/{agents-DgJf2-ez.cjs → agents-Di9DKPzn.cjs} +16 -17
- package/dist/src/{agents-DNvSH78i.js → agents-cLXA8a_8.js} +17 -19
- package/dist/src/{aimlapi-DtgPI0nE.js → aimlapi-B4rcnZgv.js} +15 -17
- package/dist/src/{aimlapi-BE_Tg9Fl.cjs → aimlapi-BvlNH0gr.cjs} +15 -16
- package/dist/src/{aimlapi-DOib86oE.js → aimlapi-CnkC2HqE.js} +16 -18
- package/dist/src/{aimlapi-DTPACCB1.js → aimlapi-DHJU_kcV.js} +15 -4
- package/dist/src/app/assets/index-4LKxG2CG.js +439 -0
- package/dist/src/app/assets/{index-NCn4eVBv.css → index-C3zcsZFQ.css} +1 -1
- package/dist/src/app/assets/vendor-charts-BnDWwBlI.js +36 -0
- package/dist/src/app/index.html +3 -3
- package/dist/src/app/tsconfig.app.tsbuildinfo +1 -1
- package/dist/src/{audio-BnRUGAm_.js → audio-Bkv46et0.js} +6 -5
- package/dist/src/{audio-Cwo68yZS.cjs → audio-CGMyULza.cjs} +6 -7
- package/dist/src/{audio-MSRki4JU.js → audio-ClI_AFre.js} +6 -8
- package/dist/src/{audio-BRYU0BFo.js → audio-Dz3z7s3J.js} +7 -9
- package/dist/src/{base-pGVmXNl4.cjs → base-CGrhspbK.cjs} +36 -38
- package/dist/src/{base-h961VXYk.js → base-CpjcHe4e.js} +11 -13
- package/dist/src/base-DLKtKMFh.js +193 -0
- package/dist/src/{base-XB2tDJrB.js → base-Dy1V8--Z.js} +11 -13
- package/dist/src/blobs-BDbfYdrJ.js +236 -0
- package/dist/src/{blobs-CR5C4Ihh.js → blobs-CBO20krR.js} +9 -12
- package/dist/src/{blobs-BM_e6hCa.js → blobs-CMHN0Qcz.js} +9 -12
- package/dist/src/{blobs-B-KQAFhX.cjs → blobs-D23XLin-.cjs} +34 -37
- package/dist/src/{cache-jsiwsAJv.js → cache-BVeDlD87.js} +132 -117
- package/dist/src/{cache-CIpsoBZR.js → cache-C4Nxf52C.js} +132 -118
- package/dist/src/cache-CeUpFm3M.cjs +5 -0
- package/dist/src/{cache-BTVYfbka.cjs → cache-Dh5WtQps.cjs} +182 -168
- package/dist/src/cache-i1P6crbO.js +756 -0
- package/dist/src/cache-n-RCJ-hL.js +6 -0
- package/dist/src/{chat-BcPjZXIp.js → chat-BiKyneZl.js} +45 -46
- package/dist/src/{chat-D31K7C4u.cjs → chat-C1Qst7jL.cjs} +20 -21
- package/dist/src/{chat-B84t99NW.js → chat-C2jrdPMx.js} +20 -9
- package/dist/src/{chat-BE44YOc6.cjs → chat-CgF-J-Jj.cjs} +65 -66
- package/dist/src/{chat-DwWifjxi.js → chat-CzkrVDfz.js} +20 -22
- package/dist/src/chat-DJIw17u0.js +766 -0
- package/dist/src/{chat-CcUCysjU.js → chat-DqxYYtWA.js} +45 -46
- package/dist/src/{chat-DZM2GUHO.js → chat-qmatte1u.js} +21 -23
- package/dist/src/{chatkit-D67HS_0b.js → chatkit-65VXf5SR.js} +58 -58
- package/dist/src/{chatkit-DAB_qfzI.js → chatkit-Be-Q-a9F.js} +58 -60
- package/dist/src/{chatkit-Biqb_wsD.js → chatkit-BxFvW8KY.js} +58 -60
- package/dist/src/{chatkit-PGG4ZYIn.cjs → chatkit-DKyPi1Gs.cjs} +58 -60
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/chunk-DRamLcfz.js +16 -0
- package/dist/src/{claude-agent-sdk-SVM6AdBu.js → claude-agent-sdk-Apiy0iaz.js} +31 -31
- package/dist/src/{claude-agent-sdk-C-IOTPfo.js → claude-agent-sdk-D2bJee9S.js} +31 -29
- package/dist/src/{claude-agent-sdk-C9SiaQub.cjs → claude-agent-sdk-D9Z5Pr9X.cjs} +31 -28
- package/dist/src/{claude-agent-sdk-CiluSyW1.js → claude-agent-sdk-DfCoW0E6.js} +33 -20
- package/dist/src/cloud-BBh91EUK.js +4 -0
- package/dist/src/{cloud-CZ-q9Ier.js → cloud-C0dlstV_.js} +7 -9
- package/dist/src/{cloudflare-ai-BahKHyhh.js → cloudflare-ai-8TDxHR0x.js} +16 -18
- package/dist/src/{cloudflare-ai-v_qZD6_q.js → cloudflare-ai-BxAGvfju.js} +17 -19
- package/dist/src/{cloudflare-ai-Dfahv5SY.cjs → cloudflare-ai-CknbZ5LJ.cjs} +16 -17
- package/dist/src/{cloudflare-ai-Dxyt50Nl.js → cloudflare-ai-g7PB6VHR.js} +16 -4
- package/dist/src/{cloudflare-gateway-Bi_FpOFy.js → cloudflare-gateway-B9HWA5wf.js} +23 -23
- package/dist/src/{cloudflare-gateway-BPWoZIzJ.cjs → cloudflare-gateway-BSnDmHYo.cjs} +21 -22
- package/dist/src/{cloudflare-gateway-C0guUNwk.js → cloudflare-gateway-CKDb4dJ8.js} +26 -14
- package/dist/src/{cloudflare-gateway-btS7h1OZ.js → cloudflare-gateway-CP9QEWYS.js} +21 -25
- package/dist/src/{codex-sdk-DSxAnbfT.js → codex-sdk-C6UMlxwV.js} +28 -29
- package/dist/src/{codex-sdk-IYVi9fuM.js → codex-sdk-DUwKWezN.js} +28 -27
- package/dist/src/{codex-sdk-DulY0ZRq.js → codex-sdk-GGAw0qbD.js} +28 -29
- package/dist/src/{codex-sdk-DFKMtAyf.cjs → codex-sdk-fAO0c3yA.cjs} +28 -29
- package/dist/src/{cometapi-DzrR3SR_.js → cometapi-BL9yvj_f.js} +16 -4
- package/dist/src/{cometapi-DIO64tf4.cjs → cometapi-C4xSqeID.cjs} +21 -22
- package/dist/src/{cometapi-C9EEpJzT.js → cometapi-CUQq3H_a.js} +21 -24
- package/dist/src/{cometapi-DkNBMk0G.js → cometapi-DFNiKmSz.js} +17 -19
- package/dist/src/{completion-CG29bfKX.js → completion-5MzrpJxT.js} +11 -13
- package/dist/src/{completion-CCRT4kX1.cjs → completion-CM6oK8PS.cjs} +21 -23
- package/dist/src/{completion-Bgf1VJoq.js → completion-DZ083F31.js} +11 -13
- package/dist/src/completion-qRoZAYRB.js +120 -0
- package/dist/src/{createHash-Dw_iLu31.js → createHash-CTQmL3G2.js} +2 -3
- package/dist/src/{createHash-CYQy4YeL.cjs → createHash-CfZSc0b4.cjs} +13 -14
- package/dist/src/{createHash-CJcfskIZ.js → createHash-Da8fMwqB.js} +2 -3
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/{docker-D-ayp2FW.js → docker-Bb5dcxr8.js} +18 -20
- package/dist/src/{docker-B81N0t4e.js → docker-BvfL2BrW.js} +19 -21
- package/dist/src/{docker-DNcLR4Ig.cjs → docker-DcF2pRrj.cjs} +18 -19
- package/dist/src/{docker-egERKxCF.js → docker-ExVyLp0S.js} +18 -7
- package/dist/src/entrypoint.js +2 -3
- package/dist/src/{errors-DnGCbnx8.js → errors-P6ll7XSJ.js} +2 -2
- package/dist/src/{esm-B9dPm_BF.js → esm-C03C-mv3.js} +17 -20
- package/dist/src/{esm-D2pZ87fL.js → esm-CaIwzWR5.js} +18 -21
- package/dist/src/esm-Cd1AjG1D.js +379 -0
- package/dist/src/{esm-Ct-Joyue.cjs → esm-CnNt7sI4.cjs} +47 -49
- package/dist/src/eval-B3r2CVXr.js +15 -0
- package/dist/src/{eval-C-Nr6wX_.js → eval-Dg2nG4v2.js} +47 -54
- package/dist/src/evalResult-5xwYnECe.js +12 -0
- package/dist/src/evalResult-71lY93Kj.cjs +10 -0
- package/dist/src/{evalResult-DXMWJ3sx.js → evalResult-BBRNtX4I.js} +10 -11
- package/dist/src/{evalResult-4BzI2tmj.js → evalResult-BDMqrapS.js} +16 -12
- package/dist/src/evalResult-Dx5P5cIv.js +10 -0
- package/dist/src/{evalResult-CX8wQecI.cjs → evalResult-fuaI8HkH.cjs} +20 -21
- package/dist/src/{evaluator-8aGyV12L.js → evaluator-BhoWwp5b.js} +211 -235
- package/dist/src/evaluator-Jx6bRZV6.js +36 -0
- package/dist/src/{extractor-V5x_m1i0.js → extractor-C0EVHewb.js} +22 -24
- package/dist/src/extractor-D25qpmGX.js +374 -0
- package/dist/src/{extractor-CD5yKL-G.js → extractor-DReVID0K.js} +22 -24
- package/dist/src/{extractor-C031XmTA.cjs → extractor-pYLLi3wS.cjs} +37 -39
- package/dist/src/{fetch-BmbD-v1L.cjs → fetch-BPkYtG8K.cjs} +244 -277
- package/dist/src/fetch-BxNb_Lp3.js +5 -0
- package/dist/src/{fetch-D3OHf-lV.js → fetch-Cwxnd8zz.js} +36 -44
- package/dist/src/{fetch-CXZI9RRr.js → fetch-Dxpd4_sr.js} +23 -35
- package/dist/src/fetch-HaqdX7U1.js +780 -0
- package/dist/src/{fileExtensions-ePDqouxn.js → fileExtensions-DnqA1y9x.js} +2 -2
- package/dist/src/{fileExtensions-BpuMmaFL.js → fileExtensions-Ds-foDzt.js} +2 -2
- package/dist/src/fileExtensions-LcDYkU4v.js +85 -0
- package/dist/src/{fileExtensions-DkJYkWUy.cjs → fileExtensions-bYh77CN8.cjs} +27 -28
- package/dist/src/{formatDuration-CdevI3An.js → formatDuration-DgBVMN65.js} +2 -2
- package/dist/src/{genaiTracer-Ce19n68P.js → genaiTracer-70Z8BIuV.js} +2 -3
- package/dist/src/{genaiTracer-CqNnnXrE.js → genaiTracer-C1rxGO8Q.js} +2 -3
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/{genaiTracer-Dres3qrN.cjs → genaiTracer-DN4dQywX.cjs} +13 -14
- package/dist/src/{graders--1y2u9HO.js → graders-BTeBGqjJ.js} +349 -397
- package/dist/src/graders-B_pgMLS2.js +34 -0
- package/dist/src/{graders-DTeBrzWp.js → graders-Bj_Odv7c.js} +349 -397
- package/dist/src/graders-DErokPDO.cjs +32 -0
- package/dist/src/graders-DP7KFFo-.js +13466 -0
- package/dist/src/graders-DR_uNe54.js +32 -0
- package/dist/src/{graders-DohM2dir.cjs → graders-DU49_J8Y.cjs} +684 -732
- package/dist/src/graders-w3176Wz-.js +32 -0
- package/dist/src/{image-B0U4Hqll.js → image-B02ogr_b.js} +7 -9
- package/dist/src/{image-DmE-niFE.js → image-B0h9VEMc.js} +6 -5
- package/dist/src/{image-CuKHuccK.cjs → image-BLmROtN3.cjs} +29 -30
- package/dist/src/{image-DNEIf_aI.js → image-Bb4vWQLM.js} +6 -8
- package/dist/src/{image-DpKl2F15.cjs → image-C1madmKh.cjs} +6 -7
- package/dist/src/{image-C3wHC9_h.js → image-CHfWvljl.js} +9 -10
- package/dist/src/{image-O1u4bCFg.js → image-DS-o-0ph.js} +9 -10
- package/dist/src/image-Dpxa1Jt6.js +257 -0
- package/dist/src/index.cjs +615 -695
- package/dist/src/index.d.cts +271 -7
- package/dist/src/index.d.ts +271 -3
- package/dist/src/index.js +580 -664
- package/dist/src/{interactiveCheck-Bxj1Swex.js → interactiveCheck-BgLZUIt3.js} +7 -8
- package/dist/src/{invariant-DT20jrBd.js → invariant-BtWWVVhl.js} +2 -2
- package/dist/src/{invariant-1pAf2CD1.js → invariant-Ddh24eXh.js} +2 -2
- package/dist/src/{invariant-CKcJAQ6M.cjs → invariant-kfQ8Bu82.cjs} +7 -8
- package/dist/src/invariant-vgHWClmd.js +25 -0
- package/dist/src/{knowledgeBase-CEzQobWX.js → knowledgeBase-B3OoKIej.js} +14 -9
- package/dist/src/{knowledgeBase-Be_zyW4L.js → knowledgeBase-CYTLHOt1.js} +16 -16
- package/dist/src/{knowledgeBase-BZ41IFwq.js → knowledgeBase-D33Ty2l6.js} +14 -18
- package/dist/src/{knowledgeBase-D-5BMXlr.cjs → knowledgeBase-DOO_BM9b.cjs} +14 -15
- package/dist/src/{litellm-DnbRJ2if.js → litellm-AaeZcZQF.js} +18 -19
- package/dist/src/{litellm-hUSNM_M2.cjs → litellm-I_hbp_dc.cjs} +17 -17
- package/dist/src/{litellm-CRDqPhNI.js → litellm-NbjknEh6.js} +17 -18
- package/dist/src/{litellm-9vR8zpfU.js → litellm-TrljxD9G.js} +17 -5
- package/dist/src/{logger-CG1uZPbQ.js → logger-CT3IKMKA.js} +10 -29
- package/dist/src/{logger-B7sBeGa0.cjs → logger-Cp1GPUjj.cjs} +152 -180
- package/dist/src/logger-DLcq4dWf.js +713 -0
- package/dist/src/{logger-LSBxlt7a.js → logger-KkObSCzq.js} +13 -31
- package/dist/src/{luma-ray-4blv9iZ2.js → luma-ray-BS2_tY8L.js} +22 -21
- package/dist/src/{luma-ray-drvgdpP9.js → luma-ray-DDsjcgZZ.js} +20 -13
- package/dist/src/{luma-ray-Hm3d6VJE.cjs → luma-ray-Due0n7di.cjs} +20 -21
- package/dist/src/{luma-ray-B2__8lYH.js → luma-ray-f6I2fft-.js} +20 -23
- package/dist/src/main.js +1170 -1321
- package/dist/src/{messages-Uee41Mj5.js → messages-BS17jdMx.js} +22 -24
- package/dist/src/{messages-XhiwCbi4.cjs → messages-Bs1kC7P4.cjs} +32 -34
- package/dist/src/{messages-CGPPidQr.js → messages-D0lx5qK7.js} +22 -24
- package/dist/src/messages-ZJk778GH.js +240 -0
- package/dist/src/{meteor-BYykdXrV.js → meteor-44VjEACX.js} +3 -4
- package/dist/src/{meteor-CsopaHrH.js → meteor-D-SotUw9.js} +3 -4
- package/dist/src/{meteor-e-E-2vVl.cjs → meteor-DLZZ3osF.cjs} +3 -4
- package/dist/src/{meteor-C8lGP6P4.js → meteor-DUiCJRC-.js} +3 -4
- package/dist/src/{modelslab-yKz-ZNB4.js → modelslab-Bmni6skY.js} +17 -10
- package/dist/src/{modelslab-E9gO-bYd.js → modelslab-Bx9IrZfS.js} +18 -20
- package/dist/src/{modelslab-lUVW0cmB.cjs → modelslab-CoUX6Jc_.cjs} +17 -18
- package/dist/src/{modelslab-ClBkr8_9.js → modelslab-DRb74SP4.js} +17 -19
- package/dist/src/{nova-reel-Dk8jNpId.js → nova-reel-BfPq-0Yk.js} +20 -13
- package/dist/src/{nova-reel-D8CuO6QH.cjs → nova-reel-C_QM18Xn.cjs} +20 -21
- package/dist/src/{nova-reel-u2eF2Cxm.js → nova-reel-D_W1tjMH.js} +22 -21
- package/dist/src/{nova-reel-P9bwvtYX.js → nova-reel-bgjxilYW.js} +20 -23
- package/dist/src/{nova-sonic-CK2rAiKi.js → nova-sonic-CFb5GYhg.js} +30 -26
- package/dist/src/{nova-sonic-BaqWlkds.js → nova-sonic-DIGQNR07.js} +30 -31
- package/dist/src/{nova-sonic-yZapPLv7.js → nova-sonic-De1HW5fD.js} +31 -32
- package/dist/src/{nova-sonic-Ds1C-dpm.cjs → nova-sonic-zfcljeRp.cjs} +30 -31
- package/dist/src/{openai-DUFopMrH.cjs → openai-Cuif0GEt.cjs} +8 -9
- package/dist/src/{openai-PblZ3jUE.js → openai-DElQ-fPX.js} +3 -4
- package/dist/src/{openai-CcN1B8Sb.js → openai-DhbB7eWK.js} +3 -4
- package/dist/src/openai-j-sE2O7r.js +44 -0
- package/dist/src/{openclaw-B6qqDr_u.cjs → openclaw-CSugPYAr.cjs} +188 -130
- package/dist/src/{openclaw-A-3_loM7.js → openclaw-DiSz3I5L.js} +180 -109
- package/dist/src/{openclaw-a3lylB-V.js → openclaw-DuvJKEW5.js} +178 -124
- package/dist/src/{openclaw-COn6QzDi.js → openclaw-tiVYRtr-.js} +178 -122
- package/dist/src/opencode-sdk-0j6rTWNb.js +562 -0
- package/dist/src/opencode-sdk-B3CWY9h_.js +560 -0
- package/dist/src/opencode-sdk-BL764Jdi.cjs +564 -0
- package/dist/src/opencode-sdk-C2y6UkP2.js +560 -0
- package/dist/src/{otlpReceiver-oyf5wLGC.js → otlpReceiver-C99PPb48.js} +53 -51
- package/dist/src/{otlpReceiver-lXsYVbpj.cjs → otlpReceiver-CGq6LspY.cjs} +53 -55
- package/dist/src/{otlpReceiver-94URx7UW.js → otlpReceiver-CdNBdbsk.js} +53 -55
- package/dist/src/{otlpReceiver-BmmTiMjA.js → otlpReceiver-D89fR-rC.js} +53 -55
- package/dist/src/{providerRegistry-Cq_JK_CJ.js → providerRegistry-B0RUOLI_.js} +7 -8
- package/dist/src/{providerRegistry-DSSHjMKf.js → providerRegistry-CD8MEar9.js} +7 -8
- package/dist/src/{providerRegistry-CvHEVJad.cjs → providerRegistry-Civky8Ar.cjs} +12 -13
- package/dist/src/providerRegistry-DM8rZYol.js +45 -0
- package/dist/src/providers-B7V0njNs.js +32 -0
- package/dist/src/providers-BEwbhv0X.js +30 -0
- package/dist/src/{providers-Iil64vk9.js → providers-BlqUifFg.js} +1543 -1676
- package/dist/src/providers-CH3C7zf7.js +30 -0
- package/dist/src/{providers-DHbjzW2e.cjs → providers-CgKOSgTR.cjs} +1896 -2029
- package/dist/src/providers-D8lF1sqW.js +33246 -0
- package/dist/src/{providers-BnFpbY_s.js → providers-Dk_6ocUX.js} +1536 -1669
- package/dist/src/providers-zyB6k_38.cjs +31 -0
- package/dist/src/{pythonUtils-CcT5LH1M.js → pythonUtils-C3py6GC1.js} +18 -19
- package/dist/src/{pythonUtils-DBbuI3QJ.cjs → pythonUtils-CTU3Y3lw.cjs} +42 -43
- package/dist/src/{pythonUtils-hZ8LeQLv.js → pythonUtils-D5nxkQ0P.js} +18 -19
- package/dist/src/pythonUtils-D6fwaDSg.js +249 -0
- package/dist/src/{quiverai-BuI0tE39.js → quiverai-BbOUOn2L.js} +8 -7
- package/dist/src/{quiverai-DCGSZt4U.js → quiverai-CIaELU_m.js} +8 -10
- package/dist/src/{quiverai-DiMVJQDz.cjs → quiverai-PdShCPox.cjs} +8 -9
- package/dist/src/{quiverai-fQNkExW4.js → quiverai-uH-dcTIr.js} +9 -11
- package/dist/src/{render-Dj1smHEb.js → render-Drod8m7K.js} +4 -5
- package/dist/src/responses-CB2jwoAr.js +660 -0
- package/dist/src/{responses-ghR3IOfy.cjs → responses-D8SBTL64.cjs} +39 -42
- package/dist/src/{responses-DOAFFENS.js → responses-DIR9Ud3j.js} +24 -27
- package/dist/src/{responses-CxzoQoBe.js → responses-WNGNYe3K.js} +24 -27
- package/dist/src/rubyUtils-BUHu6PhO.js +5 -0
- package/dist/src/{rubyUtils-CwbGmgYN.js → rubyUtils-BUVePouc.js} +27 -20
- package/dist/src/rubyUtils-BcuGX77l.js +222 -0
- package/dist/src/{rubyUtils-DudlFZed.js → rubyUtils-Boc4HZzX.js} +18 -19
- package/dist/src/rubyUtils-CP42kMvq.cjs +4 -0
- package/dist/src/{rubyUtils-C8MhKGHb.cjs → rubyUtils-DhCAlxZr.cjs} +48 -50
- package/dist/src/{sagemaker-gmskuyre.js → sagemaker-CNBxx5CJ.js} +75 -70
- package/dist/src/{sagemaker-CcxhlOAR.js → sagemaker-CemTFp2h.js} +75 -79
- package/dist/src/{sagemaker-77zbJ2Q2.cjs → sagemaker-Cl28mZU2.cjs} +75 -76
- package/dist/src/{sagemaker-DuM71dVU.js → sagemaker-YSyBXQQh.js} +77 -77
- package/dist/src/{scanner-DJYiSXQj.js → scanner-BsBlNXNn.js} +100 -121
- package/dist/src/server/index.js +5520 -67427
- package/dist/src/{server-B5v33lvE.cjs → server-C_7Ax-hA.cjs} +57 -67
- package/dist/src/{server-BJ4m4f1D.js → server-CqzrVGpF.js} +26 -29
- package/dist/src/server-CuxBbeSY.js +229 -0
- package/dist/src/server-DA4Cyrrq.js +7 -0
- package/dist/src/server-Dulb-4-K.cjs +5 -0
- package/dist/src/{server-RV_i_YX5.js → server-VWgWb00X.js} +19 -24
- package/dist/src/{signal-BW33JuId.js → signal-4U3mfRvL.js} +9 -11
- package/dist/src/{slack-DEURelTy.cjs → slack-BmVAVGaK.cjs} +7 -8
- package/dist/src/{slack-BQYeW9L3.js → slack-DCUPTzS2.js} +8 -8
- package/dist/src/{slack-BB6yuZzp.js → slack-DOdy_kyv.js} +7 -8
- package/dist/src/{slack-2pRrhhgJ.js → slack-DXMKtA-f.js} +7 -9
- package/dist/src/store-CXGFv4aR.js +228 -0
- package/dist/src/store-CXS-Q_91.js +6 -0
- package/dist/src/{store-D7CgQzAR.cjs → store-DLlFCC4h.cjs} +44 -45
- package/dist/src/{store-DJNsD1iC.js → store-DXilxTl-.js} +40 -36
- package/dist/src/{store-s3SftUwF.js → store-Dim__MDd.js} +34 -35
- package/dist/src/store-eYkaKMwq.cjs +5 -0
- package/dist/src/{tables-DfTsNN7X.js → tables-6YKwjN9-.js} +19 -21
- package/dist/src/tables-DLJPUdUE.js +288 -0
- package/dist/src/{tables-BKTmd6u7.cjs → tables-DPi7wKeM.cjs} +89 -91
- package/dist/src/{tables-DMegD0Xf.js → tables-gftXzE9I.js} +21 -23
- package/dist/src/telemetry-BpMfhthR.cjs +5 -0
- package/dist/src/{telemetry--WAdAfVi.js → telemetry-CMrFgtPB.js} +11 -13
- package/dist/src/telemetry-Cps3mIU-.js +171 -0
- package/dist/src/{telemetry-DQgVBCAb.cjs → telemetry-DaX14Chu.cjs} +21 -24
- package/dist/src/{telemetry-BedSm-bZ.js → telemetry-Dthj_BbD.js} +17 -14
- package/dist/src/telemetry-Dw38hanS.js +7 -0
- package/dist/src/{text-oiSbwSOI.js → text-B_UCRPp2.js} +2 -2
- package/dist/src/{text-oKzCBnK6.cjs → text-CW1cyrwj.cjs} +12 -13
- package/dist/src/{text-B_IrO4GZ.js → text-Db-Wt2u2.js} +2 -2
- package/dist/src/text-TIv0QYnd.js +22 -0
- package/dist/src/{tokenUsageUtils-FZd5O_4A.js → tokenUsageUtils-BDGe-iyI.js} +2 -2
- package/dist/src/{tokenUsageUtils-DmZSD2eU.js → tokenUsageUtils-DflFMjS0.js} +2 -2
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/{tokenUsageUtils-CXhxVj72.cjs → tokenUsageUtils-bVa1ga6f.cjs} +32 -33
- package/dist/src/{transcription-mYS9vd5v.js → transcription-BNYURcXg.js} +14 -7
- package/dist/src/{transcription-X2-B4vkX.js → transcription-B_OdaHp7.js} +14 -16
- package/dist/src/{transcription-BO1AHegO.cjs → transcription-NLVG9MT1.cjs} +14 -15
- package/dist/src/{transcription-lzBLiTFJ.js → transcription-s6A-bNrZ.js} +15 -17
- package/dist/src/{transform-B1Hi5lWS.cjs → transform-CzK1Q0zl.cjs} +24 -26
- package/dist/src/{transform-DeGlxb0D.js → transform-D5HsjduX.js} +39 -47
- package/dist/src/{transform-CYDILYDe.js → transform-DECvGmzp.js} +15 -13
- package/dist/src/transform-DTGDnAzW.js +6 -0
- package/dist/src/{transform-BEgStbHK.js → transform-DilY9wbS.js} +10 -12
- package/dist/src/{transform-D5PjiWiZ.cjs → transform-DuHvhZpj.cjs} +179 -187
- package/dist/src/transform-aa6tmVpZ.js +216 -0
- package/dist/src/transform-m3qNw4KP.cjs +5 -0
- package/dist/src/transform-uAytVuyX.js +1506 -0
- package/dist/src/{transform-Dfl89yi4.js → transform-vNucnNr0.js} +39 -47
- package/dist/src/{transformersAvailability-SZnTS3pJ.js → transformersAvailability-CEVM2GNQ.js} +2 -2
- package/dist/src/{transformersAvailability-D-glmEy7.cjs → transformersAvailability-CwayUSlh.cjs} +2 -3
- package/dist/src/{transformersAvailability-CjeFXhuJ.js → transformersAvailability-D6c6ROpT.js} +2 -2
- package/dist/src/{types-DWNf48sT.cjs → types-C_7nyzr1.cjs} +538 -574
- package/dist/src/{types-CXQduE9o.js → types-Cbd8uOMq.js} +68 -100
- package/dist/src/types-CzW2QFyi.js +3288 -0
- package/dist/src/{types-C5hEkb-x.js → types-DmyIJ-sR.js} +63 -99
- package/dist/src/{util-CoQjmE3u.js → util-B3xGByQh.js} +4 -5
- package/dist/src/{util-aLhtl3fe.cjs → util-B9vlHIIh.cjs} +208 -223
- package/dist/src/{util-Du96oyYS.js → util-BHGHw5G1.js} +4 -5
- package/dist/src/{util-DQ984syk.js → util-BRYkYPTd.js} +36 -51
- package/dist/src/{util-D9eLdGfa.js → util-BV4XUC0n.js} +5 -6
- package/dist/src/util-Bv6uGDfH.js +293 -0
- package/dist/src/{util-1wWM599Z.cjs → util-BzMcevZc.cjs} +50 -51
- package/dist/src/{util-_h4pVqrz.js → util-C1CeHl-P.js} +36 -51
- package/dist/src/{util-Bm_-UMD_.js → util-CMy69ZgQ.js} +5 -6
- package/dist/src/{util-CyUdMzV0.cjs → util-DGNOS1db.cjs} +34 -35
- package/dist/src/util-Dnmk2mBQ.js +599 -0
- package/dist/src/util-ZzmqNPlg.js +1426 -0
- package/dist/src/{utils-BjLy-Q72.cjs → utils-Cz9qXqII.cjs} +29 -32
- package/dist/src/{utils-CFMn2yHW.js → utils-XiOAgly5.js} +4 -7
- package/dist/src/utils-dLokC-eR.js +94 -0
- package/dist/src/{utils-DvWMzuMx.js → utils-f2-Moju7.js} +4 -7
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +38 -38
- package/dist/src/app/assets/index-B2D0bCSI.js +0 -439
- package/dist/src/app/assets/vendor-charts-CCl15Imd.js +0 -36
- package/dist/src/cache-ChPcurj7.js +0 -6
- package/dist/src/cache-VVu_W-yg.js +0 -8
- package/dist/src/cache-YLNCFEM2.cjs +0 -6
- package/dist/src/chunk-DHDDz29n.js +0 -22
- package/dist/src/chunk-FhC4c-0y.js +0 -21
- package/dist/src/cloud-BndfXy4H.js +0 -5
- package/dist/src/eval-BhHvMY82.js +0 -17
- package/dist/src/evalResult-Dq2gFNQY.js +0 -12
- package/dist/src/evalResult-nmcP5VKH.cjs +0 -12
- package/dist/src/evalResult-trqZjVYh.js +0 -14
- package/dist/src/evaluator-CnfPstzT.js +0 -39
- package/dist/src/fetch-IDPDue6F.cjs +0 -4
- package/dist/src/fetch-hKJ-It8q.js +0 -6
- package/dist/src/fetch-ouKnrWK-.js +0 -4
- package/dist/src/graders-CQn7WUsd.cjs +0 -34
- package/dist/src/graders-DC6QAbpW.js +0 -35
- package/dist/src/graders-DUWz3Y7j.js +0 -37
- package/dist/src/opencode-sdk-4bL9n-Gk.js +0 -382
- package/dist/src/opencode-sdk-BfC2zWcR.js +0 -376
- package/dist/src/opencode-sdk-DMJyuwMg.js +0 -380
- package/dist/src/opencode-sdk-Da-9adza.cjs +0 -383
- package/dist/src/providers-CsXB2Ix-.js +0 -35
- package/dist/src/providers-DO8ltjLC.js +0 -33
- package/dist/src/providers-Dtq-xnXd.cjs +0 -33
- package/dist/src/rubyUtils-BUbcND2f.js +0 -6
- package/dist/src/rubyUtils-Cr55X_KE.js +0 -5
- package/dist/src/rubyUtils-DlIiqoYo.cjs +0 -5
- package/dist/src/server-C2eQH4Gu.js +0 -6
- package/dist/src/server-CXWycu7H.cjs +0 -6
- package/dist/src/server-Q6OGlxxT.js +0 -8
- package/dist/src/store-B3EDO9Q3.js +0 -7
- package/dist/src/store-Dl9F8aw5.js +0 -6
- package/dist/src/store-SnrGrlt9.cjs +0 -6
- package/dist/src/telemetry-BGhiPZtl.js +0 -8
- package/dist/src/telemetry-CFfiYan6.cjs +0 -6
- package/dist/src/telemetry-DHzEduxX.js +0 -6
- package/dist/src/transform-C1x1ZlMQ.cjs +0 -6
- package/dist/src/transform-DYHjFmQu.js +0 -8
- package/dist/src/transform-rmwJT5JQ.js +0 -7
- package/dist/src/transformersAvailability-eJooj0gX.js +0 -35
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-
|
|
2
|
+
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-Ds-foDzt.js";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import dedent from "dedent";
|
|
5
|
-
|
|
6
5
|
//#region src/types/shared.ts
|
|
7
6
|
const CompletionTokenDetailsSchema = z.object({
|
|
8
7
|
reasoning: z.number().optional(),
|
|
@@ -29,10 +28,8 @@ const BaseTokenUsageSchema = z.object({
|
|
|
29
28
|
}).optional()
|
|
30
29
|
});
|
|
31
30
|
const InputsSchema = z.record(z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" }), z.string().min(1, { error: "Input descriptions must be non-empty strings" }));
|
|
32
|
-
|
|
33
31
|
//#endregion
|
|
34
32
|
//#region src/redteam/constants/plugins.ts
|
|
35
|
-
const DEFAULT_NUM_TESTS_PER_PLUGIN = 5;
|
|
36
33
|
const MULTI_INPUT_VAR = "__prompt";
|
|
37
34
|
const REDTEAM_MODEL = "openai:chat:gpt-5-2025-08-07";
|
|
38
35
|
const LLAMA_GUARD_REPLICATE_PROVIDER = "replicate:moderation:meta/llama-guard-4-12b";
|
|
@@ -230,6 +227,7 @@ const PHARMACY_PLUGINS = [
|
|
|
230
227
|
];
|
|
231
228
|
const INSURANCE_PLUGINS = [
|
|
232
229
|
"insurance:coverage-discrimination",
|
|
230
|
+
"insurance:data-disclosure",
|
|
233
231
|
"insurance:network-misinformation",
|
|
234
232
|
"insurance:phi-disclosure"
|
|
235
233
|
];
|
|
@@ -315,6 +313,7 @@ const ADDITIONAL_PLUGINS = [
|
|
|
315
313
|
"ecommerce:price-manipulation",
|
|
316
314
|
"goal-misalignment",
|
|
317
315
|
"insurance:coverage-discrimination",
|
|
316
|
+
"insurance:data-disclosure",
|
|
318
317
|
"insurance:network-misinformation",
|
|
319
318
|
"insurance:phi-disclosure",
|
|
320
319
|
"off-topic",
|
|
@@ -392,7 +391,7 @@ const DEFAULT_PLUGINS = new Set([...[
|
|
|
392
391
|
...PII_PLUGINS,
|
|
393
392
|
...BIAS_PLUGINS
|
|
394
393
|
].sort()]);
|
|
395
|
-
|
|
394
|
+
new Set([
|
|
396
395
|
...DEFAULT_PLUGINS,
|
|
397
396
|
"bola",
|
|
398
397
|
"bfla",
|
|
@@ -452,12 +451,11 @@ const REMOTE_ONLY_PLUGIN_IDS = [
|
|
|
452
451
|
...TELECOM_PLUGINS,
|
|
453
452
|
...REALESTATE_PLUGINS
|
|
454
453
|
];
|
|
455
|
-
|
|
454
|
+
[
|
|
456
455
|
...Object.keys(UNALIGNED_PROVIDER_HARM_PLUGINS),
|
|
457
456
|
...BIAS_PLUGINS,
|
|
458
457
|
...REMOTE_ONLY_PLUGIN_IDS
|
|
459
458
|
];
|
|
460
|
-
|
|
461
459
|
//#endregion
|
|
462
460
|
//#region src/redteam/constants/frameworks.ts
|
|
463
461
|
const OWASP_LLM_TOP_10_MAPPING = {
|
|
@@ -1014,7 +1012,8 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1014
1012
|
plugins: [
|
|
1015
1013
|
"hijacking",
|
|
1016
1014
|
"intent",
|
|
1017
|
-
"excessive-agency"
|
|
1015
|
+
"excessive-agency",
|
|
1016
|
+
"harmful:misinformation-disinformation"
|
|
1018
1017
|
],
|
|
1019
1018
|
strategies: [
|
|
1020
1019
|
"jailbreak",
|
|
@@ -1027,12 +1026,21 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1027
1026
|
plugins: [
|
|
1028
1027
|
"hijacking",
|
|
1029
1028
|
"imitation",
|
|
1030
|
-
"harmful:misinformation-disinformation"
|
|
1029
|
+
"harmful:misinformation-disinformation",
|
|
1030
|
+
"bias:age",
|
|
1031
|
+
"bias:disability"
|
|
1031
1032
|
],
|
|
1032
|
-
strategies: []
|
|
1033
|
+
strategies: ["jailbreak"]
|
|
1033
1034
|
},
|
|
1034
1035
|
"eu:ai-act:art5:social-scoring": {
|
|
1035
|
-
plugins: [
|
|
1036
|
+
plugins: [
|
|
1037
|
+
"harmful:misinformation-disinformation",
|
|
1038
|
+
"overreliance",
|
|
1039
|
+
"bias:age",
|
|
1040
|
+
"bias:gender",
|
|
1041
|
+
"bias:race",
|
|
1042
|
+
"bias:disability"
|
|
1043
|
+
],
|
|
1036
1044
|
strategies: []
|
|
1037
1045
|
},
|
|
1038
1046
|
"eu:ai-act:art5:biometric-categorisation": {
|
|
@@ -1080,7 +1088,10 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1080
1088
|
plugins: [
|
|
1081
1089
|
"harmful:misinformation-disinformation",
|
|
1082
1090
|
"hallucination",
|
|
1083
|
-
"overreliance"
|
|
1091
|
+
"overreliance",
|
|
1092
|
+
"bias:race",
|
|
1093
|
+
"bias:gender",
|
|
1094
|
+
"bias:disability"
|
|
1084
1095
|
],
|
|
1085
1096
|
strategies: []
|
|
1086
1097
|
},
|
|
@@ -1088,7 +1099,11 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1088
1099
|
plugins: [
|
|
1089
1100
|
"imitation",
|
|
1090
1101
|
"pii:direct",
|
|
1091
|
-
"overreliance"
|
|
1102
|
+
"overreliance",
|
|
1103
|
+
"bias:gender",
|
|
1104
|
+
"bias:race",
|
|
1105
|
+
"bias:age",
|
|
1106
|
+
"bias:disability"
|
|
1092
1107
|
],
|
|
1093
1108
|
strategies: []
|
|
1094
1109
|
},
|
|
@@ -1096,7 +1111,9 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1096
1111
|
plugins: [
|
|
1097
1112
|
"pii:direct",
|
|
1098
1113
|
"pii:session",
|
|
1099
|
-
"excessive-agency"
|
|
1114
|
+
"excessive-agency",
|
|
1115
|
+
"bias:race",
|
|
1116
|
+
"bias:gender"
|
|
1100
1117
|
],
|
|
1101
1118
|
strategies: []
|
|
1102
1119
|
},
|
|
@@ -1104,7 +1121,8 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1104
1121
|
plugins: [
|
|
1105
1122
|
"pii:direct",
|
|
1106
1123
|
"pii:api-db",
|
|
1107
|
-
"harmful:privacy"
|
|
1124
|
+
"harmful:privacy",
|
|
1125
|
+
"bias:race"
|
|
1108
1126
|
],
|
|
1109
1127
|
strategies: []
|
|
1110
1128
|
},
|
|
@@ -1112,7 +1130,8 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1112
1130
|
plugins: [
|
|
1113
1131
|
"pii:direct",
|
|
1114
1132
|
"harmful:hate",
|
|
1115
|
-
"harmful:privacy"
|
|
1133
|
+
"harmful:privacy",
|
|
1134
|
+
"bias:race"
|
|
1116
1135
|
],
|
|
1117
1136
|
strategies: []
|
|
1118
1137
|
},
|
|
@@ -1120,7 +1139,9 @@ const EU_AI_ACT_MAPPING = {
|
|
|
1120
1139
|
plugins: [
|
|
1121
1140
|
"hallucination",
|
|
1122
1141
|
"harmful:misinformation-disinformation",
|
|
1123
|
-
"pii:direct"
|
|
1142
|
+
"pii:direct",
|
|
1143
|
+
"bias:race",
|
|
1144
|
+
"bias:gender"
|
|
1124
1145
|
],
|
|
1125
1146
|
strategies: []
|
|
1126
1147
|
}
|
|
@@ -1454,7 +1475,6 @@ const ALIASED_PLUGIN_MAPPINGS = {
|
|
|
1454
1475
|
strategies: []
|
|
1455
1476
|
} }
|
|
1456
1477
|
};
|
|
1457
|
-
|
|
1458
1478
|
//#endregion
|
|
1459
1479
|
//#region src/redteam/constants/metadata.ts
|
|
1460
1480
|
const subCategoryDescriptions = {
|
|
@@ -1532,6 +1552,7 @@ const subCategoryDescriptions = {
|
|
|
1532
1552
|
imitation: "Tests handling of unauthorized impersonation of entities",
|
|
1533
1553
|
"indirect-prompt-injection": "Tests for injection vulnerabilities via untrusted variables",
|
|
1534
1554
|
"insurance:coverage-discrimination": "Tests for discriminatory coverage decisions based on protected characteristics (age, disability, race, etc.)",
|
|
1555
|
+
"insurance:data-disclosure": "Tests for unauthorized disclosure of protected policyholder data including claims history, property details, driving records, and financial information",
|
|
1535
1556
|
"insurance:network-misinformation": "Tests for inaccurate provider network information",
|
|
1536
1557
|
"insurance:phi-disclosure": "Tests for unauthorized disclosure of Protected Health Information in violation of HIPAA privacy regulations",
|
|
1537
1558
|
"ecommerce:pci-dss": "Tests for exposure of payment card data including credit card numbers, CVVs, and bank account details in violation of PCI DSS requirements",
|
|
@@ -1614,7 +1635,7 @@ const subCategoryDescriptions = {
|
|
|
1614
1635
|
bias: "Bias detection across protected characteristics",
|
|
1615
1636
|
medical: "Medical AI safety testing across healthcare vulnerabilities",
|
|
1616
1637
|
pharmacy: "Pharmacy AI safety testing across pharmaceutical vulnerabilities",
|
|
1617
|
-
insurance: "Insurance AI safety testing across
|
|
1638
|
+
insurance: "Insurance AI safety testing across all insurance types",
|
|
1618
1639
|
financial: "Financial AI safety testing across financial services vulnerabilities",
|
|
1619
1640
|
ecommerce: "E-commerce AI safety testing across payment security and transaction integrity vulnerabilities",
|
|
1620
1641
|
telecom: "Telecommunications AI safety testing across CPNI protection, account security, and regulatory compliance vulnerabilities",
|
|
@@ -1654,20 +1675,8 @@ const SeveritySchema = z.enum([
|
|
|
1654
1675
|
"low",
|
|
1655
1676
|
"informational"
|
|
1656
1677
|
]);
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
[Severity.High]: "High",
|
|
1660
|
-
[Severity.Medium]: "Medium",
|
|
1661
|
-
[Severity.Low]: "Low",
|
|
1662
|
-
[Severity.Informational]: "Informational"
|
|
1663
|
-
};
|
|
1664
|
-
const severityRiskScores = {
|
|
1665
|
-
[Severity.Critical]: 9,
|
|
1666
|
-
[Severity.High]: 7,
|
|
1667
|
-
[Severity.Medium]: 4,
|
|
1668
|
-
[Severity.Low]: 0,
|
|
1669
|
-
[Severity.Informational]: 0
|
|
1670
|
-
};
|
|
1678
|
+
Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
|
|
1679
|
+
Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
|
|
1671
1680
|
const riskCategorySeverityMap = {
|
|
1672
1681
|
["agentic:memory-poisoning"]: Severity.High,
|
|
1673
1682
|
aegis: Severity.Medium,
|
|
@@ -1756,6 +1765,7 @@ const riskCategorySeverityMap = {
|
|
|
1756
1765
|
imitation: Severity.Low,
|
|
1757
1766
|
"indirect-prompt-injection": Severity.High,
|
|
1758
1767
|
"insurance:coverage-discrimination": Severity.Critical,
|
|
1768
|
+
"insurance:data-disclosure": Severity.Critical,
|
|
1759
1769
|
"insurance:network-misinformation": Severity.High,
|
|
1760
1770
|
"insurance:phi-disclosure": Severity.Critical,
|
|
1761
1771
|
"ecommerce:pci-dss": Severity.Critical,
|
|
@@ -1816,7 +1826,7 @@ const riskCategorySeverityMap = {
|
|
|
1816
1826
|
wordplay: Severity.Low,
|
|
1817
1827
|
xstest: Severity.Low
|
|
1818
1828
|
};
|
|
1819
|
-
|
|
1829
|
+
Object.entries({
|
|
1820
1830
|
"Security & Access Control": [
|
|
1821
1831
|
"ascii-smuggling",
|
|
1822
1832
|
"bfla",
|
|
@@ -1839,6 +1849,7 @@ const riskCategories = {
|
|
|
1839
1849
|
"data-exfil",
|
|
1840
1850
|
"divergent-repetition",
|
|
1841
1851
|
"harmful:privacy",
|
|
1852
|
+
"insurance:data-disclosure",
|
|
1842
1853
|
"insurance:phi-disclosure",
|
|
1843
1854
|
"pii:api-db",
|
|
1844
1855
|
"pii:direct",
|
|
@@ -1964,14 +1975,12 @@ const riskCategories = {
|
|
|
1964
1975
|
"vlsu",
|
|
1965
1976
|
"xstest"
|
|
1966
1977
|
]
|
|
1967
|
-
}
|
|
1968
|
-
const categoryMapReverse = Object.entries(riskCategories).reduce((acc, [category, harms]) => {
|
|
1978
|
+
}).reduce((acc, [category, harms]) => {
|
|
1969
1979
|
harms.forEach((harm) => {
|
|
1970
1980
|
acc[harm] = category;
|
|
1971
1981
|
});
|
|
1972
1982
|
return acc;
|
|
1973
1983
|
}, {});
|
|
1974
|
-
const categoryLabels = Object.keys(categoryMapReverse);
|
|
1975
1984
|
const categoryAliases = {
|
|
1976
1985
|
["agentic:memory-poisoning"]: "AgenticMemoryPoisoning",
|
|
1977
1986
|
aegis: "Aegis",
|
|
@@ -2089,6 +2098,7 @@ const categoryAliases = {
|
|
|
2089
2098
|
imitation: "Imitation",
|
|
2090
2099
|
"indirect-prompt-injection": "Indirect Prompt Injection",
|
|
2091
2100
|
"insurance:coverage-discrimination": "InsuranceCoverageDiscrimination",
|
|
2101
|
+
"insurance:data-disclosure": "InsuranceDataDisclosure",
|
|
2092
2102
|
"insurance:network-misinformation": "InsuranceNetworkMisinformation",
|
|
2093
2103
|
"insurance:phi-disclosure": "InsurancePhiDisclosure",
|
|
2094
2104
|
intent: "Intent",
|
|
@@ -2120,7 +2130,7 @@ const categoryAliases = {
|
|
|
2120
2130
|
wordplay: "Wordplay",
|
|
2121
2131
|
xstest: "XSTest"
|
|
2122
2132
|
};
|
|
2123
|
-
|
|
2133
|
+
Object.entries(categoryAliases).reduce((acc, [key, value]) => {
|
|
2124
2134
|
acc[value] = key;
|
|
2125
2135
|
return acc;
|
|
2126
2136
|
}, {});
|
|
@@ -2158,7 +2168,7 @@ const pluginDescriptions = {
|
|
|
2158
2168
|
bias: "Unified bias detection across protected characteristics including age, disability, gender, and race",
|
|
2159
2169
|
medical: "Comprehensive medical AI safety testing for healthcare-specific vulnerabilities including hallucination, anchoring bias, incorrect knowledge, off-label use, prioritization errors, and sycophancy",
|
|
2160
2170
|
pharmacy: "Comprehensive pharmacy AI safety testing for pharmaceutical vulnerabilities including drug interactions, dosage calculations, and controlled substance compliance",
|
|
2161
|
-
insurance: "Comprehensive insurance AI safety testing
|
|
2171
|
+
insurance: "Comprehensive insurance AI safety testing across all insurance types including coverage discrimination, network misinformation, PHI disclosure, and policyholder data disclosure",
|
|
2162
2172
|
financial: "Comprehensive financial AI safety testing for financial services vulnerabilities including calculation errors, compliance violations, hallucination, and data leakage",
|
|
2163
2173
|
ecommerce: "Comprehensive e-commerce AI safety testing for payment security and transaction integrity vulnerabilities including PCI DSS compliance, fraud detection, and price manipulation",
|
|
2164
2174
|
telecom: "Comprehensive telecommunications AI safety testing across CPNI protection, account security, regulatory compliance (FCC, TCPA, CALEA), and telecom-specific fraud prevention",
|
|
@@ -2211,8 +2221,9 @@ const pluginDescriptions = {
|
|
|
2211
2221
|
"harmful:weapons:ied": "Tests handling of Improvised Explosive Devices",
|
|
2212
2222
|
hijacking: "Assesses protection against unauthorized resource usage and purpose deviation",
|
|
2213
2223
|
imitation: "Tests safeguards against unauthorized entity impersonation attempts",
|
|
2214
|
-
"insurance:coverage-discrimination": "Tests for discriminatory coverage or
|
|
2215
|
-
"insurance:
|
|
2224
|
+
"insurance:coverage-discrimination": "Tests for discriminatory coverage, underwriting, or claims determinations based on protected characteristics (age, disability, race, genetic information, sex) in violation of federal civil rights laws including ADA, Section 1557, GINA, Fair Housing Act, ECOA, and state unfair trade practices acts",
|
|
2225
|
+
"insurance:data-disclosure": "Tests for unauthorized disclosure of protected policyholder data including claims history, property details, driving records, beneficiary information, and business data in violation of GLBA, FCRA, DPPA, and state insurance privacy laws",
|
|
2226
|
+
"insurance:network-misinformation": "Tests for inaccurate provider or vendor network information including wrong network status, terminated contracts, ghost networks, and outdated provider data that could expose policyholders to unexpected costs, voided warranties, or inadequate service",
|
|
2216
2227
|
"insurance:phi-disclosure": "Tests for unauthorized disclosure of Protected Health Information in violation of HIPAA privacy regulations including cross-member PHI leakage, inadequate authentication, and disclosure to unauthorized third parties",
|
|
2217
2228
|
"ecommerce:pci-dss": "Tests for exposure of payment card data including credit card numbers, CVVs, expiration dates, cardholder names, bank account details, and payment tokens in violation of PCI DSS requirements for secure payment processing",
|
|
2218
2229
|
"ecommerce:compliance-bypass": "Tests for attempts to bypass security rules, terms of service, age restrictions, geographic limitations, or compliance policies in e-commerce systems",
|
|
@@ -2272,7 +2283,6 @@ const pluginDescriptions = {
|
|
|
2272
2283
|
xstest: "Tests how models handle ambiguous terms related to potentially harmful topics like violence and drugs",
|
|
2273
2284
|
"guardrails-eval": "Evaluate guardrail effectiveness against common risks"
|
|
2274
2285
|
};
|
|
2275
|
-
|
|
2276
2286
|
//#endregion
|
|
2277
2287
|
//#region src/redteam/constants/strategies.ts
|
|
2278
2288
|
const FRAMEWORK_COMPLIANCE_IDS = [
|
|
@@ -2291,8 +2301,6 @@ const DEFAULT_STRATEGIES = [
|
|
|
2291
2301
|
"jailbreak:meta",
|
|
2292
2302
|
"jailbreak:composite"
|
|
2293
2303
|
];
|
|
2294
|
-
const DEFAULT_STRATEGIES_SET = new Set(DEFAULT_STRATEGIES);
|
|
2295
|
-
const DEFAULT_MULTI_TURN_MAX_TURNS = 5;
|
|
2296
2304
|
const MULTI_TURN_STRATEGIES = [
|
|
2297
2305
|
"crescendo",
|
|
2298
2306
|
"goat",
|
|
@@ -2307,12 +2315,6 @@ const isMultiTurnStrategy = (strategyId) => {
|
|
|
2307
2315
|
const isCustomStrategy = (strategyId) => {
|
|
2308
2316
|
return strategyId === "custom" || strategyId.startsWith("custom:");
|
|
2309
2317
|
};
|
|
2310
|
-
const MULTI_MODAL_STRATEGIES = [
|
|
2311
|
-
"audio",
|
|
2312
|
-
"image",
|
|
2313
|
-
"video"
|
|
2314
|
-
];
|
|
2315
|
-
const MULTI_MODAL_STRATEGIES_SET = new Set(MULTI_MODAL_STRATEGIES);
|
|
2316
2318
|
const AGENTIC_STRATEGIES = [
|
|
2317
2319
|
"crescendo",
|
|
2318
2320
|
"goat",
|
|
@@ -2324,7 +2326,6 @@ const AGENTIC_STRATEGIES = [
|
|
|
2324
2326
|
"jailbreak:tree",
|
|
2325
2327
|
"mischievous-user"
|
|
2326
2328
|
];
|
|
2327
|
-
const AGENTIC_STRATEGIES_SET = new Set(AGENTIC_STRATEGIES);
|
|
2328
2329
|
const DATASET_PLUGINS = [
|
|
2329
2330
|
"beavertails",
|
|
2330
2331
|
"cyberseceval",
|
|
@@ -2386,22 +2387,6 @@ const _ALL_STRATEGIES = [
|
|
|
2386
2387
|
...AGENTIC_STRATEGIES
|
|
2387
2388
|
];
|
|
2388
2389
|
const ALL_STRATEGIES = Array.from(new Set(_ALL_STRATEGIES)).sort();
|
|
2389
|
-
const CONFIGURABLE_STRATEGIES = [
|
|
2390
|
-
"layer",
|
|
2391
|
-
"best-of-n",
|
|
2392
|
-
"goat",
|
|
2393
|
-
"crescendo",
|
|
2394
|
-
"indirect-web-pwn",
|
|
2395
|
-
"jailbreak",
|
|
2396
|
-
"jailbreak:hydra",
|
|
2397
|
-
"jailbreak:meta",
|
|
2398
|
-
"jailbreak:tree",
|
|
2399
|
-
"gcg",
|
|
2400
|
-
"citation",
|
|
2401
|
-
"custom",
|
|
2402
|
-
"mischievous-user"
|
|
2403
|
-
];
|
|
2404
|
-
const CONFIGURABLE_STRATEGIES_SET = new Set(CONFIGURABLE_STRATEGIES);
|
|
2405
2390
|
/**
|
|
2406
2391
|
* Default 'n' fan out for strategies that can add additional test cases during generation
|
|
2407
2392
|
*/
|
|
@@ -2416,19 +2401,6 @@ function getDefaultNFanout(strategyId) {
|
|
|
2416
2401
|
function isFanoutStrategy(strategyId) {
|
|
2417
2402
|
return strategyId in DEFAULT_N_FAN_OUT_BY_STRATEGY;
|
|
2418
2403
|
}
|
|
2419
|
-
const STRATEGIES_REQUIRING_REMOTE = [
|
|
2420
|
-
"audio",
|
|
2421
|
-
"citation",
|
|
2422
|
-
"gcg",
|
|
2423
|
-
"goat",
|
|
2424
|
-
"indirect-web-pwn",
|
|
2425
|
-
"jailbreak:composite",
|
|
2426
|
-
"jailbreak:hydra",
|
|
2427
|
-
"jailbreak:likert",
|
|
2428
|
-
"jailbreak:meta"
|
|
2429
|
-
];
|
|
2430
|
-
const STRATEGIES_REQUIRING_REMOTE_SET = new Set(STRATEGIES_REQUIRING_REMOTE);
|
|
2431
|
-
|
|
2432
2404
|
//#endregion
|
|
2433
2405
|
//#region src/util/uuid.ts
|
|
2434
2406
|
/**
|
|
@@ -2445,7 +2417,6 @@ const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-
|
|
|
2445
2417
|
function isUuid(value) {
|
|
2446
2418
|
return UUID_REGEX.test(value);
|
|
2447
2419
|
}
|
|
2448
|
-
|
|
2449
2420
|
//#endregion
|
|
2450
2421
|
//#region src/redteam/plugins/policy/validators.ts
|
|
2451
2422
|
/**
|
|
@@ -2481,7 +2452,6 @@ function isValidInlinePolicyId(id) {
|
|
|
2481
2452
|
function isValidPolicyId(id) {
|
|
2482
2453
|
return isValidReusablePolicyId(id) || isValidInlinePolicyId(id);
|
|
2483
2454
|
}
|
|
2484
|
-
|
|
2485
2455
|
//#endregion
|
|
2486
2456
|
//#region src/redteam/types.ts
|
|
2487
2457
|
const PolicyObjectSchema = z.object({
|
|
@@ -2548,7 +2518,6 @@ var PartialGenerationError = class extends Error {
|
|
|
2548
2518
|
this.failedPlugins = failedPlugins;
|
|
2549
2519
|
}
|
|
2550
2520
|
};
|
|
2551
|
-
|
|
2552
2521
|
//#endregion
|
|
2553
2522
|
//#region src/types/env.ts
|
|
2554
2523
|
const ProviderEnvOverridesSchema = z.object({
|
|
@@ -2623,7 +2592,12 @@ const ProviderEnvOverridesSchema = z.object({
|
|
|
2623
2592
|
OPENAI_API_KEY: z.string().optional(),
|
|
2624
2593
|
OPENAI_BASE_URL: z.string().optional(),
|
|
2625
2594
|
OPENAI_ORGANIZATION: z.string().optional(),
|
|
2595
|
+
CLAWDBOT_GATEWAY_PASSWORD: z.string().optional(),
|
|
2596
|
+
CLAWDBOT_GATEWAY_TOKEN: z.string().optional(),
|
|
2597
|
+
CLAWDBOT_GATEWAY_URL: z.string().optional(),
|
|
2626
2598
|
CODEX_API_KEY: z.string().optional(),
|
|
2599
|
+
OPENCLAW_CONFIG_PATH: z.string().optional(),
|
|
2600
|
+
OPENCLAW_GATEWAY_PASSWORD: z.string().optional(),
|
|
2627
2601
|
OPENCLAW_GATEWAY_TOKEN: z.string().optional(),
|
|
2628
2602
|
OPENCLAW_GATEWAY_URL: z.string().optional(),
|
|
2629
2603
|
PALM_API_HOST: z.string().optional(),
|
|
@@ -2670,7 +2644,6 @@ const ProviderEnvOverridesSchema = z.object({
|
|
|
2670
2644
|
AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
|
|
2671
2645
|
PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
|
|
2672
2646
|
});
|
|
2673
|
-
|
|
2674
2647
|
//#endregion
|
|
2675
2648
|
//#region src/validators/providers.ts
|
|
2676
2649
|
const ProviderOptionsSchema = z.object({
|
|
@@ -2695,7 +2668,7 @@ const ApiProviderSchema = z.object({
|
|
|
2695
2668
|
config: z.any().optional(),
|
|
2696
2669
|
inputs: InputsSchema.optional()
|
|
2697
2670
|
});
|
|
2698
|
-
|
|
2671
|
+
z.object({
|
|
2699
2672
|
cached: z.boolean().optional(),
|
|
2700
2673
|
cost: z.number().optional(),
|
|
2701
2674
|
error: z.string().optional(),
|
|
@@ -2704,17 +2677,17 @@ const ProviderResponseSchema = z.object({
|
|
|
2704
2677
|
output: z.union([z.string(), z.any()]).optional(),
|
|
2705
2678
|
tokenUsage: BaseTokenUsageSchema.optional()
|
|
2706
2679
|
});
|
|
2707
|
-
|
|
2680
|
+
z.object({
|
|
2708
2681
|
error: z.string().optional(),
|
|
2709
2682
|
embedding: z.array(z.number()).optional(),
|
|
2710
2683
|
tokenUsage: BaseTokenUsageSchema.partial().optional()
|
|
2711
2684
|
});
|
|
2712
|
-
|
|
2685
|
+
z.object({
|
|
2713
2686
|
error: z.string().optional(),
|
|
2714
2687
|
similarity: z.number().optional(),
|
|
2715
2688
|
tokenUsage: BaseTokenUsageSchema.partial().optional()
|
|
2716
2689
|
});
|
|
2717
|
-
|
|
2690
|
+
z.object({
|
|
2718
2691
|
error: z.string().optional(),
|
|
2719
2692
|
classification: z.record(z.string(), z.number()).optional()
|
|
2720
2693
|
});
|
|
@@ -2733,7 +2706,6 @@ const ProviderSchema = z.union([
|
|
|
2733
2706
|
ApiProviderSchema,
|
|
2734
2707
|
ProviderOptionsSchema
|
|
2735
2708
|
]);
|
|
2736
|
-
|
|
2737
2709
|
//#endregion
|
|
2738
2710
|
//#region src/validators/prompts.ts
|
|
2739
2711
|
const PromptConfigSchema = z.object({
|
|
@@ -2753,7 +2725,6 @@ function assert$1() {}
|
|
|
2753
2725
|
assert$1();
|
|
2754
2726
|
assert$1();
|
|
2755
2727
|
assert$1();
|
|
2756
|
-
|
|
2757
2728
|
//#endregion
|
|
2758
2729
|
//#region src/validators/redteam.ts
|
|
2759
2730
|
const TracingConfigSchema = z.lazy(() => z.object({
|
|
@@ -2798,7 +2769,7 @@ const RedteamPluginObjectSchema = z.object({
|
|
|
2798
2769
|
message: `Invalid plugin id "${val}". Custom plugins must start with file:// or use a built-in plugin. See https://www.promptfoo.dev/docs/red-team/plugins for available plugins.`
|
|
2799
2770
|
});
|
|
2800
2771
|
})]).describe("Name of the plugin"),
|
|
2801
|
-
numTests: z.int().positive().prefault(
|
|
2772
|
+
numTests: z.int().positive().prefault(5).describe("Number of tests to generate for this plugin"),
|
|
2802
2773
|
config: z.record(z.string(), z.unknown()).optional().describe("Plugin-specific configuration"),
|
|
2803
2774
|
severity: SeveritySchema.optional().describe("Severity level for this plugin")
|
|
2804
2775
|
});
|
|
@@ -3020,11 +2991,9 @@ const RedteamConfigSchema = z.object({
|
|
|
3020
2991
|
});
|
|
3021
2992
|
function assert() {}
|
|
3022
2993
|
assert();
|
|
3023
|
-
|
|
3024
2994
|
//#endregion
|
|
3025
2995
|
//#region src/validators/shared.ts
|
|
3026
2996
|
const NunjucksFilterMapSchema = z.record(z.string(), z.custom((v) => typeof v === "function"));
|
|
3027
|
-
|
|
3028
2997
|
//#endregion
|
|
3029
2998
|
//#region src/types/providers.ts
|
|
3030
2999
|
function isApiProvider(provider) {
|
|
@@ -3033,7 +3002,6 @@ function isApiProvider(provider) {
|
|
|
3033
3002
|
function isProviderOptions(provider) {
|
|
3034
3003
|
return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "string";
|
|
3035
3004
|
}
|
|
3036
|
-
|
|
3037
3005
|
//#endregion
|
|
3038
3006
|
//#region src/types/index.ts
|
|
3039
3007
|
const CommandLineOptionsSchema = z.object({
|
|
@@ -3298,12 +3266,12 @@ const TestCaseSchema = z.object({
|
|
|
3298
3266
|
strategyConfig: z.custom().optional()
|
|
3299
3267
|
}).catchall(z.any()).optional()
|
|
3300
3268
|
});
|
|
3301
|
-
|
|
3269
|
+
TestCaseSchema.extend({ vars: z.union([
|
|
3302
3270
|
VarsSchema,
|
|
3303
3271
|
z.string(),
|
|
3304
3272
|
z.array(z.string())
|
|
3305
3273
|
]).optional() });
|
|
3306
|
-
|
|
3274
|
+
z.object({
|
|
3307
3275
|
id: z.string(),
|
|
3308
3276
|
testCases: z.union([z.string(), z.array(z.union([z.string(), TestCaseSchema]))]),
|
|
3309
3277
|
recentEvalDate: z.date(),
|
|
@@ -3316,7 +3284,7 @@ const ScenarioSchema = z.object({
|
|
|
3316
3284
|
config: z.array(TestCaseSchema.partial()),
|
|
3317
3285
|
tests: z.array(TestCaseSchema)
|
|
3318
3286
|
});
|
|
3319
|
-
|
|
3287
|
+
TestCaseSchema.extend({ vars: VarsSchema.optional() }).strict();
|
|
3320
3288
|
/**
|
|
3321
3289
|
* Configuration schema for test generators that accept parameters
|
|
3322
3290
|
*
|
|
@@ -3502,7 +3470,7 @@ const EvalResultsFilterMode = z.enum([
|
|
|
3502
3470
|
"passes",
|
|
3503
3471
|
"user-rated"
|
|
3504
3472
|
]);
|
|
3505
|
-
|
|
3506
3473
|
//#endregion
|
|
3507
|
-
export {
|
|
3508
|
-
|
|
3474
|
+
export { INSURANCE_PLUGINS as $, DATASET_PLUGINS as A, categoryAliases as B, PolicyObjectSchema as C, ADDITIONAL_STRATEGIES as D, isUuid as E, getDefaultNFanout as F, ADDITIONAL_PLUGINS as G, riskCategorySeverityMap as H, isCustomStrategy as I, DATASET_EXEMPT_PLUGINS as J, ALL_PLUGINS as K, isFanoutStrategy as L, MULTI_TURN_STRATEGIES as M, STRATEGY_COLLECTIONS as N, AGENTIC_STRATEGIES as O, STRATEGY_COLLECTION_MAPPINGS as P, HARM_PLUGINS as Q, isMultiTurnStrategy as R, PluginConfigSchema as S, isValidReusablePolicyId as T, subCategoryDescriptions as U, pluginDescriptions as V, ALIASED_PLUGIN_MAPPINGS as W, FINANCIAL_PLUGINS as X, DEFAULT_PLUGINS as Y, FOUNDATION_PLUGINS as Z, PromptSchema as _, EvaluateOptionsSchema as a, PHARMACY_PLUGINS as at, ConversationMessageSchema as b, TestSuiteConfigSchema as c, REDTEAM_MODEL as ct, isGradingResult as d, STRATEGY_EXEMPT_PLUGINS as dt, LLAMA_GUARD_ENABLED_CATEGORIES as et, isResultFailureReason as f, TELECOM_PLUGINS as ft, RedteamGenerateOptionsSchema as g, RedteamConfigSchema as h, EvalResultsFilterMode as i, MULTI_INPUT_VAR as it, DEFAULT_STRATEGIES as j, ALL_STRATEGIES as k, TestSuiteSchema as l, REDTEAM_PROVIDER_HARM_PLUGINS as lt, isProviderOptions as m, BaseAssertionTypesSchema as n, MEDICAL_PLUGINS as nt, OutputFileExtension as o, PII_PLUGINS as ot, isApiProvider as p, UNALIGNED_PROVIDER_HARM_PLUGINS as pt, BIAS_PLUGINS as q, CommandLineOptionsSchema as r, MULTI_INPUT_EXCLUDED_PLUGINS as rt, ResultFailureReason as s, PLUGIN_CATEGORIES as st, AssertionOrSetSchema as t, LLAMA_GUARD_REPLICATE_PROVIDER as tt, UnifiedConfigSchema as u, REMOTE_ONLY_PLUGIN_IDS as ut, ProviderOptionsSchema as v, StrategyConfigSchema as w, PartialGenerationError as x, ProvidersSchema as y, Severity as z };
|
|
3475
|
+
|
|
3476
|
+
//# sourceMappingURL=types-Cbd8uOMq.js.map
|