promptfoo 0.121.4 → 0.121.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/{ListApp-DQkFNqE9.js → ListApp-DLmM02JS.js} +1 -1
- package/dist/src/{accounts-DhMYUUbu.js → accounts-Ca7WIoPY.js} +12 -7
- package/dist/src/{accounts-F9d_5sMC.js → accounts-CjFnOPmb.js} +14 -9
- package/dist/src/{accounts-Dy17bs4D.cjs → accounts-CmWzeD2d.cjs} +16 -10
- package/dist/src/{accounts-DdJ2pHMI.js → accounts-DanM1wq_.js} +13 -8
- package/dist/src/{agentic-utils-qFlm6zes.js → agentic-utils-CJ0j3fBi.js} +3 -3
- package/dist/src/{agentic-utils-w68v6_Dz.js → agentic-utils-DDEGRV9v.js} +3 -3
- package/dist/src/{agentic-utils-BpX5b23w.cjs → agentic-utils-DvPWSUpb.cjs} +8 -7
- package/dist/src/{agentic-utils-P172hM8B.js → agentic-utils-TxUEMPYS.js} +2 -2
- package/dist/src/{agents-BahDpe5G.cjs → agents-B4sRuXg3.cjs} +7 -6
- package/dist/src/{agents-pQeBEXMm.js → agents-B8q7h_ek.js} +5 -5
- package/dist/src/{agents-CgaMXvLM.js → agents-CBgJvRkB.js} +21 -10
- package/dist/src/{agents-C-R_jfzI.js → agents-CYn2n3QP.js} +4 -4
- package/dist/src/{agents-8FDnTriG.js → agents-D-vDNFx4.js} +21 -10
- package/dist/src/{agents-aYPQLf8W.js → agents-LrHuQqr1.js} +20 -9
- package/dist/src/{agents-DJ35I3Nt.js → agents-QGg76OF-.js} +5 -5
- package/dist/src/{agents-D7-HGxUj.cjs → agents-eHZ9nlgA.cjs} +21 -10
- package/dist/src/{aimlapi-sgYnkE54.js → aimlapi-CJEbQ0o6.js} +7 -7
- package/dist/src/{aimlapi-BD6J9oKt.js → aimlapi-D5HXzZ0s.js} +6 -6
- package/dist/src/{aimlapi-qcK4OT55.cjs → aimlapi-T6HGNxNe.cjs} +7 -7
- package/dist/src/{aimlapi-BCq3MHeL.js → aimlapi-eYv3a_DK.js} +7 -7
- package/dist/src/app/app/tsconfig.app.tsbuildinfo +1 -1
- package/dist/src/app/assets/Report-BNHJKN35.js +1 -0
- package/dist/src/app/assets/index-BnT6P6sF.js +388 -0
- package/dist/src/app/assets/index-yhM8y1PP.css +1 -0
- package/dist/src/app/assets/{scroll-timeline-D9IT_e8Z.js → scroll-timeline-RpeTwOvs.js} +1 -1
- package/dist/src/app/assets/sync-5gq6fmG4.js +4 -0
- package/dist/src/app/assets/vendor-charts-BL9OMNU7.js +36 -0
- package/dist/src/app/assets/{vendor-markdown-Ch00wnNI.js → vendor-markdown-BYsQqn7Z.js} +10 -10
- package/dist/src/app/assets/{vendor-react-CVvmk1UB.js → vendor-react-CqWgVW6T.js} +2 -2
- package/dist/src/app/assets/{vendor-utils-BnEYbx2Q.js → vendor-utils-BHPO71pu.js} +1 -1
- package/dist/src/app/index.html +31 -6
- package/dist/src/{audio-COrn8rM6.js → audio-BqnRvcWG.js} +3 -3
- package/dist/src/{audio-DcVKoInv.js → audio-CPMtV1yR.js} +4 -4
- package/dist/src/{audio-B7izf48x.js → audio-DyiebVB3.js} +4 -4
- package/dist/src/{audio-BQtNuYBj.cjs → audio-FnxbEnSE.cjs} +4 -4
- package/dist/src/authoritativeMarkupInjection-BZIywVjG.js +74 -0
- package/dist/src/authoritativeMarkupInjection-DyAXAsSr.js +75 -0
- package/dist/src/authoritativeMarkupInjection-F2gBw0lN.cjs +74 -0
- package/dist/src/authoritativeMarkupInjection-QEQmFS83.js +74 -0
- package/dist/src/{base-PYJvBE1i.js → base-CKLo890h.js} +4 -3
- package/dist/src/{base-fZ9wgg50.js → base-Co80MMCi.js} +5 -4
- package/dist/src/{base-D-670DX8.cjs → base-DGJW48uz.cjs} +5 -4
- package/dist/src/{base-yrI1Yal4.js → base-E9I8zXjz.js} +5 -4
- package/dist/src/bestOfN-B3wNzjSB.js +137 -0
- package/dist/src/bestOfN-BBsO41z4.js +136 -0
- package/dist/src/bestOfN-CAwmg5UL.cjs +140 -0
- package/dist/src/bestOfN-_kTi8Bxe.js +136 -0
- package/dist/src/{blobs-D2FAd1Q5.cjs → blobs-B0977K1O.cjs} +7 -6
- package/dist/src/{blobs-BCZavS8s.js → blobs-CeFdPn_T.js} +3 -3
- package/dist/src/{blobs-BQWqnnvL.js → blobs-DODuTK-a.js} +3 -3
- package/dist/src/{blobs-C-F78Kfn.js → blobs-Dwef1Ao1.js} +2 -2
- package/dist/src/{cache-BIyPcp5v.cjs → cache-CPGUA4Yl.cjs} +135 -25
- package/dist/src/cache-Cf7b4pWE.js +3 -0
- package/dist/src/{cache-D5NZmMiT.js → cache-DIXbtkNO.js} +125 -10
- package/dist/src/{cache-mb7c8hbp.js → cache-DpPWrkTE.js} +128 -12
- package/dist/src/{cache-C4Xb-hNb.js → cache-roFAE0cI.js} +126 -11
- package/dist/src/{chat-I9izLm49.js → chat-CUCorGiL.js} +12 -12
- package/dist/src/{chat-BPXSW8Bv.cjs → chat-DG1wG4w0.cjs} +6 -6
- package/dist/src/{chat-BfPaS15_.js → chat-Dabu84Br.js} +12 -12
- package/dist/src/{chat-Dr3DUQ0D.js → chat-DqUFcWI0.js} +12 -12
- package/dist/src/{chat-CclRbxGf.cjs → chat-DxTDQ83C.cjs} +14 -13
- package/dist/src/{chat-MKxMnZJZ.js → chat-GmlolEwo.js} +4 -4
- package/dist/src/{chat-0bwXjVP0.js → chat-TP8Qifkh.js} +6 -6
- package/dist/src/{chat-mW0ORo8G.js → chat-iwaM5UTQ.js} +6 -6
- package/dist/src/{chatkit-zUIVoDos.js → chatkit-B6DWi70Q.js} +4 -4
- package/dist/src/{chatkit-BoWoSgXl.cjs → chatkit-BYveR48_.cjs} +6 -5
- package/dist/src/{chatkit-Cv6AhukM.js → chatkit-fARZwEfV.js} +3 -3
- package/dist/src/{chatkit-CJnHRRMM.js → chatkit-lb6FK02w.js} +4 -4
- package/dist/src/{claude-agent-sdk-Dtq_L-Sc.js → claude-agent-sdk-BQNp_y-F.js} +212 -67
- package/dist/src/{claude-agent-sdk-BQNuLaAK.js → claude-agent-sdk-D5Jl0SDh.js} +212 -67
- package/dist/src/{claude-agent-sdk-CPJo3dBQ.cjs → claude-agent-sdk-DH416NBD.cjs} +218 -72
- package/dist/src/{claude-agent-sdk-nfAIcxNf.js → claude-agent-sdk-x1XJ1-pU.js} +212 -67
- package/dist/src/{cloud-DQZ5sVjW.js → cloud-D3DiFqH6.js} +3 -3
- package/dist/src/cloud-p96PA4MH.js +3 -0
- package/dist/src/{cloudflare-ai-BIB567w6.js → cloudflare-ai-B6NVI3ax.js} +4 -4
- package/dist/src/{cloudflare-ai-Dl3N9OVD.cjs → cloudflare-ai-CEAW-xQa.cjs} +6 -6
- package/dist/src/{cloudflare-ai-DlKr0rY7.js → cloudflare-ai-RFSojyXG.js} +6 -6
- package/dist/src/{cloudflare-ai-DGLte7Py.js → cloudflare-ai-r4tbYmWU.js} +6 -6
- package/dist/src/{cloudflare-gateway-CiIZHU0Q.js → cloudflare-gateway-BCkLouto.js} +5 -5
- package/dist/src/{cloudflare-gateway-DI1HNP5F.js → cloudflare-gateway-BaZ4insB.js} +3 -3
- package/dist/src/{cloudflare-gateway-BDZrYydE.js → cloudflare-gateway-CF-Vb-2Z.js} +5 -5
- package/dist/src/{cloudflare-gateway-BYDp495F.cjs → cloudflare-gateway-TJMLBj6I.cjs} +5 -5
- package/dist/src/codex-app-server-B8KHEiF4.js +1915 -0
- package/dist/src/codex-app-server-CnrLBCeA.cjs +1921 -0
- package/dist/src/codex-app-server-DIXZ230V.js +1915 -0
- package/dist/src/codex-app-server-Dd22dC_N.js +1916 -0
- package/dist/src/{codex-sdk-CpqiOqDO.js → codex-sdk-B6Wah8Pa.js} +6 -6
- package/dist/src/codex-sdk-BGjVAk23.js +3 -0
- package/dist/src/{codex-sdk-C2_M2pl_.cjs → codex-sdk-CFF6gUyi.cjs} +18 -10
- package/dist/src/{codex-sdk-Rtky3M4I.js → codex-sdk-CmQABzV3.js} +6 -6
- package/dist/src/{codex-sdk-CErXn7qh.js → codex-sdk-D2d54RL8.js} +5 -5
- package/dist/src/{cometapi-CtJ-mS8R.js → cometapi-Bu9B8NUY.js} +8 -8
- package/dist/src/{cometapi-DT-jlVCB.js → cometapi-CtzNCHKu.js} +7 -7
- package/dist/src/{cometapi-UVOryo4W.cjs → cometapi-DHCDlQUI.cjs} +8 -8
- package/dist/src/{cometapi-BUlt_ELa.js → cometapi-OBILPLlu.js} +8 -8
- package/dist/src/{completion-HUe8wDhZ.js → completion-CO2e1_62.js} +6 -6
- package/dist/src/{completion-BozdoXba.cjs → completion-CSYfl2cd.cjs} +6 -6
- package/dist/src/{completion-x0a_c2y1.js → completion-DZNxcyfG.js} +6 -6
- package/dist/src/{completion-Dnxn7E-j.js → completion-sNvCLTAP.js} +5 -5
- package/dist/src/constants-BjJV0cRr.js +6 -0
- package/dist/src/constants-DH5XYLKZ.js +7 -0
- package/dist/src/constants-DZGEFLsu.js +6 -0
- package/dist/src/constants-a2kYssQk.cjs +11 -0
- package/dist/src/{createHash-4gFQpDDv.js → createHash-BtbSX3mj.js} +1 -1
- package/dist/src/{createHash-CwDVU5xr.js → createHash-CGVzWdjj.js} +1 -1
- package/dist/src/{createHash-B7KvgoOD.cjs → createHash-CSiqnK5P.cjs} +2 -2
- package/dist/src/{createHash-ChI45QR1.js → createHash-CgRvs4Fn.js} +1 -1
- package/dist/src/crescendo-BXEJK_bi.cjs +704 -0
- package/dist/src/crescendo-CU_Y2i-m.js +702 -0
- package/dist/src/crescendo-J1Xx4_zb.js +703 -0
- package/dist/src/crescendo-QiaSLW0d.js +701 -0
- package/dist/src/custom-BJfP00Bh.js +619 -0
- package/dist/src/custom-CZVn-1-r.js +620 -0
- package/dist/src/custom-Cqia7M0D.cjs +621 -0
- package/dist/src/custom-notggYVl.js +618 -0
- package/dist/src/{docker-DCgsveLD.js → docker-4D1eL6Gq.js} +6 -6
- package/dist/src/{docker-ClnmCf1Z.js → docker-BBv1WUDu.js} +5 -5
- package/dist/src/{docker-DS4_Osau.cjs → docker-D06JUoe2.cjs} +6 -6
- package/dist/src/{docker-CQmlA2NU.js → docker-DdJQBxK9.js} +6 -6
- package/dist/src/{embedding-D3xTseo7.js → embedding--UZVe4_7.js} +6 -6
- package/dist/src/{embedding-I45KG3o7.cjs → embedding-BbrwopfX.cjs} +6 -6
- package/dist/src/{embedding-nFbumxcv.js → embedding-Bi3rxrZF.js} +5 -5
- package/dist/src/{embedding-DD9wa3ae.js → embedding-C251p1-8.js} +6 -6
- package/dist/src/{errors-Cw810C93.js → errors-9PcUL8BC.js} +1 -1
- package/dist/src/{esm-Dh4dOLlt.js → esm-B6whoAcf.js} +2 -2
- package/dist/src/{esm-CtEPLdAj.cjs → esm-BIKakvNa.cjs} +8 -7
- package/dist/src/{esm-C7PnfdF8.js → esm-BTK1W7lG.js} +1 -1
- package/dist/src/{esm-tVgYPY-f.js → esm-Bexx2PFc.js} +2 -2
- package/dist/src/{eval-u4UVafl6.js → eval-0VRANImH.js} +21 -21
- package/dist/src/{eval-CzJFfFO9.js → eval-DscR5iOM.js} +1 -1
- package/dist/src/{evalResult-Bgm9ZH31.js → evalResult-2RRJvFyB.js} +41 -16
- package/dist/src/{evalResult-KZqXl4XP.cjs → evalResult-CvtS8h8u.cjs} +51 -15
- package/dist/src/evalResult-DqzsS6_W.js +3 -0
- package/dist/src/{evalResult-D3hVYFis.js → evalResult-eUkJv9Ko.js} +40 -15
- package/dist/src/evaluator-DNdJF1Gv.js +3 -0
- package/dist/src/{evaluator-IvuDYSvQ.js → evaluator-DRoiYB2q.js} +1060 -187
- package/dist/src/evaluatorHelpers-BsYP_muT.js +511 -0
- package/dist/src/evaluatorHelpers-CRqTvSux.cjs +537 -0
- package/dist/src/evaluatorHelpers-DuqFFfq7.js +510 -0
- package/dist/src/{extractor-CAfTSraf.js → extractor-BR7XAzAL.js} +6 -6
- package/dist/src/{extractor-WVPOrH43.cjs → extractor-BdxEtt3J.cjs} +6 -6
- package/dist/src/{extractor-DNSeBVOJ.js → extractor-CIW3iN-b.js} +6 -6
- package/dist/src/{extractor-Dk6bRWkv.js → extractor-CxRtnaHl.js} +5 -5
- package/dist/src/{fetch-B0Z3Oe4k.js → fetch-BufrQtvR.js} +93 -40
- package/dist/src/{fetch-BEWnXrrG.js → fetch-DXUnXkVU.js} +89 -40
- package/dist/src/{fetch-CJU5ELPa.cjs → fetch-Dw4XZHjj.cjs} +330 -270
- package/dist/src/{fetch-Di00EQrc.js → fetch-It34O8Ur.js} +305 -252
- package/dist/src/fetch-_YgGd2qv.js +3 -0
- package/dist/src/{fileExtensions-bYh77CN8.cjs → fileExtensions-BhdwzYaD.cjs} +24 -1
- package/dist/src/{fileExtensions-DnqA1y9x.js → fileExtensions-CXRfY3Ss.js} +12 -2
- package/dist/src/{fileExtensions-AWa2ZML4.js → fileExtensions-D4GCJ67J.js} +12 -2
- package/dist/src/{formatDuration-DZzPsexs.js → formatDuration-CMVNrYvE.js} +1 -1
- package/dist/src/{genaiTracer-yRuxj9-L.cjs → genaiTracer-14nugQQx.cjs} +14 -2
- package/dist/src/{genaiTracer-DWdZ28hY.js → genaiTracer-BPVvltoW.js} +2 -2
- package/dist/src/{genaiTracer-XnrcgDCe.js → genaiTracer-D18lYzhB.js} +2 -2
- package/dist/src/{genaiTracer-COYDi-tC.js → genaiTracer-jJKYsnjc.js} +2 -2
- package/dist/src/goat-Ckd3q3AY.js +467 -0
- package/dist/src/goat-Qgurm-NP.js +466 -0
- package/dist/src/goat-ghadEDdy.js +465 -0
- package/dist/src/goat-una6pZGP.cjs +469 -0
- package/dist/src/graders-BDT7dif6.js +3 -0
- package/dist/src/{graders-eIHhRqoC.js → graders-BGP99PdK.js} +2416 -2224
- package/dist/src/{graders-Zy3x0zqX.js → graders-BX0f2tvS.js} +2423 -2226
- package/dist/src/{graders-pvbReLLn.js → graders-C0nXU_ZP.js} +1806 -1609
- package/dist/src/{graders--zknU_uk.cjs → graders-ClrU2fnd.cjs} +2219 -1949
- package/dist/src/hydra-BSNZZm2M.js +543 -0
- package/dist/src/hydra-BxdG4nkg.js +541 -0
- package/dist/src/hydra-DE4xWwyc.js +542 -0
- package/dist/src/hydra-DrJttnvw.cjs +542 -0
- package/dist/src/image-B4oBtu6J.js +443 -0
- package/dist/src/{image-dnoUgPrC.js → image-BN-hjLL9.js} +4 -4
- package/dist/src/{image-9302QVqR.js → image-B_fPIwdg.js} +3 -3
- package/dist/src/image-BvUAW344.js +442 -0
- package/dist/src/image-Cvjwx1uY.js +442 -0
- package/dist/src/{image-De2FBmYV.cjs → image-DfVCGPbI.cjs} +4 -4
- package/dist/src/{image-u7-rKnYU.js → image-QzmydkiG.js} +4 -4
- package/dist/src/image-X0oY4350.cjs +465 -0
- package/dist/src/index.cjs +1689 -558
- package/dist/src/index.d.cts +3270 -1624
- package/dist/src/index.d.ts +3270 -1624
- package/dist/src/index.js +1553 -438
- package/dist/src/indirectWebPwn-02ZIghCS.js +259 -0
- package/dist/src/indirectWebPwn-BJ22AbQa.cjs +397 -0
- package/dist/src/indirectWebPwn-CbjUG0rh.js +385 -0
- package/dist/src/indirectWebPwn-CfQJt3gk.cjs +260 -0
- package/dist/src/indirectWebPwn-DBQhOjoD.js +260 -0
- package/dist/src/indirectWebPwn-OsXnKejv.js +259 -0
- package/dist/src/indirectWebPwn-tNx9OZ35.js +385 -0
- package/dist/src/indirectWebPwn-uyWdHx04.js +386 -0
- package/dist/src/inputVariables-B0qUChbV.js +467 -0
- package/dist/src/inputVariables-DUGMb9Ka.js +464 -0
- package/dist/src/inputVariables-DXFdi7AI.js +468 -0
- package/dist/src/inputVariables-Dq9W-Z3a.cjs +475 -0
- package/dist/src/{interactiveCheck-CLERUB0c.js → interactiveCheck-C4QlIuoR.js} +2 -2
- package/dist/src/{invariant-BtWWVVhl.js → invariant-B2Rf6avk.js} +1 -1
- package/dist/src/{invariant-vgHWClmd.js → invariant-DIYf9sP1.js} +1 -1
- package/dist/src/{invariant-kfQ8Bu82.cjs → invariant-QtnLD03y.cjs} +1 -1
- package/dist/src/iterative-CpU6i2As.js +490 -0
- package/dist/src/iterative-DJQEQpG3.js +491 -0
- package/dist/src/iterative-DQBuWM-j.cjs +493 -0
- package/dist/src/iterative-FTS4Bz67.js +492 -0
- package/dist/src/iterativeImage-BUABMVOA.js +413 -0
- package/dist/src/iterativeImage-ByFWkxax.cjs +415 -0
- package/dist/src/iterativeImage-BzUapOUi.js +414 -0
- package/dist/src/iterativeImage-Doz8mgxF.js +413 -0
- package/dist/src/iterativeMeta-B3YiAOc8.js +386 -0
- package/dist/src/iterativeMeta-C7APE_P1.js +385 -0
- package/dist/src/iterativeMeta-CSS8M6Ds.cjs +385 -0
- package/dist/src/iterativeMeta-DgoQ7bLh.js +384 -0
- package/dist/src/iterativeTree-B5zxBBSW.js +769 -0
- package/dist/src/iterativeTree-CNyIk0Yn.js +768 -0
- package/dist/src/iterativeTree-CPMF10ve.cjs +771 -0
- package/dist/src/iterativeTree-DvZ7GBwt.js +770 -0
- package/dist/src/{knowledgeBase-Dgc7CBWF.js → knowledgeBase-BadkINlJ.js} +24 -10
- package/dist/src/{knowledgeBase-RhFPGWDc.js → knowledgeBase-Bi_8sV-H.js} +25 -11
- package/dist/src/{knowledgeBase-lm9RXSAm.js → knowledgeBase-CkMljjdg.js} +25 -11
- package/dist/src/{knowledgeBase-Bpoe_nLu.cjs → knowledgeBase-DUh34xba.cjs} +25 -11
- package/dist/src/{litellm-DRjpcSa7.js → litellm-BKBo0jpC.js} +5 -5
- package/dist/src/{litellm-C2kqjxqp.js → litellm-BXyn5kZK.js} +5 -5
- package/dist/src/{litellm-p37R1dzQ.js → litellm-CNcfbCfa.js} +4 -4
- package/dist/src/{litellm-CoyI4IAl.cjs → litellm-CtAr7bKG.cjs} +5 -5
- package/dist/src/{logger-DksKw1Qc.js → logger-BbY6ypFL.js} +2 -2
- package/dist/src/{logger-B88EkIn6.js → logger-KD8JjCRJ.js} +2 -2
- package/dist/src/{logger-COuQb2xB.cjs → logger-cfNpzI4o.cjs} +13 -55
- package/dist/src/{luma-ray-KgTCXrZC.js → luma-ray-BMX1iEB6.js} +5 -5
- package/dist/src/{luma-ray-B863CmuZ.js → luma-ray-CR5TSpp4.js} +5 -5
- package/dist/src/{luma-ray-BxVKaW2a.cjs → luma-ray-D3FUc2K3.cjs} +9 -8
- package/dist/src/{luma-ray-BTTLtqQ8.js → luma-ray-OEMmS1RB.js} +6 -6
- package/dist/src/main.js +909 -369
- package/dist/src/memoryPoisoning-CM83NWYl.js +107 -0
- package/dist/src/memoryPoisoning-D8h9gXJF.js +106 -0
- package/dist/src/memoryPoisoning-Dp-btinn.cjs +106 -0
- package/dist/src/memoryPoisoning-cLuCoTuJ.js +106 -0
- package/dist/src/{messages-BTQz42fn.js → messages-BabO-cX8.js} +273 -17
- package/dist/src/{messages-811uVVW5.cjs → messages-DBPir0TQ.cjs} +278 -18
- package/dist/src/{messages-zWbkLLHz.js → messages-DGUlSNU7.js} +273 -17
- package/dist/src/{messages-MYTQ2TWp.js → messages-vsE_-Lv0.js} +273 -17
- package/dist/src/{meteor-DHdzY1Ss.js → meteor--TZYICTI.js} +2 -2
- package/dist/src/{meteor-Co1VQ1u5.cjs → meteor-CR226f7Z.cjs} +2 -2
- package/dist/src/{meteor-CU5UAE-H.js → meteor-Cl_yd7rJ.js} +2 -2
- package/dist/src/{meteor-DuAFv6gF.js → meteor-Dce-_zGQ.js} +1 -1
- package/dist/src/mischievousUser-0l8GD7Dp.js +46 -0
- package/dist/src/mischievousUser-BUOP9W5r.js +46 -0
- package/dist/src/mischievousUser-frFYKxu6.js +47 -0
- package/dist/src/mischievousUser-olGgHIVR.cjs +46 -0
- package/dist/src/{modelslab-Dk1JAtVo.cjs → modelslab-CNV5bMSk.cjs} +7 -7
- package/dist/src/{modelslab-D0erNWKe.js → modelslab-Cogmu4mG.js} +6 -6
- package/dist/src/{modelslab-DIq-6y7x.js → modelslab-Dzst7VTU.js} +6 -6
- package/dist/src/{modelslab-wu9yi5GE.js → modelslab-EyDczZ5A.js} +7 -7
- package/dist/src/{nova-reel-CCFRfeRb.js → nova-reel-BGPNBOMS.js} +6 -6
- package/dist/src/{nova-reel-DQrm74ng.js → nova-reel-B_5NKFu1.js} +5 -5
- package/dist/src/{nova-reel-gr11WG7f.js → nova-reel-C4eUJGse.js} +5 -5
- package/dist/src/{nova-reel-CrLXVKQf.cjs → nova-reel-CjJRxI1X.cjs} +9 -8
- package/dist/src/{nova-sonic-BYdp-QLs.js → nova-sonic-BNGmgfFz.js} +4 -4
- package/dist/src/{nova-sonic-TDgrlTk7.js → nova-sonic-ChPlh5na.js} +4 -4
- package/dist/src/{nova-sonic-B_ZXcUJB.js → nova-sonic-CrV0iaY_.js} +3 -3
- package/dist/src/{nova-sonic-i5tUvXKn.cjs → nova-sonic-DuOG9Aun.cjs} +5 -4
- package/dist/src/{openai-DhVEmgeZ.js → openai-BMHD2Huo.js} +2 -2
- package/dist/src/{openai-URNyItar.cjs → openai-C3uXv8wS.cjs} +2 -2
- package/dist/src/{openai-Qsvz25mV.js → openai-CJrsh9n4.js} +2 -2
- package/dist/src/{openai-iYtrXzOX.js → openai-zgwBb4Ff.js} +1 -1
- package/dist/src/{openclaw-CnQ363Wi.js → openclaw-BIHlu_36.js} +10 -8
- package/dist/src/{openclaw-CwzlQSQX.js → openclaw-CF7fMido.js} +9 -7
- package/dist/src/{openclaw-wX9rtfke.cjs → openclaw-Dphc01BY.cjs} +18 -15
- package/dist/src/{openclaw-CLWrW03k.js → openclaw-zIJAsz3P.js} +10 -8
- package/dist/src/{opencode-sdk-BUu5Nevv.js → opencode-sdk-B3vlPLsp.js} +40 -5
- package/dist/src/{opencode-sdk-BxD8vXp_.js → opencode-sdk-D05JSgMQ.js} +40 -5
- package/dist/src/{opencode-sdk-BZ2idgYA.cjs → opencode-sdk-DoY6GbWw.cjs} +46 -10
- package/dist/src/{opencode-sdk-GI2KaAXq.js → opencode-sdk-sRKYHGoI.js} +39 -4
- package/dist/src/{otlpReceiver-BntK801g.js → otlpReceiver--gTpSagc.js} +120 -4
- package/dist/src/{otlpReceiver-DmVulbhC.js → otlpReceiver-B2eaKC8C.js} +120 -4
- package/dist/src/{otlpReceiver-B2z58l4e.js → otlpReceiver-BXjcRqAM.js} +119 -3
- package/dist/src/{otlpReceiver-BfcVq2Nq.cjs → otlpReceiver-CvJdBGSc.cjs} +125 -7
- package/dist/src/packageParser--MWTSrPW.js +36 -0
- package/dist/src/packageParser-CgE-ziRo.js +35 -0
- package/dist/src/packageParser-QoCS1FMl.cjs +54 -0
- package/dist/src/packageParser-hwwSGnAZ.js +35 -0
- package/dist/src/processShim-BBxt7LKO.js +95 -0
- package/dist/src/processShim-BcGzU8fY.js +94 -0
- package/dist/src/processShim-C_z3aRvF.js +94 -0
- package/dist/src/processShim-DSY9BV2T.cjs +98 -0
- package/dist/src/promptLength-0qIHyhA5.js +71 -0
- package/dist/src/promptLength-4X-Wd8PG.js +72 -0
- package/dist/src/promptLength-B9nZEfO6.js +71 -0
- package/dist/src/promptLength-BbBbDHNj.cjs +94 -0
- package/dist/src/promptfoo-BDrfT30-.js +180 -0
- package/dist/src/promptfoo-Cm4hiy1Y.js +180 -0
- package/dist/src/promptfoo-Rjp-MeBb.js +181 -0
- package/dist/src/promptfoo-b-baRMj-.cjs +205 -0
- package/dist/src/prompts-BYMtqPCw.js +259 -0
- package/dist/src/prompts-C-bqE1Yp.js +260 -0
- package/dist/src/prompts-Cp_Qx5Ml.js +270 -0
- package/dist/src/prompts-DHhQsANy.js +259 -0
- package/dist/src/prompts-D_QpZ2Dm.js +271 -0
- package/dist/src/prompts-hNvWBD3z.cjs +284 -0
- package/dist/src/prompts-huDVH2CI.js +270 -0
- package/dist/src/prompts-p78Hul5i.cjs +289 -0
- package/dist/src/{providerRegistry-CPQ_CmVO.js → providerRegistry-1gB5vtzQ.js} +2 -2
- package/dist/src/{providerRegistry-CQMdTmHP.cjs → providerRegistry-CZO_w7ue.cjs} +2 -2
- package/dist/src/{providerRegistry-Bvh8mv85.js → providerRegistry-DHcFiVWX.js} +1 -1
- package/dist/src/{providerRegistry-CWoPjKFZ.js → providerRegistry-ReCd0sFa.js} +2 -2
- package/dist/src/{providers-BV_KMZje.js → providers-B9KzWxAX.js} +10558 -21587
- package/dist/src/{providers-DruaQfwu.js → providers-BCCz6_IX.js} +1228 -12196
- package/dist/src/{providers-1eKkXBKp.cjs → providers-BDVVIQM6.cjs} +10649 -21843
- package/dist/src/{providers-iUt5fbAN.js → providers-BYAn82cf.js} +1 -1
- package/dist/src/{providers-Domz_llv.js → providers-DVYRZP4E.js} +10589 -21570
- package/dist/src/{pythonUtils-Cldx7huE.js → pythonUtils-CLCgQ9tt.js} +3 -3
- package/dist/src/{pythonUtils-CnndUbW-.js → pythonUtils-CgYxeSmO.js} +3 -3
- package/dist/src/{pythonUtils-tAJvvpS-.cjs → pythonUtils-Cokhluq3.cjs} +8 -7
- package/dist/src/{pythonUtils-C2UQ30Rz.js → pythonUtils-D0BYebvX.js} +3 -3
- package/dist/src/{quiverai-DFotyafY.cjs → quiverai-BAp6iTZD.cjs} +4 -4
- package/dist/src/{quiverai-aPPvXOgn.js → quiverai-BvIhI_0l.js} +4 -4
- package/dist/src/{quiverai-DR0SnIQV.js → quiverai-CdTWPe-A.js} +3 -3
- package/dist/src/{quiverai-CtWi6x_g.js → quiverai-Cv7rJKDz.js} +4 -4
- package/dist/src/registry-BUJrgjwv.js +124 -0
- package/dist/src/registry-DXm1t_x0.js +125 -0
- package/dist/src/registry-Dp5EqoXc.js +124 -0
- package/dist/src/registry-KCVF1CFC.cjs +124 -0
- package/dist/src/{server-D6Il2Sob.js → remoteGeneration-B1_XsKXU.js} +16 -108
- package/dist/src/{server-BSB45Nt9.js → remoteGeneration-COpWcmWd.js} +15 -146
- package/dist/src/{server-Dx2TyCH2.cjs → remoteGeneration-DS9N3pgB.cjs} +30 -119
- package/dist/src/remoteGeneration-DsaSwmG2.js +217 -0
- package/dist/src/render-BNTrbmBw.cjs +384 -0
- package/dist/src/render-CSP99NLm.js +348 -0
- package/dist/src/render-DFfDeYUK.js +347 -0
- package/dist/src/{render-CgVDrJmM.js → render-DznWrxGO.js} +2 -2
- package/dist/src/render-_6ur1fhE.js +347 -0
- package/dist/src/resourceAttributes-D1jP3kL5.js +17 -0
- package/dist/src/resourceAttributes-DQbBB--2.js +16 -0
- package/dist/src/resourceAttributes-ephgOvdR.cjs +27 -0
- package/dist/src/resourceAttributes-v6-I67fn.js +16 -0
- package/dist/src/{responses-Bi9vBuW_.cjs → responses-1UFFF9N_.cjs} +51 -16
- package/dist/src/{responses-DL9m8CyY.js → responses-B3W2JvOQ.js} +49 -15
- package/dist/src/{responses--OsX2aYW.js → responses-B6ktc3Ra.js} +49 -15
- package/dist/src/{responses-C-flexAY.js → responses-URRzV8qE.js} +49 -15
- package/dist/src/rolldown-runtime-D_mwlA32.cjs +43 -0
- package/dist/src/rubyUtils-BYVlQ94c.js +3 -0
- package/dist/src/{rubyUtils-DsGrTx8R.js → rubyUtils-CXlFM2rR.js} +3 -3
- package/dist/src/{rubyUtils-DVLeA2jg.js → rubyUtils-CnlW8AYb.js} +3 -3
- package/dist/src/{rubyUtils-B6eljPuh.cjs → rubyUtils-CqUWBZAt.cjs} +18 -27
- package/dist/src/{rubyUtils-CYSQEG4a.js → rubyUtils-DdGojpfv.js} +3 -3
- package/dist/src/runtimeTransform-BJOpL9Yc.js +142 -0
- package/dist/src/runtimeTransform-Dgh_D7DU.js +143 -0
- package/dist/src/runtimeTransform-DigbjU1r.js +142 -0
- package/dist/src/runtimeTransform-ON3YYILw.cjs +147 -0
- package/dist/src/{sagemaker-BVkaG2-l.js → sagemaker-CujrzP1a.js} +62 -51
- package/dist/src/{sagemaker-XnfhheQv.cjs → sagemaker-DzffAqo_.cjs} +65 -53
- package/dist/src/{sagemaker-D67yzMzs.js → sagemaker-vhtSV7JI.js} +62 -51
- package/dist/src/{sagemaker-BveBvuxm.js → sagemaker-yr1QKeBs.js} +61 -50
- package/dist/src/{scanner-1DqWi1Ej.js → scanner-DS0109SS.js} +7 -7
- package/dist/src/server/index.js +5105 -605
- package/dist/src/server-B8rqV126.cjs +126 -0
- package/dist/src/server-BaLytskk.js +3 -0
- package/dist/src/server-CMJD10J4.js +107 -0
- package/dist/src/server-Ddp8GNMp.js +146 -0
- package/dist/src/server-DhMHosWj.js +182 -0
- package/dist/src/shared-7pmVZLNO.js +1334 -0
- package/dist/src/shared-9WHQ1oNE.js +1335 -0
- package/dist/src/{fileExtensions-BArZuxsI.js → shared-BoG7qLMv.js} +12 -2
- package/dist/src/shared-D6IjElRI.js +1334 -0
- package/dist/src/shared-WkgnDkcg.cjs +1436 -0
- package/dist/src/{signal-CE5G3a7x.js → signal-CSurUUyV.js} +3 -3
- package/dist/src/simulatedUser-C9aQObBI.js +222 -0
- package/dist/src/simulatedUser-Cu601Dd4.cjs +227 -0
- package/dist/src/simulatedUser-U_qAHnuB.js +222 -0
- package/dist/src/simulatedUser-p3tACcmw.js +223 -0
- package/dist/src/{slack-DDUe-5MC.js → slack-Bapo-7_8.js} +2 -2
- package/dist/src/{slack-1Rhq0EoV.cjs → slack-DMC1QVEg.cjs} +3 -2
- package/dist/src/{slack-D5Wpy8LM.js → slack-DTEFhrMn.js} +2 -2
- package/dist/src/{slack-acRb0IqQ.js → slack-k-_CP84Q.js} +1 -1
- package/dist/src/storage-BU4qcnOb.js +875 -0
- package/dist/src/storage-CA-v9V2v.cjs +911 -0
- package/dist/src/storage-CD-GWAdx.js +822 -0
- package/dist/src/storage-QdU-SmvD.js +834 -0
- package/dist/src/{store-DAAyxcy6.cjs → store-B2NDDooM.cjs} +60 -24
- package/dist/src/{store-CYEy5J2D.js → store-DKd5592Q.js} +51 -20
- package/dist/src/{store-M0b1WfYb.js → store-HpopRVzl.js} +50 -19
- package/dist/src/store-IbiRIF3k.js +3 -0
- package/dist/src/strategies-7CS3Alao.cjs +2360 -0
- package/dist/src/strategies-CiSeroPH.js +2331 -0
- package/dist/src/strategies-DRJjGTIY.js +2333 -0
- package/dist/src/{tables-DQ4WU5tX.js → tables-CRSXQ2Ke.js} +2 -2
- package/dist/src/{tables-CsWou1Bx.js → tables-CxjU7bBd.js} +3 -3
- package/dist/src/{tables-DUfh1F7Z.cjs → tables-DBIJU0WE.cjs} +6 -5
- package/dist/src/{tables-C4CH3zRr.js → tables-DafUHOeh.js} +3 -3
- package/dist/src/{telemetry-CQPez_Jp.js → telemetry-00ezXr_t.js} +5 -4
- package/dist/src/telemetry-ByPqDcKC.js +3 -0
- package/dist/src/{telemetry-Dsw_faFj.cjs → telemetry-CJ7FnCsc.cjs} +18 -11
- package/dist/src/{telemetry-dbaJ0E98.js → telemetry-DmXYcJNV.js} +5 -4
- package/dist/src/{telemetry-Dvqxv3YC.js → telemetry-DwX9XUN5.js} +4 -3
- package/dist/src/{text-KvuD2Iko.js → text-Db-Wt2u2.js} +1 -1
- package/dist/src/{text-DHxdyQqT.js → text-DwYK5EBn.js} +1 -1
- package/dist/src/{text-BVi-cLPJ.cjs → text-nywWsRBM.cjs} +1 -1
- package/dist/src/{tokenUsageUtils-C-bmyHoE.js → tokenUsageUtils-BjVkdk18.js} +1 -1
- package/dist/src/{tokenUsageUtils-CXrvO-wA.js → tokenUsageUtils-CDet74yk.js} +1 -1
- package/dist/src/tokenUsageUtils-CmnQ0G2m.js +142 -0
- package/dist/src/{tokenUsageUtils-Bb7DkZPz.cjs → tokenUsageUtils-_B-P8IAi.cjs} +1 -1
- package/dist/src/toolAttributes-BAjwcBf0.cjs +103 -0
- package/dist/src/toolAttributes-COVgDrBG.js +87 -0
- package/dist/src/toolAttributes-DJ9ZEKXD.js +86 -0
- package/dist/src/tracingOptions-BnwKCkSB.js +221 -0
- package/dist/src/tracingOptions-Chi74lOD.js +219 -0
- package/dist/src/tracingOptions-DrbSFaKy.cjs +249 -0
- package/dist/src/tracingOptions-ji2OuXbT.js +220 -0
- package/dist/src/{transcription-DuWDupG7.js → transcription-B8uIgCYX.js} +5 -5
- package/dist/src/{transcription-CJspiD2c.js → transcription-CfU5loSq.js} +6 -6
- package/dist/src/{transcription-V2HaAmy2.js → transcription-Dkd22_4K.js} +6 -6
- package/dist/src/{transcription-BvjmiYB1.cjs → transcription-mzuf18Mq.cjs} +9 -8
- package/dist/src/{transform-lQrDE1BQ.js → transform-BIMynQsA.js} +9 -9
- package/dist/src/transform-BnSTnFlp.js +187 -0
- package/dist/src/transform-BnSXWmU_2.cjs +221 -0
- package/dist/src/transform-CGt7Kt3y2.js +186 -0
- package/dist/src/transform-CrPGTsij.js +186 -0
- package/dist/src/{transform-CTeuTR3S.cjs → transform-DhNkAUs8.cjs} +13 -12
- package/dist/src/{transform-CG0ehZNG.js → transform-DmvYBRll.js} +9 -9
- package/dist/src/{transform-zDhMmzwX.js → transform-EtD4jAWi.js} +9 -9
- package/dist/src/{transformersAvailability-CcHusyhw.js → transformersAvailability-0ThtPved.js} +1 -1
- package/dist/src/transformersAvailability-BYydDE5U.js +35 -0
- package/dist/src/{transformersAvailability-DLlROWhg.js → transformersAvailability-BvyU9vDD.js} +1 -1
- package/dist/src/{transformersAvailability-Cju9mHgR.cjs → transformersAvailability-BytPvKUW.cjs} +1 -1
- package/dist/src/{types-Dm9JM6Vb.js → types-BFevViUY.js} +115 -19
- package/dist/src/{types-Bgh5SOn6.js → types-BJQBBPTP.js} +115 -19
- package/dist/src/{types-CeaeaZdP.cjs → types-CxJvaY2S.cjs} +357 -172
- package/dist/src/{types-BGQDAP8i.js → types-D6glLbdF.js} +271 -170
- package/dist/src/{util-BYvQUPp7.js → util--WMgw7wM.js} +28 -8
- package/dist/src/{util-C9J8ahRn.js → util-5WnCSb0h.js} +72 -48
- package/dist/src/{util-CN3SrLT4.cjs → util-BSIuSLVK.cjs} +74 -49
- package/dist/src/{util-C8e5uydV.js → util-Bx677_k2.js} +154 -147
- package/dist/src/util-CN8om2rz.cjs +386 -0
- package/dist/src/{util-DDs-7g6-.js → util-CoQWM76y.js} +28 -8
- package/dist/src/util-DNl96nNs.js +327 -0
- package/dist/src/{util-DxWpWjhc.js → util-DURocbYR.js} +667 -507
- package/dist/src/util-Df8YMvS1.js +327 -0
- package/dist/src/{util-DvU2Pw8c.js → util-DiQ3QvBB.js} +28 -8
- package/dist/src/{util-oGMLA7vc.js → util-I-Rf-KaD.js} +862 -577
- package/dist/src/{util-olYL5C6N.cjs → util-IYzs5Y04.cjs} +33 -7
- package/dist/src/{util-D9TisOyk.js → util-LKTmNsMQ.js} +71 -47
- package/dist/src/{util-Bxn8emtE.cjs → util-SPsvFONY.cjs} +738 -582
- package/dist/src/{util-D3q0WQ-0.js → util-efByNxcr.js} +72 -48
- package/dist/src/util-kDURhgJW.js +328 -0
- package/dist/src/{utils-DJfvjyMj.js → utils-B0lzitHZ.js} +3 -3
- package/dist/src/{utils-BLJKfv0y.js → utils-BFOh20Gb.js} +3 -3
- package/dist/src/{utils-hXtCYanr.js → utils-BGY69tk_.js} +2 -2
- package/dist/src/{utils-B05gLxER.cjs → utils-Ve6kuJsa.cjs} +3 -3
- package/dist/src/version-BK20a4sw.js +16 -0
- package/dist/src/version-BWCSaByA.cjs +27 -0
- package/dist/src/version-eRkNuGv8.js +17 -0
- package/dist/src/version-lpHV_53E.js +16 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +56 -28
- package/dist/src/app/assets/Report-CQYFezYu.js +0 -1
- package/dist/src/app/assets/index-BXGkeMwh.css +0 -1
- package/dist/src/app/assets/index-BzJt18Jz.js +0 -385
- package/dist/src/app/assets/sync-IjzpWrOE.js +0 -4
- package/dist/src/app/assets/vendor-charts-BNdH8TCw.js +0 -36
- package/dist/src/cache-Cr9oLMUa.js +0 -3
- package/dist/src/cache-DbLsVWB2.cjs +0 -3
- package/dist/src/cloud-Hphvo8kr.js +0 -3
- package/dist/src/codex-sdk-BAmYE7qy.js +0 -3
- package/dist/src/codex-sdk-CWEnH70W.cjs +0 -2
- package/dist/src/evalResult-D8MT9p0s.js +0 -3
- package/dist/src/evalResult-DElBuddX.js +0 -2
- package/dist/src/evalResult-Dvc-iucu.cjs +0 -2
- package/dist/src/evaluator-CVessDWe.js +0 -3
- package/dist/src/fetch-C7bGKDlQ.js +0 -3
- package/dist/src/graders-BOAzQEUe.cjs +0 -2
- package/dist/src/graders-D4BTsZdG2.js +0 -3
- package/dist/src/graders-DOJK1XpV.js +0 -2
- package/dist/src/graders-NAv9LcBn.js +0 -2
- package/dist/src/image-B5Mv-Z3h.js +0 -257
- package/dist/src/image-DVz2RiMF.js +0 -258
- package/dist/src/image-qUpPvmNZ.js +0 -257
- package/dist/src/image-x6KqLQl4.cjs +0 -280
- package/dist/src/providers-Bp4S-FvO.js +0 -2
- package/dist/src/providers-DV3ax9e_.cjs +0 -3
- package/dist/src/providers-u9Enmfok.js +0 -2
- package/dist/src/render-CH-62LbA.js +0 -135
- package/dist/src/render-CMEpfLaO.js +0 -136
- package/dist/src/render-DHIZ6_k8.js +0 -135
- package/dist/src/render-DfQSFxGE.cjs +0 -165
- package/dist/src/rubyUtils-D1L2d3jb.js +0 -3
- package/dist/src/rubyUtils-DUbq4tff.cjs +0 -2
- package/dist/src/server-BNYztJkh.js +0 -385
- package/dist/src/server-DCtHUqlp.js +0 -3
- package/dist/src/server-DaA2eR26.cjs +0 -2
- package/dist/src/store-CWOSz6D_.cjs +0 -2
- package/dist/src/store-DCDBhv7B.js +0 -3
- package/dist/src/store-Dn9HUkdW.js +0 -240
- package/dist/src/telemetry-C1IqxcdW.js +0 -3
- package/dist/src/telemetry-C4ZEa_es.cjs +0 -2
- package/dist/src/transform-Bbg6A8Jk.js +0 -216
- package/dist/src/transform-CUnzlsbn.cjs +0 -228
- package/dist/src/transform-DYX1_Xnh.js +0 -216
- package/dist/src/transform-DgKlRr73.cjs +0 -2
- package/dist/src/transform-M6ITAESf.js +0 -3
- package/dist/src/transform-UN5UGu8U.js +0 -213
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { s as logger } from "./logger-BbY6ypFL.js";
|
|
3
|
+
import { n as fetchWithProxy } from "./fetch-DXUnXkVU.js";
|
|
4
|
+
import { n as VERSION } from "./version-eRkNuGv8.js";
|
|
5
|
+
import { t as invariant } from "./invariant-B2Rf6avk.js";
|
|
6
|
+
import { o as getUserEmail } from "./accounts-CjFnOPmb.js";
|
|
7
|
+
import { n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-B1_XsKXU.js";
|
|
8
|
+
import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CDet74yk.js";
|
|
9
|
+
import { o as getSessionId } from "./util-kDURhgJW.js";
|
|
10
|
+
import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-4X-Wd8PG.js";
|
|
11
|
+
import { n as renderPrompt } from "./evaluatorHelpers-BsYP_muT.js";
|
|
12
|
+
import chalk from "chalk";
|
|
13
|
+
import dedent from "dedent";
|
|
14
|
+
import async from "async";
|
|
15
|
+
//#region src/redteam/providers/bestOfN.ts
|
|
16
|
+
var BestOfNProvider = class {
|
|
17
|
+
config;
|
|
18
|
+
id() {
|
|
19
|
+
return "promptfoo:redteam:best-of-n";
|
|
20
|
+
}
|
|
21
|
+
constructor(options = {}) {
|
|
22
|
+
if (neverGenerateRemote()) throw new Error(getRemoteGenerationExplicitlyDisabledError("Best-of-N strategy"));
|
|
23
|
+
invariant(typeof options.injectVar === "string", "Expected injectVar to be set");
|
|
24
|
+
this.config = {
|
|
25
|
+
injectVar: options.injectVar,
|
|
26
|
+
maxConcurrency: options.maxConcurrency || 3,
|
|
27
|
+
nSteps: options.nSteps,
|
|
28
|
+
maxCandidatesPerStep: options.maxCandidatesPerStep
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
async callApi(_prompt, context, options) {
|
|
32
|
+
logger.debug("[Best-of-N] callApi context", { context });
|
|
33
|
+
invariant(context?.originalProvider, "Expected originalProvider to be set");
|
|
34
|
+
invariant(context?.vars, "Expected vars to be set");
|
|
35
|
+
const targetProvider = context.originalProvider;
|
|
36
|
+
const targetTokenUsage = createEmptyTokenUsage();
|
|
37
|
+
const sessionIds = [];
|
|
38
|
+
try {
|
|
39
|
+
const data = await (await fetchWithProxy(getRemoteGenerationUrl(), {
|
|
40
|
+
method: "POST",
|
|
41
|
+
headers: { "Content-Type": "application/json" },
|
|
42
|
+
body: JSON.stringify({
|
|
43
|
+
task: "jailbreak:best-of-n",
|
|
44
|
+
prompt: context.vars[this.config.injectVar],
|
|
45
|
+
nSteps: this.config.nSteps,
|
|
46
|
+
maxCandidatesPerStep: this.config.maxCandidatesPerStep,
|
|
47
|
+
version: VERSION,
|
|
48
|
+
email: getUserEmail()
|
|
49
|
+
})
|
|
50
|
+
}, options?.abortSignal)).json();
|
|
51
|
+
invariant(Array.isArray(data.modifiedPrompts), "Expected modifiedPrompts array in response");
|
|
52
|
+
logger.debug(dedent`
|
|
53
|
+
${chalk.bold.green("Best-of-N candidates:")}
|
|
54
|
+
${chalk.cyan(JSON.stringify(data.modifiedPrompts, null, 2))}
|
|
55
|
+
`);
|
|
56
|
+
let successfulResponse = null;
|
|
57
|
+
let lastResponse = null;
|
|
58
|
+
let currentStep = 0;
|
|
59
|
+
await async.eachLimit(data.modifiedPrompts, this.config.maxConcurrency, async (candidatePrompt) => {
|
|
60
|
+
if (successfulResponse) return;
|
|
61
|
+
if (typeof candidatePrompt !== "string") {
|
|
62
|
+
logger.warn("[Best-of-N] Skipping non-string candidate prompt from remote generation", {
|
|
63
|
+
component: "Best-of-N",
|
|
64
|
+
event: "SkippingCandidatePrompt",
|
|
65
|
+
reason: "non-string",
|
|
66
|
+
candidatePromptType: typeof candidatePrompt
|
|
67
|
+
});
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
const unsafeCandidateScheme = /^\s*(file:\/\/|package:)/i.exec(candidatePrompt)?.[1].toLowerCase();
|
|
71
|
+
if (unsafeCandidateScheme) {
|
|
72
|
+
const schemeLabel = unsafeCandidateScheme.startsWith("file") ? "file://" : "package:";
|
|
73
|
+
logger.warn(`[Best-of-N] Skipping unsafe ${schemeLabel} candidate prompt from remote generation`, {
|
|
74
|
+
component: "Best-of-N",
|
|
75
|
+
event: "SkippingCandidatePrompt",
|
|
76
|
+
reason: schemeLabel === "file://" ? "unsafe-file-protocol" : "unsafe-package-protocol"
|
|
77
|
+
});
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const targetVars = {
|
|
81
|
+
...context.vars,
|
|
82
|
+
[this.config.injectVar]: candidatePrompt
|
|
83
|
+
};
|
|
84
|
+
const renderedPrompt = await renderPrompt(context.prompt, targetVars, context.filters, targetProvider, [this.config.injectVar]);
|
|
85
|
+
try {
|
|
86
|
+
throwIfTargetPromptExceedsMaxChars(renderedPrompt);
|
|
87
|
+
const response = await targetProvider.callApi(renderedPrompt, context, options);
|
|
88
|
+
const sessionId = getSessionId(response, context);
|
|
89
|
+
if (sessionId) sessionIds.push(sessionId);
|
|
90
|
+
lastResponse = response;
|
|
91
|
+
accumulateResponseTokenUsage(targetTokenUsage, response);
|
|
92
|
+
currentStep++;
|
|
93
|
+
if (!response.error) {
|
|
94
|
+
successfulResponse = response;
|
|
95
|
+
successfulResponse.prompt = candidatePrompt;
|
|
96
|
+
successfulResponse.metadata = {
|
|
97
|
+
...successfulResponse.metadata,
|
|
98
|
+
redteamFinalPrompt: candidatePrompt,
|
|
99
|
+
step: currentStep
|
|
100
|
+
};
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
} catch (err) {
|
|
104
|
+
logger.debug(`[Best-of-N] Candidate failed: ${err}`);
|
|
105
|
+
lastResponse = { error: String(err) };
|
|
106
|
+
currentStep++;
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
if (successfulResponse) {
|
|
110
|
+
successfulResponse.tokenUsage = targetTokenUsage;
|
|
111
|
+
return successfulResponse;
|
|
112
|
+
}
|
|
113
|
+
if (lastResponse) {
|
|
114
|
+
lastResponse.tokenUsage = targetTokenUsage;
|
|
115
|
+
lastResponse.metadata = {
|
|
116
|
+
...lastResponse.metadata ?? {},
|
|
117
|
+
sessionIds
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
return lastResponse || {
|
|
121
|
+
error: "All candidates failed",
|
|
122
|
+
metadata: { sessionIds }
|
|
123
|
+
};
|
|
124
|
+
} catch (err) {
|
|
125
|
+
if (err instanceof Error && err.name === "AbortError") throw err;
|
|
126
|
+
logger.error(`[Best-of-N] Error: ${err}`);
|
|
127
|
+
return {
|
|
128
|
+
error: String(err),
|
|
129
|
+
metadata: { sessionIds }
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
//#endregion
|
|
135
|
+
export { BestOfNProvider as default };
|
|
136
|
+
|
|
137
|
+
//# sourceMappingURL=bestOfN-B3wNzjSB.js.map
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { a as logger } from "./logger-KD8JjCRJ.js";
|
|
2
|
+
import { t as fetchWithProxy } from "./fetch-BufrQtvR.js";
|
|
3
|
+
import { n as VERSION } from "./version-BK20a4sw.js";
|
|
4
|
+
import { t as invariant } from "./invariant-DIYf9sP1.js";
|
|
5
|
+
import { o as getUserEmail } from "./accounts-DanM1wq_.js";
|
|
6
|
+
import { n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-COpWcmWd.js";
|
|
7
|
+
import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-BjVkdk18.js";
|
|
8
|
+
import { o as getSessionId } from "./util-Df8YMvS1.js";
|
|
9
|
+
import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-0qIHyhA5.js";
|
|
10
|
+
import { n as renderPrompt } from "./server/index.js";
|
|
11
|
+
import chalk from "chalk";
|
|
12
|
+
import dedent from "dedent";
|
|
13
|
+
import async from "async";
|
|
14
|
+
//#region src/redteam/providers/bestOfN.ts
|
|
15
|
+
var BestOfNProvider = class {
|
|
16
|
+
config;
|
|
17
|
+
id() {
|
|
18
|
+
return "promptfoo:redteam:best-of-n";
|
|
19
|
+
}
|
|
20
|
+
constructor(options = {}) {
|
|
21
|
+
if (neverGenerateRemote()) throw new Error(getRemoteGenerationExplicitlyDisabledError("Best-of-N strategy"));
|
|
22
|
+
invariant(typeof options.injectVar === "string", "Expected injectVar to be set");
|
|
23
|
+
this.config = {
|
|
24
|
+
injectVar: options.injectVar,
|
|
25
|
+
maxConcurrency: options.maxConcurrency || 3,
|
|
26
|
+
nSteps: options.nSteps,
|
|
27
|
+
maxCandidatesPerStep: options.maxCandidatesPerStep
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
async callApi(_prompt, context, options) {
|
|
31
|
+
logger.debug("[Best-of-N] callApi context", { context });
|
|
32
|
+
invariant(context?.originalProvider, "Expected originalProvider to be set");
|
|
33
|
+
invariant(context?.vars, "Expected vars to be set");
|
|
34
|
+
const targetProvider = context.originalProvider;
|
|
35
|
+
const targetTokenUsage = createEmptyTokenUsage();
|
|
36
|
+
const sessionIds = [];
|
|
37
|
+
try {
|
|
38
|
+
const data = await (await fetchWithProxy(getRemoteGenerationUrl(), {
|
|
39
|
+
method: "POST",
|
|
40
|
+
headers: { "Content-Type": "application/json" },
|
|
41
|
+
body: JSON.stringify({
|
|
42
|
+
task: "jailbreak:best-of-n",
|
|
43
|
+
prompt: context.vars[this.config.injectVar],
|
|
44
|
+
nSteps: this.config.nSteps,
|
|
45
|
+
maxCandidatesPerStep: this.config.maxCandidatesPerStep,
|
|
46
|
+
version: VERSION,
|
|
47
|
+
email: getUserEmail()
|
|
48
|
+
})
|
|
49
|
+
}, options?.abortSignal)).json();
|
|
50
|
+
invariant(Array.isArray(data.modifiedPrompts), "Expected modifiedPrompts array in response");
|
|
51
|
+
logger.debug(dedent`
|
|
52
|
+
${chalk.bold.green("Best-of-N candidates:")}
|
|
53
|
+
${chalk.cyan(JSON.stringify(data.modifiedPrompts, null, 2))}
|
|
54
|
+
`);
|
|
55
|
+
let successfulResponse = null;
|
|
56
|
+
let lastResponse = null;
|
|
57
|
+
let currentStep = 0;
|
|
58
|
+
await async.eachLimit(data.modifiedPrompts, this.config.maxConcurrency, async (candidatePrompt) => {
|
|
59
|
+
if (successfulResponse) return;
|
|
60
|
+
if (typeof candidatePrompt !== "string") {
|
|
61
|
+
logger.warn("[Best-of-N] Skipping non-string candidate prompt from remote generation", {
|
|
62
|
+
component: "Best-of-N",
|
|
63
|
+
event: "SkippingCandidatePrompt",
|
|
64
|
+
reason: "non-string",
|
|
65
|
+
candidatePromptType: typeof candidatePrompt
|
|
66
|
+
});
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const unsafeCandidateScheme = /^\s*(file:\/\/|package:)/i.exec(candidatePrompt)?.[1].toLowerCase();
|
|
70
|
+
if (unsafeCandidateScheme) {
|
|
71
|
+
const schemeLabel = unsafeCandidateScheme.startsWith("file") ? "file://" : "package:";
|
|
72
|
+
logger.warn(`[Best-of-N] Skipping unsafe ${schemeLabel} candidate prompt from remote generation`, {
|
|
73
|
+
component: "Best-of-N",
|
|
74
|
+
event: "SkippingCandidatePrompt",
|
|
75
|
+
reason: schemeLabel === "file://" ? "unsafe-file-protocol" : "unsafe-package-protocol"
|
|
76
|
+
});
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const targetVars = {
|
|
80
|
+
...context.vars,
|
|
81
|
+
[this.config.injectVar]: candidatePrompt
|
|
82
|
+
};
|
|
83
|
+
const renderedPrompt = await renderPrompt(context.prompt, targetVars, context.filters, targetProvider, [this.config.injectVar]);
|
|
84
|
+
try {
|
|
85
|
+
throwIfTargetPromptExceedsMaxChars(renderedPrompt);
|
|
86
|
+
const response = await targetProvider.callApi(renderedPrompt, context, options);
|
|
87
|
+
const sessionId = getSessionId(response, context);
|
|
88
|
+
if (sessionId) sessionIds.push(sessionId);
|
|
89
|
+
lastResponse = response;
|
|
90
|
+
accumulateResponseTokenUsage(targetTokenUsage, response);
|
|
91
|
+
currentStep++;
|
|
92
|
+
if (!response.error) {
|
|
93
|
+
successfulResponse = response;
|
|
94
|
+
successfulResponse.prompt = candidatePrompt;
|
|
95
|
+
successfulResponse.metadata = {
|
|
96
|
+
...successfulResponse.metadata,
|
|
97
|
+
redteamFinalPrompt: candidatePrompt,
|
|
98
|
+
step: currentStep
|
|
99
|
+
};
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
} catch (err) {
|
|
103
|
+
logger.debug(`[Best-of-N] Candidate failed: ${err}`);
|
|
104
|
+
lastResponse = { error: String(err) };
|
|
105
|
+
currentStep++;
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
if (successfulResponse) {
|
|
109
|
+
successfulResponse.tokenUsage = targetTokenUsage;
|
|
110
|
+
return successfulResponse;
|
|
111
|
+
}
|
|
112
|
+
if (lastResponse) {
|
|
113
|
+
lastResponse.tokenUsage = targetTokenUsage;
|
|
114
|
+
lastResponse.metadata = {
|
|
115
|
+
...lastResponse.metadata ?? {},
|
|
116
|
+
sessionIds
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
return lastResponse || {
|
|
120
|
+
error: "All candidates failed",
|
|
121
|
+
metadata: { sessionIds }
|
|
122
|
+
};
|
|
123
|
+
} catch (err) {
|
|
124
|
+
if (err instanceof Error && err.name === "AbortError") throw err;
|
|
125
|
+
logger.error(`[Best-of-N] Error: ${err}`);
|
|
126
|
+
return {
|
|
127
|
+
error: String(err),
|
|
128
|
+
metadata: { sessionIds }
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
//#endregion
|
|
134
|
+
export { BestOfNProvider as default };
|
|
135
|
+
|
|
136
|
+
//# sourceMappingURL=bestOfN-BBsO41z4.js.map
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
|
|
2
|
+
const require_logger = require("./logger-cfNpzI4o.cjs");
|
|
3
|
+
const require_invariant = require("./invariant-QtnLD03y.cjs");
|
|
4
|
+
const require_fetch = require("./fetch-Dw4XZHjj.cjs");
|
|
5
|
+
const require_version = require("./version-BWCSaByA.cjs");
|
|
6
|
+
const require_accounts = require("./accounts-CmWzeD2d.cjs");
|
|
7
|
+
const require_remoteGeneration = require("./remoteGeneration-DS9N3pgB.cjs");
|
|
8
|
+
const require_tokenUsageUtils = require("./tokenUsageUtils-_B-P8IAi.cjs");
|
|
9
|
+
const require_util = require("./util-CN8om2rz.cjs");
|
|
10
|
+
const require_promptLength = require("./promptLength-BbBbDHNj.cjs");
|
|
11
|
+
const require_evaluatorHelpers = require("./evaluatorHelpers-CRqTvSux.cjs");
|
|
12
|
+
let async = require("async");
|
|
13
|
+
async = require_rolldown_runtime.__toESM(async, 1);
|
|
14
|
+
let chalk = require("chalk");
|
|
15
|
+
chalk = require_rolldown_runtime.__toESM(chalk, 1);
|
|
16
|
+
let dedent = require("dedent");
|
|
17
|
+
dedent = require_rolldown_runtime.__toESM(dedent, 1);
|
|
18
|
+
//#region src/redteam/providers/bestOfN.ts
|
|
19
|
+
var BestOfNProvider = class {
|
|
20
|
+
config;
|
|
21
|
+
id() {
|
|
22
|
+
return "promptfoo:redteam:best-of-n";
|
|
23
|
+
}
|
|
24
|
+
constructor(options = {}) {
|
|
25
|
+
if (require_remoteGeneration.neverGenerateRemote()) throw new Error(require_remoteGeneration.getRemoteGenerationExplicitlyDisabledError("Best-of-N strategy"));
|
|
26
|
+
require_invariant.invariant(typeof options.injectVar === "string", "Expected injectVar to be set");
|
|
27
|
+
this.config = {
|
|
28
|
+
injectVar: options.injectVar,
|
|
29
|
+
maxConcurrency: options.maxConcurrency || 3,
|
|
30
|
+
nSteps: options.nSteps,
|
|
31
|
+
maxCandidatesPerStep: options.maxCandidatesPerStep
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
async callApi(_prompt, context, options) {
|
|
35
|
+
require_logger.logger.debug("[Best-of-N] callApi context", { context });
|
|
36
|
+
require_invariant.invariant(context?.originalProvider, "Expected originalProvider to be set");
|
|
37
|
+
require_invariant.invariant(context?.vars, "Expected vars to be set");
|
|
38
|
+
const targetProvider = context.originalProvider;
|
|
39
|
+
const targetTokenUsage = require_tokenUsageUtils.createEmptyTokenUsage();
|
|
40
|
+
const sessionIds = [];
|
|
41
|
+
try {
|
|
42
|
+
const data = await (await require_fetch.fetchWithProxy(require_remoteGeneration.getRemoteGenerationUrl(), {
|
|
43
|
+
method: "POST",
|
|
44
|
+
headers: { "Content-Type": "application/json" },
|
|
45
|
+
body: JSON.stringify({
|
|
46
|
+
task: "jailbreak:best-of-n",
|
|
47
|
+
prompt: context.vars[this.config.injectVar],
|
|
48
|
+
nSteps: this.config.nSteps,
|
|
49
|
+
maxCandidatesPerStep: this.config.maxCandidatesPerStep,
|
|
50
|
+
version: require_version.VERSION,
|
|
51
|
+
email: require_accounts.getUserEmail()
|
|
52
|
+
})
|
|
53
|
+
}, options?.abortSignal)).json();
|
|
54
|
+
require_invariant.invariant(Array.isArray(data.modifiedPrompts), "Expected modifiedPrompts array in response");
|
|
55
|
+
require_logger.logger.debug(dedent.default`
|
|
56
|
+
${chalk.default.bold.green("Best-of-N candidates:")}
|
|
57
|
+
${chalk.default.cyan(JSON.stringify(data.modifiedPrompts, null, 2))}
|
|
58
|
+
`);
|
|
59
|
+
let successfulResponse = null;
|
|
60
|
+
let lastResponse = null;
|
|
61
|
+
let currentStep = 0;
|
|
62
|
+
await async.default.eachLimit(data.modifiedPrompts, this.config.maxConcurrency, async (candidatePrompt) => {
|
|
63
|
+
if (successfulResponse) return;
|
|
64
|
+
if (typeof candidatePrompt !== "string") {
|
|
65
|
+
require_logger.logger.warn("[Best-of-N] Skipping non-string candidate prompt from remote generation", {
|
|
66
|
+
component: "Best-of-N",
|
|
67
|
+
event: "SkippingCandidatePrompt",
|
|
68
|
+
reason: "non-string",
|
|
69
|
+
candidatePromptType: typeof candidatePrompt
|
|
70
|
+
});
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const unsafeCandidateScheme = /^\s*(file:\/\/|package:)/i.exec(candidatePrompt)?.[1].toLowerCase();
|
|
74
|
+
if (unsafeCandidateScheme) {
|
|
75
|
+
const schemeLabel = unsafeCandidateScheme.startsWith("file") ? "file://" : "package:";
|
|
76
|
+
require_logger.logger.warn(`[Best-of-N] Skipping unsafe ${schemeLabel} candidate prompt from remote generation`, {
|
|
77
|
+
component: "Best-of-N",
|
|
78
|
+
event: "SkippingCandidatePrompt",
|
|
79
|
+
reason: schemeLabel === "file://" ? "unsafe-file-protocol" : "unsafe-package-protocol"
|
|
80
|
+
});
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
const targetVars = {
|
|
84
|
+
...context.vars,
|
|
85
|
+
[this.config.injectVar]: candidatePrompt
|
|
86
|
+
};
|
|
87
|
+
const renderedPrompt = await require_evaluatorHelpers.renderPrompt(context.prompt, targetVars, context.filters, targetProvider, [this.config.injectVar]);
|
|
88
|
+
try {
|
|
89
|
+
require_promptLength.throwIfTargetPromptExceedsMaxChars(renderedPrompt);
|
|
90
|
+
const response = await targetProvider.callApi(renderedPrompt, context, options);
|
|
91
|
+
const sessionId = require_util.getSessionId(response, context);
|
|
92
|
+
if (sessionId) sessionIds.push(sessionId);
|
|
93
|
+
lastResponse = response;
|
|
94
|
+
require_tokenUsageUtils.accumulateResponseTokenUsage(targetTokenUsage, response);
|
|
95
|
+
currentStep++;
|
|
96
|
+
if (!response.error) {
|
|
97
|
+
successfulResponse = response;
|
|
98
|
+
successfulResponse.prompt = candidatePrompt;
|
|
99
|
+
successfulResponse.metadata = {
|
|
100
|
+
...successfulResponse.metadata,
|
|
101
|
+
redteamFinalPrompt: candidatePrompt,
|
|
102
|
+
step: currentStep
|
|
103
|
+
};
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
} catch (err) {
|
|
107
|
+
require_logger.logger.debug(`[Best-of-N] Candidate failed: ${err}`);
|
|
108
|
+
lastResponse = { error: String(err) };
|
|
109
|
+
currentStep++;
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
if (successfulResponse) {
|
|
113
|
+
successfulResponse.tokenUsage = targetTokenUsage;
|
|
114
|
+
return successfulResponse;
|
|
115
|
+
}
|
|
116
|
+
if (lastResponse) {
|
|
117
|
+
lastResponse.tokenUsage = targetTokenUsage;
|
|
118
|
+
lastResponse.metadata = {
|
|
119
|
+
...lastResponse.metadata ?? {},
|
|
120
|
+
sessionIds
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
return lastResponse || {
|
|
124
|
+
error: "All candidates failed",
|
|
125
|
+
metadata: { sessionIds }
|
|
126
|
+
};
|
|
127
|
+
} catch (err) {
|
|
128
|
+
if (err instanceof Error && err.name === "AbortError") throw err;
|
|
129
|
+
require_logger.logger.error(`[Best-of-N] Error: ${err}`);
|
|
130
|
+
return {
|
|
131
|
+
error: String(err),
|
|
132
|
+
metadata: { sessionIds }
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
//#endregion
|
|
138
|
+
exports.default = BestOfNProvider;
|
|
139
|
+
|
|
140
|
+
//# sourceMappingURL=bestOfN-CAwmg5UL.cjs.map
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { a as logger } from "./logger-Ct2S6Yx-.js";
|
|
2
|
+
import { t as invariant } from "./invariant-Ddh24eXh.js";
|
|
3
|
+
import { t as fetchWithProxy } from "./fetch-It34O8Ur.js";
|
|
4
|
+
import { n as VERSION } from "./version-lpHV_53E.js";
|
|
5
|
+
import { i as getUserEmail } from "./accounts-Ca7WIoPY.js";
|
|
6
|
+
import { n as getRemoteGenerationExplicitlyDisabledError, r as getRemoteGenerationUrl, s as neverGenerateRemote } from "./remoteGeneration-DsaSwmG2.js";
|
|
7
|
+
import { a as createEmptyTokenUsage, n as accumulateResponseTokenUsage } from "./tokenUsageUtils-CmnQ0G2m.js";
|
|
8
|
+
import { o as getSessionId } from "./util-DNl96nNs.js";
|
|
9
|
+
import { i as throwIfTargetPromptExceedsMaxChars } from "./promptLength-B9nZEfO6.js";
|
|
10
|
+
import { n as renderPrompt } from "./evaluatorHelpers-DuqFFfq7.js";
|
|
11
|
+
import async from "async";
|
|
12
|
+
import chalk from "chalk";
|
|
13
|
+
import dedent from "dedent";
|
|
14
|
+
//#region src/redteam/providers/bestOfN.ts
|
|
15
|
+
var BestOfNProvider = class {
|
|
16
|
+
config;
|
|
17
|
+
id() {
|
|
18
|
+
return "promptfoo:redteam:best-of-n";
|
|
19
|
+
}
|
|
20
|
+
constructor(options = {}) {
|
|
21
|
+
if (neverGenerateRemote()) throw new Error(getRemoteGenerationExplicitlyDisabledError("Best-of-N strategy"));
|
|
22
|
+
invariant(typeof options.injectVar === "string", "Expected injectVar to be set");
|
|
23
|
+
this.config = {
|
|
24
|
+
injectVar: options.injectVar,
|
|
25
|
+
maxConcurrency: options.maxConcurrency || 3,
|
|
26
|
+
nSteps: options.nSteps,
|
|
27
|
+
maxCandidatesPerStep: options.maxCandidatesPerStep
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
async callApi(_prompt, context, options) {
|
|
31
|
+
logger.debug("[Best-of-N] callApi context", { context });
|
|
32
|
+
invariant(context?.originalProvider, "Expected originalProvider to be set");
|
|
33
|
+
invariant(context?.vars, "Expected vars to be set");
|
|
34
|
+
const targetProvider = context.originalProvider;
|
|
35
|
+
const targetTokenUsage = createEmptyTokenUsage();
|
|
36
|
+
const sessionIds = [];
|
|
37
|
+
try {
|
|
38
|
+
const data = await (await fetchWithProxy(getRemoteGenerationUrl(), {
|
|
39
|
+
method: "POST",
|
|
40
|
+
headers: { "Content-Type": "application/json" },
|
|
41
|
+
body: JSON.stringify({
|
|
42
|
+
task: "jailbreak:best-of-n",
|
|
43
|
+
prompt: context.vars[this.config.injectVar],
|
|
44
|
+
nSteps: this.config.nSteps,
|
|
45
|
+
maxCandidatesPerStep: this.config.maxCandidatesPerStep,
|
|
46
|
+
version: VERSION,
|
|
47
|
+
email: getUserEmail()
|
|
48
|
+
})
|
|
49
|
+
}, options?.abortSignal)).json();
|
|
50
|
+
invariant(Array.isArray(data.modifiedPrompts), "Expected modifiedPrompts array in response");
|
|
51
|
+
logger.debug(dedent`
|
|
52
|
+
${chalk.bold.green("Best-of-N candidates:")}
|
|
53
|
+
${chalk.cyan(JSON.stringify(data.modifiedPrompts, null, 2))}
|
|
54
|
+
`);
|
|
55
|
+
let successfulResponse = null;
|
|
56
|
+
let lastResponse = null;
|
|
57
|
+
let currentStep = 0;
|
|
58
|
+
await async.eachLimit(data.modifiedPrompts, this.config.maxConcurrency, async (candidatePrompt) => {
|
|
59
|
+
if (successfulResponse) return;
|
|
60
|
+
if (typeof candidatePrompt !== "string") {
|
|
61
|
+
logger.warn("[Best-of-N] Skipping non-string candidate prompt from remote generation", {
|
|
62
|
+
component: "Best-of-N",
|
|
63
|
+
event: "SkippingCandidatePrompt",
|
|
64
|
+
reason: "non-string",
|
|
65
|
+
candidatePromptType: typeof candidatePrompt
|
|
66
|
+
});
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const unsafeCandidateScheme = /^\s*(file:\/\/|package:)/i.exec(candidatePrompt)?.[1].toLowerCase();
|
|
70
|
+
if (unsafeCandidateScheme) {
|
|
71
|
+
const schemeLabel = unsafeCandidateScheme.startsWith("file") ? "file://" : "package:";
|
|
72
|
+
logger.warn(`[Best-of-N] Skipping unsafe ${schemeLabel} candidate prompt from remote generation`, {
|
|
73
|
+
component: "Best-of-N",
|
|
74
|
+
event: "SkippingCandidatePrompt",
|
|
75
|
+
reason: schemeLabel === "file://" ? "unsafe-file-protocol" : "unsafe-package-protocol"
|
|
76
|
+
});
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
const targetVars = {
|
|
80
|
+
...context.vars,
|
|
81
|
+
[this.config.injectVar]: candidatePrompt
|
|
82
|
+
};
|
|
83
|
+
const renderedPrompt = await renderPrompt(context.prompt, targetVars, context.filters, targetProvider, [this.config.injectVar]);
|
|
84
|
+
try {
|
|
85
|
+
throwIfTargetPromptExceedsMaxChars(renderedPrompt);
|
|
86
|
+
const response = await targetProvider.callApi(renderedPrompt, context, options);
|
|
87
|
+
const sessionId = getSessionId(response, context);
|
|
88
|
+
if (sessionId) sessionIds.push(sessionId);
|
|
89
|
+
lastResponse = response;
|
|
90
|
+
accumulateResponseTokenUsage(targetTokenUsage, response);
|
|
91
|
+
currentStep++;
|
|
92
|
+
if (!response.error) {
|
|
93
|
+
successfulResponse = response;
|
|
94
|
+
successfulResponse.prompt = candidatePrompt;
|
|
95
|
+
successfulResponse.metadata = {
|
|
96
|
+
...successfulResponse.metadata,
|
|
97
|
+
redteamFinalPrompt: candidatePrompt,
|
|
98
|
+
step: currentStep
|
|
99
|
+
};
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
} catch (err) {
|
|
103
|
+
logger.debug(`[Best-of-N] Candidate failed: ${err}`);
|
|
104
|
+
lastResponse = { error: String(err) };
|
|
105
|
+
currentStep++;
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
if (successfulResponse) {
|
|
109
|
+
successfulResponse.tokenUsage = targetTokenUsage;
|
|
110
|
+
return successfulResponse;
|
|
111
|
+
}
|
|
112
|
+
if (lastResponse) {
|
|
113
|
+
lastResponse.tokenUsage = targetTokenUsage;
|
|
114
|
+
lastResponse.metadata = {
|
|
115
|
+
...lastResponse.metadata ?? {},
|
|
116
|
+
sessionIds
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
return lastResponse || {
|
|
120
|
+
error: "All candidates failed",
|
|
121
|
+
metadata: { sessionIds }
|
|
122
|
+
};
|
|
123
|
+
} catch (err) {
|
|
124
|
+
if (err instanceof Error && err.name === "AbortError") throw err;
|
|
125
|
+
logger.error(`[Best-of-N] Error: ${err}`);
|
|
126
|
+
return {
|
|
127
|
+
error: String(err),
|
|
128
|
+
metadata: { sessionIds }
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
//#endregion
|
|
134
|
+
export { BestOfNProvider as default };
|
|
135
|
+
|
|
136
|
+
//# sourceMappingURL=bestOfN-_kTi8Bxe.js.map
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
const
|
|
2
|
-
const
|
|
1
|
+
const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
|
|
2
|
+
const require_logger = require("./logger-cfNpzI4o.cjs");
|
|
3
|
+
const require_tables = require("./tables-DBIJU0WE.cjs");
|
|
3
4
|
let node_fs = require("node:fs");
|
|
4
|
-
node_fs =
|
|
5
|
+
node_fs = require_rolldown_runtime.__toESM(node_fs, 1);
|
|
5
6
|
let node_fs_promises = require("node:fs/promises");
|
|
6
|
-
node_fs_promises =
|
|
7
|
+
node_fs_promises = require_rolldown_runtime.__toESM(node_fs_promises, 1);
|
|
7
8
|
let node_path = require("node:path");
|
|
8
|
-
node_path =
|
|
9
|
+
node_path = require_rolldown_runtime.__toESM(node_path, 1);
|
|
9
10
|
let node_crypto = require("node:crypto");
|
|
10
11
|
let drizzle_orm = require("drizzle-orm");
|
|
11
12
|
//#region src/blobs/constants.ts
|
|
@@ -262,4 +263,4 @@ Object.defineProperty(exports, "storeBlob", {
|
|
|
262
263
|
}
|
|
263
264
|
});
|
|
264
265
|
|
|
265
|
-
//# sourceMappingURL=blobs-
|
|
266
|
+
//# sourceMappingURL=blobs-B0977K1O.cjs.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { a as logger, x as getConfigDirectoryPath } from "./logger-
|
|
2
|
-
import { h as getDb, n as blobReferencesTable, t as blobAssetsTable } from "./tables-
|
|
1
|
+
import { a as logger, x as getConfigDirectoryPath } from "./logger-KD8JjCRJ.js";
|
|
2
|
+
import { h as getDb, n as blobReferencesTable, t as blobAssetsTable } from "./tables-CxjU7bBd.js";
|
|
3
3
|
import * as path$1 from "node:path";
|
|
4
4
|
import * as fs$1 from "node:fs";
|
|
5
5
|
import * as fsPromises$1 from "node:fs/promises";
|
|
@@ -233,4 +233,4 @@ async function recordBlobReference(hash, refContext) {
|
|
|
233
233
|
//#endregion
|
|
234
234
|
export { BLOB_MAX_SIZE as a, storeBlob as i, getBlobUrl as n, BLOB_MIN_SIZE as o, recordBlobReference as r, getBlobByHash as t };
|
|
235
235
|
|
|
236
|
-
//# sourceMappingURL=blobs-
|
|
236
|
+
//# sourceMappingURL=blobs-CeFdPn_T.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { C as getConfigDirectoryPath, s as logger } from "./logger-
|
|
3
|
-
import { g as getDb, n as blobReferencesTable, t as blobAssetsTable } from "./tables-
|
|
2
|
+
import { C as getConfigDirectoryPath, s as logger } from "./logger-BbY6ypFL.js";
|
|
3
|
+
import { g as getDb, n as blobReferencesTable, t as blobAssetsTable } from "./tables-DafUHOeh.js";
|
|
4
4
|
import * as path$1 from "node:path";
|
|
5
5
|
import * as fs$1 from "node:fs";
|
|
6
6
|
import * as fsPromises$1 from "node:fs/promises";
|
|
@@ -234,4 +234,4 @@ async function recordBlobReference(hash, refContext) {
|
|
|
234
234
|
//#endregion
|
|
235
235
|
export { BLOB_MAX_SIZE as a, storeBlob as i, getBlobUrl as n, BLOB_MIN_SIZE as o, recordBlobReference as r, getBlobByHash as t };
|
|
236
236
|
|
|
237
|
-
//# sourceMappingURL=blobs-
|
|
237
|
+
//# sourceMappingURL=blobs-DODuTK-a.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { a as logger, x as getConfigDirectoryPath } from "./logger-Ct2S6Yx-.js";
|
|
2
|
-
import { n as blobReferencesTable, p as getDb, t as blobAssetsTable } from "./tables-
|
|
2
|
+
import { n as blobReferencesTable, p as getDb, t as blobAssetsTable } from "./tables-CRSXQ2Ke.js";
|
|
3
3
|
import * as path$1 from "node:path";
|
|
4
4
|
import * as fs$1 from "node:fs";
|
|
5
5
|
import * as fsPromises$1 from "node:fs/promises";
|
|
@@ -230,4 +230,4 @@ async function recordBlobReference(hash, refContext) {
|
|
|
230
230
|
//#endregion
|
|
231
231
|
export { BLOB_MIN_SIZE as a, BLOB_MAX_SIZE as i, recordBlobReference as n, storeBlob as r, getBlobByHash as t };
|
|
232
232
|
|
|
233
|
-
//# sourceMappingURL=blobs-
|
|
233
|
+
//# sourceMappingURL=blobs-Dwef1Ao1.js.map
|