promptfoo 0.73.8 → 0.74.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -1
- package/dist/package.json +13 -13
- package/dist/src/assertions/AssertionsResult.d.ts +1 -1
- package/dist/src/assertions/AssertionsResult.d.ts.map +1 -1
- package/dist/src/assertions/validateAssertions.d.ts +1 -1
- package/dist/src/assertions/validateAssertions.d.ts.map +1 -1
- package/dist/src/assertions.d.ts +2 -1
- package/dist/src/assertions.d.ts.map +1 -1
- package/dist/src/assertions.js.map +1 -1
- package/dist/src/commands/cache.d.ts +1 -1
- package/dist/src/commands/cache.d.ts.map +1 -1
- package/dist/src/commands/config.d.ts +1 -1
- package/dist/src/commands/config.d.ts.map +1 -1
- package/dist/src/commands/delete.d.ts +1 -1
- package/dist/src/commands/delete.d.ts.map +1 -1
- package/dist/src/commands/eval/filterFailingTests.d.ts +1 -1
- package/dist/src/commands/eval/filterFailingTests.d.ts.map +1 -1
- package/dist/src/commands/eval/filterTests.d.ts +1 -1
- package/dist/src/commands/eval/filterTests.d.ts.map +1 -1
- package/dist/src/commands/eval.d.ts +2 -2
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/export.d.ts +1 -1
- package/dist/src/commands/export.d.ts.map +1 -1
- package/dist/src/commands/feedback.d.ts +1 -1
- package/dist/src/commands/feedback.d.ts.map +1 -1
- package/dist/src/commands/generate/dataset.d.ts +2 -2
- package/dist/src/commands/generate/dataset.d.ts.map +1 -1
- package/dist/src/commands/generate/redteam.d.ts +3 -3
- package/dist/src/commands/generate/redteam.d.ts.map +1 -1
- package/dist/src/commands/generate/redteam.js +13 -6
- package/dist/src/commands/generate/redteam.js.map +1 -1
- package/dist/src/commands/import.d.ts +1 -1
- package/dist/src/commands/import.d.ts.map +1 -1
- package/dist/src/commands/init.d.ts +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/list.d.ts +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/redteam.d.ts +2 -1
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/redteam.js +197 -191
- package/dist/src/commands/redteam.js.map +1 -1
- package/dist/src/commands/share.d.ts +1 -1
- package/dist/src/commands/share.d.ts.map +1 -1
- package/dist/src/commands/show.d.ts +1 -1
- package/dist/src/commands/show.d.ts.map +1 -1
- package/dist/src/commands/version.d.ts +1 -1
- package/dist/src/commands/version.d.ts.map +1 -1
- package/dist/src/commands/view.d.ts +1 -1
- package/dist/src/commands/view.d.ts.map +1 -1
- package/dist/src/config.d.ts +1 -1
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/database/operations.d.ts +56 -240
- package/dist/src/database/operations.d.ts.map +1 -1
- package/dist/src/database/types.d.ts +1 -1
- package/dist/src/database/types.d.ts.map +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +13 -4
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/fetch.d.ts.map +1 -1
- package/dist/src/fetch.js +28 -2
- package/dist/src/fetch.js.map +1 -1
- package/dist/src/globalConfig.d.ts +1 -1
- package/dist/src/globalConfig.d.ts.map +1 -1
- package/dist/src/onboarding.d.ts.map +1 -1
- package/dist/src/onboarding.js +11 -0
- package/dist/src/onboarding.js.map +1 -1
- package/dist/src/prompts/grading.js +1 -1
- package/dist/src/prompts/processors/javascript.d.ts +1 -1
- package/dist/src/prompts/processors/javascript.d.ts.map +1 -1
- package/dist/src/prompts/processors/json.d.ts +1 -1
- package/dist/src/prompts/processors/json.d.ts.map +1 -1
- package/dist/src/prompts/processors/jsonl.d.ts +1 -1
- package/dist/src/prompts/processors/jsonl.d.ts.map +1 -1
- package/dist/src/prompts/processors/python.d.ts.map +1 -1
- package/dist/src/prompts/processors/python.js.map +1 -1
- package/dist/src/prompts/processors/string.d.ts +1 -1
- package/dist/src/prompts/processors/string.d.ts.map +1 -1
- package/dist/src/prompts/processors/text.d.ts +1 -1
- package/dist/src/prompts/processors/text.d.ts.map +1 -1
- package/dist/src/prompts/processors/yaml.d.ts +1 -1
- package/dist/src/prompts/processors/yaml.d.ts.map +1 -1
- package/dist/src/prompts/utils.d.ts +1 -1
- package/dist/src/prompts/utils.d.ts.map +1 -1
- package/dist/src/providers/anthropic.js +1 -1
- package/dist/src/providers/anthropic.js.map +1 -1
- package/dist/src/providers/azureopenai.d.ts.map +1 -1
- package/dist/src/providers/azureopenai.js +6 -2
- package/dist/src/providers/azureopenai.js.map +1 -1
- package/dist/src/providers/bedrock.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +2 -2
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/providers/huggingface.d.ts +1 -1
- package/dist/src/providers/huggingface.d.ts.map +1 -1
- package/dist/src/providers/manualInput.d.ts +1 -1
- package/dist/src/providers/manualInput.d.ts.map +1 -1
- package/dist/src/providers/mistral.d.ts +1 -1
- package/dist/src/providers/mistral.d.ts.map +1 -1
- package/dist/src/providers/openai.d.ts +1 -1
- package/dist/src/providers/openai.d.ts.map +1 -1
- package/dist/src/providers/openai.js +8 -3
- package/dist/src/providers/openai.js.map +1 -1
- package/dist/src/providers/shared.d.ts +1 -0
- package/dist/src/providers/shared.d.ts.map +1 -1
- package/dist/src/providers/shared.js +47 -0
- package/dist/src/providers/shared.js.map +1 -1
- package/dist/src/providers/vertex.d.ts.map +1 -1
- package/dist/src/providers/vertex.js.map +1 -1
- package/dist/src/providers.d.ts +2 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/python/pythonUtils.d.ts.map +1 -1
- package/dist/src/python/pythonUtils.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +2 -1
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +5 -4
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/extraction/entities.d.ts +3 -0
- package/dist/src/redteam/extraction/entities.d.ts.map +1 -0
- package/dist/src/redteam/extraction/entities.js +29 -0
- package/dist/src/redteam/extraction/entities.js.map +1 -0
- package/dist/src/redteam/extraction/purpose.d.ts +3 -0
- package/dist/src/redteam/extraction/purpose.d.ts.map +1 -0
- package/dist/src/redteam/extraction/purpose.js +27 -0
- package/dist/src/redteam/extraction/purpose.js.map +1 -0
- package/dist/src/redteam/extraction/util.d.ts +4 -0
- package/dist/src/redteam/extraction/util.d.ts.map +1 -0
- package/dist/src/redteam/extraction/util.js +30 -0
- package/dist/src/redteam/extraction/util.js.map +1 -0
- package/dist/src/redteam/graders/base.d.ts +1 -1
- package/dist/src/redteam/graders/base.d.ts.map +1 -1
- package/dist/src/redteam/graders/base.js +1 -0
- package/dist/src/redteam/graders/base.js.map +1 -1
- package/dist/src/redteam/graders/hallucination.js +3 -3
- package/dist/src/redteam/graders/harmful.d.ts +1 -1
- package/dist/src/redteam/graders/harmful.d.ts.map +1 -1
- package/dist/src/redteam/graders/imitation.d.ts +6 -0
- package/dist/src/redteam/graders/imitation.d.ts.map +1 -0
- package/dist/src/redteam/graders/imitation.js +36 -0
- package/dist/src/redteam/graders/imitation.js.map +1 -0
- package/dist/src/redteam/graders/index.d.ts +1 -1
- package/dist/src/redteam/graders/index.d.ts.map +1 -1
- package/dist/src/redteam/graders/index.js +2 -0
- package/dist/src/redteam/graders/index.js.map +1 -1
- package/dist/src/redteam/index.d.ts +3 -3
- package/dist/src/redteam/index.d.ts.map +1 -1
- package/dist/src/redteam/index.js +42 -25
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts +5 -8
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +2 -4
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/competitors.d.ts +2 -2
- package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
- package/dist/src/redteam/plugins/competitors.js +2 -3
- package/dist/src/redteam/plugins/competitors.js.map +1 -1
- package/dist/src/redteam/plugins/contracts.d.ts +2 -2
- package/dist/src/redteam/plugins/contracts.d.ts.map +1 -1
- package/dist/src/redteam/plugins/contracts.js +2 -3
- package/dist/src/redteam/plugins/contracts.js.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.d.ts +2 -2
- package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.js +2 -3
- package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.d.ts +2 -2
- package/dist/src/redteam/plugins/excessiveAgency.d.ts.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.js +2 -3
- package/dist/src/redteam/plugins/excessiveAgency.js.map +1 -1
- package/dist/src/redteam/plugins/hallucination.d.ts +2 -2
- package/dist/src/redteam/plugins/hallucination.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hallucination.js +2 -3
- package/dist/src/redteam/plugins/hallucination.js.map +1 -1
- package/dist/src/redteam/plugins/harmful.d.ts +1 -2
- package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful.js +7 -5
- package/dist/src/redteam/plugins/harmful.js.map +1 -1
- package/dist/src/redteam/plugins/hijacking.d.ts +2 -2
- package/dist/src/redteam/plugins/hijacking.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hijacking.js +2 -3
- package/dist/src/redteam/plugins/hijacking.js.map +1 -1
- package/dist/src/redteam/plugins/imitation.d.ts +1 -7
- package/dist/src/redteam/plugins/imitation.d.ts.map +1 -1
- package/dist/src/redteam/plugins/imitation.js +3 -38
- package/dist/src/redteam/plugins/imitation.js.map +1 -1
- package/dist/src/redteam/plugins/overreliance.d.ts +2 -2
- package/dist/src/redteam/plugins/overreliance.d.ts.map +1 -1
- package/dist/src/redteam/plugins/overreliance.js +2 -3
- package/dist/src/redteam/plugins/overreliance.js.map +1 -1
- package/dist/src/redteam/plugins/politics.d.ts +2 -2
- package/dist/src/redteam/plugins/politics.d.ts.map +1 -1
- package/dist/src/redteam/plugins/politics.js +2 -3
- package/dist/src/redteam/plugins/politics.js.map +1 -1
- package/dist/src/redteam/plugins/rbac.d.ts +2 -2
- package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
- package/dist/src/redteam/plugins/rbac.js +2 -3
- package/dist/src/redteam/plugins/rbac.js.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.d.ts +2 -2
- package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.js +2 -3
- package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.d.ts +2 -2
- package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.js +2 -3
- package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
- package/dist/src/redteam/strategies/injections.d.ts +1 -1
- package/dist/src/redteam/strategies/injections.d.ts.map +1 -1
- package/dist/src/redteam/strategies/iterative.d.ts +1 -1
- package/dist/src/redteam/strategies/iterative.d.ts.map +1 -1
- package/dist/src/testCases.d.ts.map +1 -1
- package/dist/src/{types.d.ts → types/index.d.ts} +1496 -6518
- package/dist/src/{types.d.ts.map → types/index.d.ts.map} +1 -1
- package/dist/src/{types.js → types/index.js} +38 -172
- package/dist/src/types/index.js.map +1 -0
- package/dist/src/types/prompts.d.ts +29 -0
- package/dist/src/types/prompts.d.ts.map +1 -0
- package/dist/src/types/prompts.js +3 -0
- package/dist/src/types/prompts.js.map +1 -0
- package/dist/src/types/providers.d.ts +126 -0
- package/dist/src/types/providers.d.ts.map +1 -0
- package/dist/src/types/providers.js +11 -0
- package/dist/src/types/providers.js.map +1 -0
- package/dist/src/types/redteam.d.ts +51 -0
- package/dist/src/types/redteam.d.ts.map +1 -0
- package/dist/src/types/redteam.js +3 -0
- package/dist/src/types/redteam.js.map +1 -0
- package/dist/src/types/shared.d.ts +8 -0
- package/dist/src/types/shared.d.ts.map +1 -0
- package/dist/src/types/shared.js +3 -0
- package/dist/src/types/shared.js.map +1 -0
- package/dist/src/util/index.d.ts +20 -96
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +15 -8
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/templates.d.ts +1 -1
- package/dist/src/util/templates.d.ts.map +1 -1
- package/dist/src/util/transform.d.ts +1 -1
- package/dist/src/util/transform.d.ts.map +1 -1
- package/dist/src/validators/prompts.d.ts +60 -0
- package/dist/src/validators/prompts.d.ts.map +1 -0
- package/dist/src/validators/prompts.js +32 -0
- package/dist/src/validators/prompts.js.map +1 -0
- package/dist/src/validators/providers.d.ts +1145 -0
- package/dist/src/validators/providers.d.ts.map +1 -0
- package/dist/src/validators/providers.js +133 -0
- package/dist/src/validators/providers.js.map +1 -0
- package/dist/src/validators/redteam.d.ts +517 -0
- package/dist/src/validators/redteam.d.ts.map +1 -0
- package/dist/src/{redteam/types.js → validators/redteam.js} +31 -26
- package/dist/src/validators/redteam.js.map +1 -0
- package/dist/src/validators/shared.d.ts +19 -0
- package/dist/src/validators/shared.d.ts.map +1 -0
- package/dist/src/validators/shared.js +12 -0
- package/dist/src/validators/shared.js.map +1 -0
- package/dist/src/web/nextui/404/index.html +1 -1
- package/dist/src/web/nextui/404.html +1 -1
- package/dist/src/web/nextui/_next/static/chunks/258-280986e9c8fae219.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/{325-0d36870ade5e5263.js → 325-9b7b85306b746b1d.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/949-e778f1d346946d00.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/app/{layout-b24cf03b9a8a7440.js → layout-517c4420e852da35.js} +1 -1
- package/dist/src/web/nextui/_next/static/chunks/app/report/page-36e98231504ad757.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/main-app-0dc90092cd7acaaf.js +1 -0
- package/dist/src/web/nextui/_next/static/chunks/{webpack-455df73152fc7077.js → webpack-d3dba20c5c1ca02e.js} +1 -1
- package/dist/src/web/nextui/_next/static/css/32384cb640d2265e.css +1 -0
- package/dist/src/web/nextui/_next/static/css/{4463e29e24d44845.css → de0307797e31a3e6.css} +1 -1
- package/dist/src/web/nextui/_next/static/media/26a46d62cd723877-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/55c55f0601d81cf3-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/581909926a08bbc8-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/6d93bde91c0c2823-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/97e0cb1ae144a2a9-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/a34f9d1faa5f3315-s.p.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/df0a9ae256c0569c-s.woff2 +0 -0
- package/dist/src/web/nextui/auth/login/index.html +1 -1
- package/dist/src/web/nextui/auth/login/index.txt +7 -7
- package/dist/src/web/nextui/auth/signup/index.html +1 -1
- package/dist/src/web/nextui/auth/signup/index.txt +7 -7
- package/dist/src/web/nextui/datasets/index.html +1 -1
- package/dist/src/web/nextui/datasets/index.txt +7 -7
- package/dist/src/web/nextui/eval/index.html +1 -1
- package/dist/src/web/nextui/eval/index.txt +8 -8
- package/dist/src/web/nextui/index.html +1 -1
- package/dist/src/web/nextui/index.txt +7 -7
- package/dist/src/web/nextui/progress/index.html +1 -1
- package/dist/src/web/nextui/progress/index.txt +7 -7
- package/dist/src/web/nextui/prompts/index.html +1 -1
- package/dist/src/web/nextui/prompts/index.txt +7 -7
- package/dist/src/web/nextui/report/index.html +1 -1
- package/dist/src/web/nextui/report/index.txt +9 -9
- package/dist/src/web/nextui/setup/index.html +1 -1
- package/dist/src/web/nextui/setup/index.txt +8 -8
- package/dist/src/web/server.d.ts.map +1 -1
- package/dist/src/web/server.js.map +1 -1
- package/dist/test/assertions.test.js +12 -9
- package/dist/test/assertions.test.js.map +1 -1
- package/dist/test/cache.test.js.map +1 -1
- package/dist/test/evaluator.test.js +162 -0
- package/dist/test/evaluator.test.js.map +1 -1
- package/dist/test/providers.bedrock.test.js.map +1 -1
- package/dist/test/providers.shared.test.d.ts +2 -0
- package/dist/test/providers.shared.test.d.ts.map +1 -0
- package/dist/test/providers.shared.test.js +101 -0
- package/dist/test/providers.shared.test.js.map +1 -0
- package/dist/test/providers.test.js +2 -2
- package/dist/test/providers.test.js.map +1 -1
- package/dist/test/redteam/extraction/entities.test.d.ts +2 -0
- package/dist/test/redteam/extraction/entities.test.d.ts.map +1 -0
- package/dist/test/redteam/extraction/entities.test.js +34 -0
- package/dist/test/redteam/extraction/entities.test.js.map +1 -0
- package/dist/test/redteam/extraction/purpose.test.d.ts.map +1 -0
- package/dist/test/redteam/extraction/purpose.test.js +31 -0
- package/dist/test/redteam/extraction/purpose.test.js.map +1 -0
- package/dist/test/redteam/extraction/util.test.d.ts +2 -0
- package/dist/test/redteam/extraction/util.test.d.ts.map +1 -0
- package/dist/test/redteam/extraction/util.test.js +61 -0
- package/dist/test/redteam/extraction/util.test.js.map +1 -0
- package/dist/test/redteam/graders/imitation.test.d.ts +2 -0
- package/dist/test/redteam/graders/imitation.test.d.ts.map +1 -0
- package/dist/test/redteam/graders/imitation.test.js +70 -0
- package/dist/test/redteam/graders/imitation.test.js.map +1 -0
- package/dist/test/redteam/plugins/base.test.js +3 -4
- package/dist/test/redteam/plugins/base.test.js.map +1 -1
- package/dist/test/redteam/validators.test.d.ts +2 -0
- package/dist/test/redteam/validators.test.d.ts.map +1 -0
- package/dist/test/redteam/{types.test.js → validators.test.js} +84 -29
- package/dist/test/redteam/validators.test.js.map +1 -0
- package/dist/test/utils.d.ts +1 -1
- package/dist/test/utils.d.ts.map +1 -1
- package/package.json +13 -13
- package/dist/src/redteam/purpose.d.ts +0 -3
- package/dist/src/redteam/purpose.d.ts.map +0 -1
- package/dist/src/redteam/purpose.js +0 -37
- package/dist/src/redteam/purpose.js.map +0 -1
- package/dist/src/redteam/types.d.ts +0 -215
- package/dist/src/redteam/types.d.ts.map +0 -1
- package/dist/src/redteam/types.js.map +0 -1
- package/dist/src/types.js.map +0 -1
- package/dist/src/web/nextui/_next/static/chunks/258-c59fa47577d85781.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/347-49b79b8c81884e85.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/app/report/page-e79e6b27f8a5b30d.js +0 -1
- package/dist/src/web/nextui/_next/static/chunks/main-app-80560fba5a9a03ab.js +0 -1
- package/dist/src/web/nextui/_next/static/css/e17fdafd8599f69b.css +0 -1
- package/dist/src/web/nextui/_next/static/media/05a31a2ca4975f99-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/513657b02c5c193f-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/51ed15f9841b9f9d-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/d6b16ce4a6175f26-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/ec159349637c90ad-s.woff2 +0 -0
- package/dist/src/web/nextui/_next/static/media/fd4db3eb5472fc27-s.woff2 +0 -0
- package/dist/test/redteam/purpose.test.d.ts.map +0 -1
- package/dist/test/redteam/purpose.test.js +0 -38
- package/dist/test/redteam/purpose.test.js.map +0 -1
- package/dist/test/redteam/types.test.d.ts +0 -2
- package/dist/test/redteam/types.test.d.ts.map +0 -1
- package/dist/test/redteam/types.test.js.map +0 -1
- /package/dist/src/web/nextui/_next/static/{uLnwOyJlvUVVkJF5cxGRK → 9g6bpuNNR0WtmBBAmnqTc}/_buildManifest.js +0 -0
- /package/dist/src/web/nextui/_next/static/{uLnwOyJlvUVVkJF5cxGRK → 9g6bpuNNR0WtmBBAmnqTc}/_ssgManifest.js +0 -0
- /package/dist/test/redteam/{purpose.test.d.ts → extraction/purpose.test.d.ts} +0 -0
|
@@ -39,9 +39,9 @@ const config_1 = require("../../config");
|
|
|
39
39
|
const logger_1 = __importDefault(require("../../logger"));
|
|
40
40
|
const redteam_1 = require("../../redteam");
|
|
41
41
|
const constants_1 = require("../../redteam/constants");
|
|
42
|
-
const types_1 = require("../../redteam/types");
|
|
43
42
|
const telemetry_1 = __importDefault(require("../../telemetry"));
|
|
44
43
|
const util_1 = require("../../util");
|
|
44
|
+
const redteam_2 = require("../../validators/redteam");
|
|
45
45
|
async function doGenerateRedteam(options) {
|
|
46
46
|
(0, util_1.setupEnv)(options.envFile);
|
|
47
47
|
if (!options.cache) {
|
|
@@ -117,12 +117,14 @@ async function doGenerateRedteam(options) {
|
|
|
117
117
|
strategies = [
|
|
118
118
|
...new Set([
|
|
119
119
|
...strategies,
|
|
120
|
-
...options.addStrategies.map((strategy) =>
|
|
120
|
+
...options.addStrategies.map((strategy) => ({
|
|
121
|
+
id: strategy,
|
|
122
|
+
})),
|
|
121
123
|
]),
|
|
122
124
|
];
|
|
123
125
|
}
|
|
124
126
|
const strategyObjs = strategies.map((s) => typeof s === 'string' ? { id: s } : s);
|
|
125
|
-
logger_1.default.debug(`plugins: ${plugins.map((p) => p.id
|
|
127
|
+
logger_1.default.debug(`plugins: ${plugins.map((p) => p.id).join(', ')}`);
|
|
126
128
|
logger_1.default.debug(`strategies: ${strategyObjs.map((s) => s.id ?? s).join(', ')}`);
|
|
127
129
|
const config = {
|
|
128
130
|
injectVar: redteamConfig?.injectVar || options.injectVar,
|
|
@@ -132,7 +134,12 @@ async function doGenerateRedteam(options) {
|
|
|
132
134
|
purpose: redteamConfig?.purpose || options.purpose,
|
|
133
135
|
strategies: strategyObjs,
|
|
134
136
|
};
|
|
135
|
-
const parsedConfig =
|
|
137
|
+
const parsedConfig = redteam_2.RedteamConfigSchema.safeParse(config);
|
|
138
|
+
if (!parsedConfig.success) {
|
|
139
|
+
logger_1.default.error('Invalid redteam configuration:');
|
|
140
|
+
logger_1.default.error(parsedConfig.error.toString());
|
|
141
|
+
throw new Error('Invalid redteam configuration');
|
|
142
|
+
}
|
|
136
143
|
const redteamTests = await (0, redteam_1.synthesize)({
|
|
137
144
|
...parsedConfig.data,
|
|
138
145
|
prompts: testSuite.prompts.map((prompt) => prompt.raw),
|
|
@@ -206,7 +213,7 @@ function generateRedteamCommand(program, defaultConfig, defaultConfigPath) {
|
|
|
206
213
|
let overrides = {};
|
|
207
214
|
if (opts.plugins && opts.plugins.length > 0) {
|
|
208
215
|
logger_1.default.warn(`Overriding plugins: ${opts.plugins.join(', ')}`);
|
|
209
|
-
const parsed =
|
|
216
|
+
const parsed = redteam_2.RedteamConfigSchema.safeParse({
|
|
210
217
|
plugins: opts.plugins,
|
|
211
218
|
strategies: opts.strategies,
|
|
212
219
|
numTests: opts.numTests,
|
|
@@ -220,7 +227,7 @@ function generateRedteamCommand(program, defaultConfig, defaultConfigPath) {
|
|
|
220
227
|
}
|
|
221
228
|
overrides = parsed.data;
|
|
222
229
|
}
|
|
223
|
-
const validatedOpts =
|
|
230
|
+
const validatedOpts = redteam_2.RedteamGenerateOptionsSchema.parse({
|
|
224
231
|
...opts,
|
|
225
232
|
defaultConfig,
|
|
226
233
|
defaultConfigPath,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../../src/commands/generate/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../../src/commands/generate/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyBA,8CAiKC;AAED,wDAkGC;AA9RD,kDAA0B;AAE1B,oDAA4B;AAC5B,uCAAyB;AACzB,sDAA2B;AAC3B,oEAAuC;AACvC,6BAAwB;AACxB,uCAA2C;AAC3C,yCAA8C;AAC9C,0DAAkC;AAClC,2CAA2C;AAC3C,uDAMiC;AACjC,gEAAwC;AAIxC,qCAAmD;AACnD,sDAA6F;AAEtF,KAAK,UAAU,iBAAiB,CAAC,OAA+B;IACrE,IAAA,eAAQ,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAClC,IAAA,oBAAY,GAAE,CAAC;IACjB,CAAC;IAED,IAAI,SAAoB,CAAC;IACzB,IAAI,aAAwC,CAAC;IAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAC/D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAc,EACnC;YACE,MAAM,EAAE,CAAC,UAAU,CAAC;SACrB,EACD,OAAO,CAAC,aAAa,CACtB,CAAC;QACF,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;QAC/B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;IAC1C,CAAC;SAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,uFAAuF;QACvF,SAAS,GAAG;YACV,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,EAAE;SACV,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,GAAG,CACP,gDAAgD,eAAK,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CACpG,CACF,CAAC;QACF,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,IAAI,EAAE,4BAA4B;QAClC,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;KACjD,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IAEvB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,aAAa,EAAE,QAAQ,EAAE,CAAC;QAC5D,gBAAM,CAAC,IAAI,CACT,IAAA,gBAAM,EAAA,KAAK,eAAK,CAAC,IAAI,CAAC,0DAA0D,CAAC;;;;;OAKhF,CACF,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,GACT,aAAa,EAAE,OAAO;QACtB,KAAK,CAAC,IAAI,CAAC,2BAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnD,EAAE,EAAE,MAAM;YACV,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC,CAAC,CAAC;IACN,6CAA6C;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACzC,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ;SAC9C,CAAC,CAAC,CAAC;IACN,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvE,OAAO,GAAG;YACR,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,OAAO;gBACV,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBACrC,EAAE,EAAE,MAAM;oBACV,QAAQ,EAAE,OAAO,CAAC,QAAQ;iBAC3B,CAAC,CAAC;aACJ,CAAC;SACH,CAAC;IACJ,CAAC;IACD,IAAA,wBAAS,EAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,kBAAkB,CAAC,CAAC;IAEvF,IAAI,UAAU,GACZ,aAAa,EAAE,UAAU,IAAI,8BAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1E,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IAClC,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7E,UAAU,GAAG;YACX,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,UAAU;gBACb,GAAG,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC1C,EAAE,EAAE,QAAQ;iBACb,CAAC,CAAC;aACJ,CAAC;SACH,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAA4B,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACjE,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CACtC,CAAC;IAEF,gBAAM,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChE,gBAAM,CAAC,KAAK,CAAC,eAAe,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE7E,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,aAAa,EAAE,SAAS,IAAI,OAAO,CAAC,SAAS;QACxD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO;QACP,QAAQ,EAAE,aAAa,EAAE,QAAQ,IAAI,OAAO,CAAC,QAAQ;QACrD,OAAO,EAAE,aAAa,EAAE,OAAO,IAAI,OAAO,CAAC,OAAO;QAClD,UAAU,EAAE,YAAY;KACzB,CAAC;IACF,MAAM,YAAY,GAAG,6BAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QAC1B,gBAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAC/C,gBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,IAAA,oBAAU,EAAC;QACpC,GAAG,YAAY,CAAC,IAAI;QACpB,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC;QACtD,QAAQ,EAAE,OAAO,CAAC,QAAQ;KACN,CAAC,CAAC;IAExB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,YAAY,GAAG,UAAU;YAC7B,CAAC,CAAE,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA4B;YAC5E,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,WAAW,GAA2B;YAC1C,GAAG,YAAY;YACf,KAAK,EAAE,YAAY;YACnB,QAAQ,EAAE;gBACR,GAAG,YAAY,CAAC,QAAQ;gBACxB,OAAO,EAAE,IAAI;aACd;SACF,CAAC;QACF,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,iBAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAChF,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CAAC,SAAS,YAAY,CAAC,MAAM,sBAAsB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAChF,IAAA,kBAAW,GAAE,CAAC;IAChB,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAA2B,CAAC;QAChG,cAAc,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,YAAY,CAAC,CAAC;QAC1E,cAAc,CAAC,QAAQ,GAAG,cAAc,CAAC,QAAQ,IAAI,EAAE,CAAC;QACxD,cAAc,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;QACvC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;QACxD,gBAAM,CAAC,IAAI,CAAC,WAAW,YAAY,CAAC,MAAM,sBAAsB,UAAU,EAAE,CAAC,CAAC;QAC9E,gBAAM,CAAC,IAAI,CACT,IAAI,GAAG,eAAK,CAAC,KAAK,CAAC,OAAO,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,6BAA6B,CAAC,CACrF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CAAC,iBAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;QAC/B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;QACrD,IAAI,EAAE,kBAAkB;QACxB,UAAU,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;QACpC,gBAAgB,EAAE,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;QAChD,iBAAiB,EAAE,YAAY,CAAC,MAAM;KACvC,CAAC,CAAC;IACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,sBAAsB,CACpC,OAAgB,EAChB,aAAqC,EACrC,iBAAqC;IAErC,OAAO;SACJ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,iCAAiC,CAAC;SAC9C,MAAM,CAAC,qBAAqB,EAAE,8DAA8D,CAAC;SAC7F,MAAM,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;SACpD,MAAM,CAAC,aAAa,EAAE,+CAA+C,EAAE,KAAK,CAAC;SAC7E,MAAM,CACL,qBAAqB,EACrB,8FAA8F,CAC/F;SACA,MAAM,CACL,uBAAuB,EACvB,kEAAkE,yBAAa,EAAE,CAClF;SACA,MAAM,CACL,uBAAuB,EACvB,iHAAiH,CAClH;SACA,MAAM,CACL,qBAAqB,EACrB,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,2BAAuB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;KAChE,EACC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CACL,yBAAyB,EACzB,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,8BAA0B,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;OACjE,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CACL,2BAA2B,EAC3B,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,8BAAkB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;OACzD,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CACnF;SACA,MAAM,CACL,+BAA+B,EAC/B,IAAA,gBAAM,EAAA;cACE,KAAK,CAAC,IAAI,CAAC,iCAAqB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;OAC5D,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAC7C;SACA,MAAM,CACL,0BAA0B,EAC1B,6CAA6C,EAC7C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAC1D,CAAC,CACF;SACA,MAAM,CAAC,YAAY,EAAE,4CAA4C,EAAE,KAAK,CAAC;SACzE,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;SAChD,MAAM,CAAC,CAAC,IAAqC,EAAQ,EAAE;QACtD,IAAI,CAAC;YACH,IAAI,SAAS,GAAwB,EAAE,CAAC;YACxC,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5C,gBAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC9D,MAAM,MAAM,GAAG,6BAAmB,CAAC,SAAS,CAAC;oBAC3C,OAAO,EAAE,IAAI,CAAC,OAAO;oBACrB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACxB,CAAC,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpB,gBAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;oBACjC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAe,EAAE,EAAE;wBAC9C,gBAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC1D,CAAC,CAAC,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAClB,CAAC;gBACD,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;YAC1B,CAAC;YACD,MAAM,aAAa,GAAG,sCAA4B,CAAC,KAAK,CAAC;gBACvD,GAAG,IAAI;gBACP,aAAa;gBACb,iBAAiB;gBACjB,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,GAAG,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtE,CAAC,CAAC;YACH,iBAAiB,CAAC,aAAa,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,OAAC,CAAC,QAAQ,EAAE,CAAC;gBAChC,gBAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACjC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAe,EAAE,EAAE;oBACvC,gBAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1D,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YACvD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../../../src/commands/import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../../../src/commands/import.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOzC,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,QAiC7C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAKzC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA8B3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMzC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA0G3C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBzC,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,iBAkP9D;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,QAS9C"}
|
|
@@ -26,6 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.redteamInit = redteamInit;
|
|
29
30
|
exports.redteamCommand = redteamCommand;
|
|
30
31
|
const checkbox_1 = __importDefault(require("@inquirer/checkbox"));
|
|
31
32
|
const checkbox_2 = require("@inquirer/checkbox");
|
|
@@ -41,212 +42,217 @@ const path = __importStar(require("path"));
|
|
|
41
42
|
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
42
43
|
const logger_1 = __importDefault(require("../logger"));
|
|
43
44
|
const constants_1 = require("../redteam/constants");
|
|
44
|
-
const types_1 = require("../redteam/types");
|
|
45
45
|
const telemetry_1 = __importDefault(require("../telemetry"));
|
|
46
|
-
const redteam_1 = require("
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
46
|
+
const redteam_1 = require("../validators/redteam");
|
|
47
|
+
const redteam_2 = require("./generate/redteam");
|
|
48
|
+
async function redteamInit(directory) {
|
|
49
|
+
telemetry_1.default.maybeShowNotice();
|
|
50
|
+
telemetry_1.default.record('command_used', { name: 'redteam init - started' });
|
|
51
|
+
await telemetry_1.default.send();
|
|
52
|
+
console.clear();
|
|
53
|
+
logger_1.default.info(chalk_1.default.bold('Red Team Initialization\n'));
|
|
54
|
+
let projectDir = directory;
|
|
55
|
+
if (!projectDir) {
|
|
56
|
+
projectDir = await (0, input_1.default)({
|
|
57
|
+
message: 'Where do you want to create the project?',
|
|
58
|
+
default: '.',
|
|
56
59
|
});
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
const configPath = path.join(projectDir, 'promptfooconfig.yaml');
|
|
69
|
-
const previousConfigExists = fs.existsSync(configPath);
|
|
70
|
-
let existingConfig;
|
|
71
|
-
if (previousConfigExists) {
|
|
72
|
-
const overwrite = await (0, confirm_1.default)({
|
|
73
|
-
message: `A promptfoo configuration file already exists at: ${configPath} Do you want to overwrite it?`,
|
|
74
|
-
default: false,
|
|
75
|
-
});
|
|
76
|
-
if (!overwrite) {
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
79
|
-
existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
|
|
80
|
-
}
|
|
81
|
-
const promptChoices = [
|
|
82
|
-
{ name: 'Enter a prompt', value: 'enter' },
|
|
83
|
-
{ name: 'Reference a prompt file', value: 'file' },
|
|
84
|
-
];
|
|
85
|
-
if (previousConfigExists) {
|
|
86
|
-
promptChoices.push({ name: 'Use prompts from existing config', value: 'existing' });
|
|
87
|
-
}
|
|
88
|
-
// Question: Prompt
|
|
89
|
-
const promptChoice = await (0, rawlist_1.default)({
|
|
90
|
-
message: 'How would you like to specify the prompt?',
|
|
91
|
-
choices: promptChoices,
|
|
60
|
+
}
|
|
61
|
+
if (projectDir !== '.' && !fs.existsSync(projectDir)) {
|
|
62
|
+
fs.mkdirSync(projectDir, { recursive: true });
|
|
63
|
+
}
|
|
64
|
+
const configPath = path.join(projectDir, 'promptfooconfig.yaml');
|
|
65
|
+
const previousConfigExists = fs.existsSync(configPath);
|
|
66
|
+
let existingConfig;
|
|
67
|
+
if (previousConfigExists) {
|
|
68
|
+
const overwrite = await (0, confirm_1.default)({
|
|
69
|
+
message: `A promptfoo configuration file already exists at: ${configPath} Do you want to overwrite it?`,
|
|
70
|
+
default: false,
|
|
92
71
|
});
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
prompts.push(await (0, editor_1.default)({
|
|
96
|
-
message: 'Enter your prompt:',
|
|
97
|
-
default: 'You are a helpful concise assistant.\n\nUser query: {{query}}\n\n(NOTE: your prompt must include "{{query}}" as a placeholder for user input)',
|
|
98
|
-
}));
|
|
72
|
+
if (!overwrite) {
|
|
73
|
+
return;
|
|
99
74
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
prompts
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
75
|
+
existingConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf8'));
|
|
76
|
+
}
|
|
77
|
+
console.clear();
|
|
78
|
+
logger_1.default.info(chalk_1.default.bold('Prompt Configuration\n'));
|
|
79
|
+
const promptChoices = [
|
|
80
|
+
{ name: 'Enter a prompt', value: 'enter' },
|
|
81
|
+
{ name: 'Reference a prompt file', value: 'file' },
|
|
82
|
+
...(previousConfigExists
|
|
83
|
+
? [{ name: 'Use prompts from existing config', value: 'existing' }]
|
|
84
|
+
: []),
|
|
85
|
+
];
|
|
86
|
+
const promptChoice = await (0, rawlist_1.default)({
|
|
87
|
+
message: 'How would you like to specify the prompt?',
|
|
88
|
+
choices: promptChoices,
|
|
89
|
+
});
|
|
90
|
+
let prompts = [];
|
|
91
|
+
if (promptChoice === 'enter') {
|
|
92
|
+
prompts.push(await (0, editor_1.default)({
|
|
93
|
+
message: 'Enter your prompt:',
|
|
94
|
+
default: 'You are a helpful concise assistant.\n\nUser query: {{query}}\n\n(NOTE: your prompt must include "{{query}}" as a placeholder for user input)',
|
|
95
|
+
}));
|
|
96
|
+
}
|
|
97
|
+
else if (promptChoice === 'file') {
|
|
98
|
+
const promptFile = await (0, input_1.default)({
|
|
99
|
+
message: 'Enter the path to your prompt file (text or JSON):',
|
|
100
|
+
});
|
|
101
|
+
prompts.push(`file://${promptFile}`);
|
|
102
|
+
}
|
|
103
|
+
else if (promptChoice === 'existing') {
|
|
104
|
+
(0, tiny_invariant_1.default)(existingConfig?.prompts, 'No prompts found in existing configuration');
|
|
105
|
+
prompts = existingConfig?.prompts;
|
|
106
|
+
}
|
|
107
|
+
console.clear();
|
|
108
|
+
logger_1.default.info(chalk_1.default.bold('Provider Configuration\n'));
|
|
109
|
+
const providerChoices = [
|
|
110
|
+
{ name: 'openai:gpt-4o-mini', value: 'openai:gpt-4o-mini', checked: true },
|
|
111
|
+
{ name: 'openai:gpt-4o', value: 'openai:gpt-4o' },
|
|
112
|
+
{ name: 'openai:gpt-3.5-turbo', value: 'openai:gpt-3.5-turbo' },
|
|
113
|
+
{
|
|
114
|
+
name: 'anthropic:claude-3-5-sonnet-20240620',
|
|
115
|
+
value: 'anthropic:messages:claude-3-5-sonnet-20240620',
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
name: 'anthropic:claude-3-opus-20240307',
|
|
119
|
+
value: 'anthropic:messages:claude-3-opus-20240307',
|
|
120
|
+
},
|
|
121
|
+
{ name: 'vertex:gemini-pro', value: 'vertex:gemini-pro' },
|
|
122
|
+
{ name: 'Other', value: 'Other' },
|
|
123
|
+
];
|
|
124
|
+
const selectedProviders = await (0, checkbox_1.default)({
|
|
125
|
+
message: 'Choose one or more providers to target:',
|
|
126
|
+
choices: providerChoices,
|
|
127
|
+
pageSize: Math.min(providerChoices.length, process.stdout.rows - 4),
|
|
128
|
+
});
|
|
129
|
+
const providers = selectedProviders.filter((p) => p !== 'Other');
|
|
130
|
+
if (selectedProviders.includes('Other')) {
|
|
131
|
+
const customProvider = await (0, input_1.default)({
|
|
132
|
+
message: 'Enter the custom provider ID (see https://www.promptfoo.dev/docs/providers/ for options):',
|
|
133
|
+
});
|
|
134
|
+
providers.push(customProvider);
|
|
135
|
+
}
|
|
136
|
+
if (!process.env.OPENAI_API_KEY) {
|
|
137
|
+
console.clear();
|
|
138
|
+
logger_1.default.info(chalk_1.default.bold('OpenAI API Configuration\n'));
|
|
139
|
+
const apiKeyChoice = await (0, rawlist_1.default)({
|
|
140
|
+
message: 'OpenAI API key is required. How to proceed?',
|
|
114
141
|
choices: [
|
|
115
|
-
{ name: '
|
|
116
|
-
{ name: '
|
|
117
|
-
{ name: 'openai:gpt-3.5-turbo', value: 'openai:gpt-3.5-turbo' },
|
|
118
|
-
{
|
|
119
|
-
name: 'anthropic:claude-3-5-sonnet-20240620',
|
|
120
|
-
value: 'anthropic:messages:claude-3-5-sonnet-20240620',
|
|
121
|
-
},
|
|
122
|
-
{
|
|
123
|
-
name: 'anthropic:claude-3-opus-20240307',
|
|
124
|
-
value: 'anthropic:messages:claude-3-opus-20240307',
|
|
125
|
-
},
|
|
126
|
-
{ name: 'vertex:gemini-pro', value: 'vertex:gemini-pro' },
|
|
127
|
-
{ name: 'Other', value: 'Other' },
|
|
142
|
+
{ name: 'Enter API key now', value: 'enter' },
|
|
143
|
+
{ name: 'Set it later', value: 'later' },
|
|
128
144
|
],
|
|
129
145
|
});
|
|
130
|
-
if (
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
146
|
+
if (apiKeyChoice === 'enter') {
|
|
147
|
+
const apiKey = await (0, input_1.default)({ message: 'Enter your OpenAI API key:' });
|
|
148
|
+
process.env.OPENAI_API_KEY = apiKey;
|
|
149
|
+
logger_1.default.info('OPENAI_API_KEY set for this session.');
|
|
134
150
|
}
|
|
135
151
|
else {
|
|
136
|
-
|
|
152
|
+
logger_1.default.warn('Remember to set OPENAI_API_KEY before generating the dataset.');
|
|
137
153
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
154
|
+
}
|
|
155
|
+
console.clear();
|
|
156
|
+
logger_1.default.info(chalk_1.default.bold('Plugin Configuration\n'));
|
|
157
|
+
const pluginChoices = Array.from(constants_1.ALL_PLUGINS)
|
|
158
|
+
.sort()
|
|
159
|
+
.map((plugin) => ({
|
|
160
|
+
name: `${plugin} - ${constants_1.subCategoryDescriptions[plugin] || 'No description'}`,
|
|
161
|
+
value: plugin,
|
|
162
|
+
checked: existingConfig?.redteam?.plugins.some((p) => p.id === plugin) ||
|
|
163
|
+
constants_1.DEFAULT_PLUGINS.has(plugin),
|
|
164
|
+
}));
|
|
165
|
+
const plugins = await (0, checkbox_1.default)({
|
|
166
|
+
message: 'Select plugins to enable:',
|
|
167
|
+
choices: pluginChoices,
|
|
168
|
+
pageSize: Math.min(pluginChoices.length, process.stdout.rows - 4),
|
|
169
|
+
loop: false,
|
|
170
|
+
});
|
|
171
|
+
console.clear();
|
|
172
|
+
logger_1.default.info(chalk_1.default.bold('Strategy Configuration\n'));
|
|
173
|
+
const strategyChoices = [
|
|
174
|
+
...Array.from(constants_1.DEFAULT_STRATEGIES).sort(),
|
|
175
|
+
new checkbox_2.Separator(),
|
|
176
|
+
...Array.from(constants_1.ADDITIONAL_STRATEGIES).sort(),
|
|
177
|
+
].map((strategy) => typeof strategy === 'string'
|
|
178
|
+
? {
|
|
179
|
+
name: `${strategy} - ${constants_1.subCategoryDescriptions[strategy] || 'No description'}`,
|
|
180
|
+
value: strategy,
|
|
181
|
+
checked: existingConfig?.redteam?.strategies?.some((s) => s.id === strategy || (typeof s === 'string' && s === strategy)) || constants_1.DEFAULT_STRATEGIES.includes(strategy),
|
|
157
182
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
].map((strategy) => typeof strategy === 'string'
|
|
179
|
-
? {
|
|
180
|
-
name: `${strategy} - ${constants_1.subCategoryDescriptions[strategy] || 'No description available'}`,
|
|
181
|
-
value: strategy,
|
|
182
|
-
checked: existingConfig?.redteam?.strategies?.some((s) => s.id === strategy || (typeof s === 'string' && s === strategy)) || constants_1.DEFAULT_STRATEGIES.includes(strategy),
|
|
183
|
-
}
|
|
184
|
-
: strategy);
|
|
185
|
-
const strategies = await (0, checkbox_1.default)({
|
|
186
|
-
message: 'Select the strategies you want to enable (Note: Each strategy will add as many test cases as all plugins combined):',
|
|
187
|
-
choices: strategyChoices,
|
|
188
|
-
pageSize: Math.min(strategyChoices.length, process.stdout.rows - 4),
|
|
189
|
-
loop: false,
|
|
190
|
-
});
|
|
191
|
-
const numTests = await (0, number_1.default)({
|
|
192
|
-
message: 'How many test cases do you want to generate per plugin?',
|
|
193
|
-
default: 5,
|
|
194
|
-
min: 0,
|
|
195
|
-
max: 1000,
|
|
196
|
-
});
|
|
197
|
-
(0, tiny_invariant_1.default)(numTests, 'No number of tests provided');
|
|
198
|
-
// Create config file
|
|
199
|
-
const config = {
|
|
200
|
-
prompts,
|
|
201
|
-
providers: [provider],
|
|
202
|
-
tests: [],
|
|
203
|
-
redteam: types_1.redteamConfigSchema.safeParse({
|
|
204
|
-
plugins: plugins,
|
|
205
|
-
strategies: strategies,
|
|
206
|
-
numTests,
|
|
207
|
-
}).data,
|
|
208
|
-
};
|
|
209
|
-
// Write the simplified form to the config file to make it easier
|
|
210
|
-
// for people to play with. Writes 1 in the { id: ..., numTests }
|
|
211
|
-
// and then the rest as strings.
|
|
212
|
-
const parsedPlugins = types_1.redteamConfigSchema.safeParse({
|
|
183
|
+
: strategy);
|
|
184
|
+
const strategies = await (0, checkbox_1.default)({
|
|
185
|
+
message: 'Select strategies to enable:',
|
|
186
|
+
choices: strategyChoices,
|
|
187
|
+
pageSize: Math.min(strategyChoices.length, process.stdout.rows - 4),
|
|
188
|
+
loop: false,
|
|
189
|
+
});
|
|
190
|
+
const numTests = await (0, number_1.default)({
|
|
191
|
+
message: 'Number of test cases per plugin:',
|
|
192
|
+
default: 5,
|
|
193
|
+
min: 0,
|
|
194
|
+
max: 1000,
|
|
195
|
+
});
|
|
196
|
+
(0, tiny_invariant_1.default)(numTests, 'No number of tests provided');
|
|
197
|
+
// Create config file
|
|
198
|
+
const config = {
|
|
199
|
+
prompts,
|
|
200
|
+
providers: providers,
|
|
201
|
+
tests: [],
|
|
202
|
+
redteam: redteam_1.RedteamConfigSchema.safeParse({
|
|
213
203
|
plugins: plugins,
|
|
214
204
|
strategies: strategies,
|
|
215
205
|
numTests,
|
|
216
|
-
})
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
206
|
+
}).data,
|
|
207
|
+
};
|
|
208
|
+
// Write the simplified form to the config file to make it easier
|
|
209
|
+
// for people to play with. Writes 1 in the { id: ..., numTests }
|
|
210
|
+
// and then the rest as strings.
|
|
211
|
+
const parsedPlugins = redteam_1.RedteamConfigSchema.safeParse({
|
|
212
|
+
plugins: plugins,
|
|
213
|
+
strategies: strategies,
|
|
214
|
+
numTests,
|
|
215
|
+
})?.data?.plugins;
|
|
216
|
+
const configPlugins = plugins.length >= 2 ? [parsedPlugins?.[0], ...plugins.slice(1)] : plugins;
|
|
217
|
+
fs.writeFileSync(configPath, js_yaml_1.default.dump({
|
|
218
|
+
...config,
|
|
219
|
+
redteam: {
|
|
220
|
+
numTests,
|
|
221
|
+
plugins: configPlugins,
|
|
222
|
+
strategies,
|
|
223
|
+
},
|
|
224
|
+
}), 'utf8');
|
|
225
|
+
console.clear();
|
|
226
|
+
logger_1.default.info(chalk_1.default.green(`\nCreated red teaming configuration file at ${configPath}\n`));
|
|
227
|
+
const readyToGenerate = await (0, confirm_1.default)({
|
|
228
|
+
message: 'Are you ready to generate adversarial test cases?',
|
|
229
|
+
default: true,
|
|
230
|
+
});
|
|
231
|
+
if (readyToGenerate) {
|
|
232
|
+
await (0, redteam_2.doGenerateRedteam)({
|
|
233
|
+
plugins: parsedPlugins,
|
|
234
|
+
cache: false,
|
|
235
|
+
write: true,
|
|
236
|
+
defaultConfig: config,
|
|
237
|
+
defaultConfigPath: configPath,
|
|
238
|
+
numTests,
|
|
233
239
|
});
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
logger_1.default.info('\n' +
|
|
243
|
+
chalk_1.default.blue('To generate test cases later, use the command: ' +
|
|
244
|
+
chalk_1.default.bold('promptfoo generate redteam')));
|
|
245
|
+
}
|
|
246
|
+
telemetry_1.default.record('command_used', { name: 'redteam init' });
|
|
247
|
+
await telemetry_1.default.send();
|
|
248
|
+
}
|
|
249
|
+
function redteamCommand(program) {
|
|
250
|
+
const redteamCommand = program.command('redteam').description('Red team LLM applications');
|
|
251
|
+
redteamCommand
|
|
252
|
+
.command('init [directory]')
|
|
253
|
+
.description('Initialize red teaming project')
|
|
254
|
+
.action(async (directory) => {
|
|
255
|
+
await redteamInit(directory);
|
|
250
256
|
});
|
|
251
257
|
}
|
|
252
258
|
//# sourceMappingURL=redteam.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0BA,kCAkPC;AAED,wCASC;AAvRD,kEAA0C;AAC1C,iDAA+C;AAC/C,gEAAwC;AACxC,8DAAsC;AACtC,4DAAoC;AACpC,8DAAsC;AACtC,gEAAwC;AACxC,kDAA0B;AAE1B,uCAAyB;AACzB,sDAA2B;AAC3B,2CAA6B;AAC7B,oEAAuC;AACvC,uDAA+B;AAC/B,oDAM8B;AAC9B,6DAAqC;AAErC,mDAA4D;AAC5D,gDAAuD;AAEhD,KAAK,UAAU,WAAW,CAAC,SAA6B;IAC7D,mBAAS,CAAC,eAAe,EAAE,CAAC;IAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IACrE,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IAEvB,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC,CAAC;IAErD,IAAI,UAAU,GAAG,SAAS,CAAC;IAC3B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,UAAU,GAAG,MAAM,IAAA,eAAK,EAAC;YACvB,OAAO,EAAE,0CAA0C;YACnD,OAAO,EAAE,GAAG;SACb,CAAC,CAAC;IACL,CAAC;IACD,IAAI,UAAU,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;IACjE,MAAM,oBAAoB,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IACvD,IAAI,cAAqC,CAAC;IAC1C,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,MAAM,IAAA,iBAAO,EAAC;YAC9B,OAAO,EAAE,qDAAqD,UAAU,+BAA+B;YACvG,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,cAAc,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAc,CAAC;IAC/E,CAAC;IAED,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAElD,MAAM,aAAa,GAAG;QACpB,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;QAC1C,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,MAAM,EAAE;QAClD,GAAG,CAAC,oBAAoB;YACtB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,kCAAkC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;YACnE,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,IAAA,iBAAO,EAAC;QACjC,OAAO,EAAE,2CAA2C;QACpD,OAAO,EAAE,aAAa;KACvB,CAAC,CAAC;IAEH,IAAI,OAAO,GAAwB,EAAE,CAAC;IACtC,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CACV,MAAM,IAAA,gBAAM,EAAC;YACX,OAAO,EAAE,oBAAoB;YAC7B,OAAO,EACL,+IAA+I;SAClJ,CAAC,CACH,CAAC;IACJ,CAAC;SAAM,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,MAAM,IAAA,eAAK,EAAC;YAC7B,OAAO,EAAE,oDAAoD;SAC9D,CAAC,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,UAAU,UAAU,EAAE,CAAC,CAAC;IACvC,CAAC;SAAM,IAAI,YAAY,KAAK,UAAU,EAAE,CAAC;QACvC,IAAA,wBAAS,EAAC,cAAc,EAAE,OAAO,EAAE,4CAA4C,CAAC,CAAC;QACjF,OAAO,GAAG,cAAc,EAAE,OAAO,CAAC;IACpC,CAAC;IAED,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC;IAEpD,MAAM,eAAe,GAAG;QACtB,EAAE,IAAI,EAAE,oBAAoB,EAAE,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,IAAI,EAAE;QAC1E,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,eAAe,EAAE;QACjD,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,sBAAsB,EAAE;QAC/D;YACE,IAAI,EAAE,sCAAsC;YAC5C,KAAK,EAAE,+CAA+C;SACvD;QACD;YACE,IAAI,EAAE,kCAAkC;YACxC,KAAK,EAAE,2CAA2C;SACnD;QACD,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,mBAAmB,EAAE;QACzD,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE;KAClC,CAAC;IAEF,MAAM,iBAAiB,GAAG,MAAM,IAAA,kBAAQ,EAAC;QACvC,OAAO,EAAE,yCAAyC;QAClD,OAAO,EAAE,eAAe;QACxB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;KACpE,CAAC,CAAC;IAEH,MAAM,SAAS,GAAa,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC;IAC3E,IAAI,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,MAAM,IAAA,eAAK,EAAC;YACjC,OAAO,EACL,2FAA2F;SAC9F,CAAC,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;QAEtD,MAAM,YAAY,GAAG,MAAM,IAAA,iBAAO,EAAC;YACjC,OAAO,EAAE,6CAA6C;YACtD,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE;gBAC7C,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,EAAE;aACzC;SACF,CAAC,CAAC;QAEH,IAAI,YAAY,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAA,eAAK,EAAC,EAAE,OAAO,EAAE,4BAA4B,EAAE,CAAC,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC;YACpC,gBAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAElD,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,uBAAW,CAAC;SAC1C,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChB,IAAI,EAAE,GAAG,MAAM,MAAM,mCAAuB,CAAC,MAAM,CAAC,IAAI,gBAAgB,EAAE;QAC1E,KAAK,EAAE,MAAM;QACb,OAAO,EACL,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC;YAC7D,2BAAe,CAAC,GAAG,CAAC,MAAM,CAAC;KAC9B,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAQ,EAAC;QAC7B,OAAO,EAAE,2BAA2B;QACpC,OAAO,EAAE,aAAa;QACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QACjE,IAAI,EAAE,KAAK;KACZ,CAAC,CAAC;IAEH,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC;IAEpD,MAAM,eAAe,GAAG;QACtB,GAAG,KAAK,CAAC,IAAI,CAAC,8BAAkB,CAAC,CAAC,IAAI,EAAE;QACxC,IAAI,oBAAS,EAAE;QACf,GAAG,KAAK,CAAC,IAAI,CAAC,iCAAqB,CAAC,CAAC,IAAI,EAAE;KAC5C,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjB,OAAO,QAAQ,KAAK,QAAQ;QAC1B,CAAC,CAAC;YACE,IAAI,EAAE,GAAG,QAAQ,MAAM,mCAAuB,CAAC,QAAQ,CAAC,IAAI,gBAAgB,EAAE;YAC9E,KAAK,EAAE,QAAQ;YACf,OAAO,EACL,cAAc,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,QAAQ,CAAC,CACtE,IAAI,8BAAkB,CAAC,QAAQ,CAAC,QAAe,CAAC;SACpD;QACH,CAAC,CAAC,QAAQ,CACb,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,IAAA,kBAAQ,EAAC;QAChC,OAAO,EAAE,8BAA8B;QACvC,OAAO,EAAE,eAAe;QACxB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QACnE,IAAI,EAAE,KAAK;KACZ,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,MAAM,IAAA,gBAAM,EAAC;QAC5B,OAAO,EAAE,kCAAkC;QAC3C,OAAO,EAAE,CAAC;QACV,GAAG,EAAE,CAAC;QACN,GAAG,EAAE,IAAI;KACV,CAAC,CAAC;IACH,IAAA,wBAAS,EAAC,QAAQ,EAAE,6BAA6B,CAAC,CAAC;IAEnD,qBAAqB;IACrB,MAAM,MAAM,GAAG;QACb,OAAO;QACP,SAAS,EAAE,SAAS;QACpB,KAAK,EAAE,EAAE;QACT,OAAO,EAAE,6BAAmB,CAAC,SAAS,CAAC;YACrC,OAAO,EAAE,OAAO;YAChB,UAAU,EAAE,UAAU;YACtB,QAAQ;SACT,CAAC,CAAC,IAAI;KACR,CAAC;IAEF,iEAAiE;IACjE,iEAAiE;IACjE,gCAAgC;IAChC,MAAM,aAAa,GAAG,6BAAmB,CAAC,SAAS,CAAC;QAClD,OAAO,EAAE,OAAO;QAChB,UAAU,EAAE,UAAU;QACtB,QAAQ;KACT,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;IAClB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAEhG,EAAE,CAAC,aAAa,CACd,UAAU,EACV,iBAAI,CAAC,IAAI,CAAC;QACR,GAAG,MAAM;QACT,OAAO,EAAE;YACP,QAAQ;YACR,OAAO,EAAE,aAAa;YACtB,UAAU;SACX;KACF,CAAC,EACF,MAAM,CACP,CAAC;IAEF,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,KAAK,CAAC,+CAA+C,UAAU,IAAI,CAAC,CAAC,CAAC;IAExF,MAAM,eAAe,GAAG,MAAM,IAAA,iBAAO,EAAC;QACpC,OAAO,EAAE,mDAAmD;QAC5D,OAAO,EAAE,IAAI;KACd,CAAC,CAAC;IAEH,IAAI,eAAe,EAAE,CAAC;QACpB,MAAM,IAAA,2BAAiB,EAAC;YACtB,OAAO,EAAE,aAAa;YACtB,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,IAAI;YACX,aAAa,EAAE,MAAM;YACrB,iBAAiB,EAAE,UAAU;YAC7B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,gBAAM,CAAC,IAAI,CACT,IAAI;YACF,eAAK,CAAC,IAAI,CACR,iDAAiD;gBAC/C,eAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAC3C,CACJ,CAAC;IACJ,CAAC;IAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;IAC3D,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,SAAgB,cAAc,CAAC,OAAgB;IAC7C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAAC;IAE3F,cAAc;SACX,OAAO,CAAC,kBAAkB,CAAC;SAC3B,WAAW,CAAC,gCAAgC,CAAC;SAC7C,MAAM,CAAC,KAAK,EAAE,SAA6B,EAAE,EAAE;QAC9C,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC;AACP,CAAC"}
|