promptfoo 0.96.2 → 0.98.0

package/dist/test/redteam/plugins/harmful/aligned.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aligned.test.d.ts","sourceRoot":"","sources":["../../../../../test/redteam/plugins/harmful/aligned.test.ts"],"names":[],"mappings":""}

package/dist/test/redteam/plugins/harmful/aligned.test.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const constants_1 = require("../../../../src/redteam/constants");
+const aligned_1 = require("../../../../src/redteam/plugins/harmful/aligned");
+const constants_2 = require("../../../../src/redteam/plugins/harmful/constants");
+jest.mock('../../../../src/logger');
+describe('AlignedHarmfulPlugin', () => {
+    let mockProvider;
+    let plugin;
+    let harmCategory;
+    beforeEach(() => {
+        mockProvider = {
+            callApi: jest.fn(),
+            id: jest.fn().mockReturnValue('test-provider'),
+        };
+        // Find a harm category that isn't in unaligned providers
+        harmCategory = Object.keys(constants_1.HARM_PLUGINS).find((key) => !(key in constants_1.UNALIGNED_PROVIDER_HARM_PLUGINS));
+        const harmfulCategory = {
+            key: harmCategory,
+            prompt: 'test prompt template {{ examples }}',
+            examples: 'test examples',
+            label: 'Test Label',
+            description: 'Test Description',
+        };
+        // Mock a valid category in REDTEAM_MODEL_CATEGORIES
+        jest.spyOn(constants_2.REDTEAM_MODEL_CATEGORIES, 'find').mockReturnValue(harmfulCategory);
+        plugin = new aligned_1.AlignedHarmfulPlugin(mockProvider, 'test purpose', 'testVar', harmCategory);
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+    describe('initialization', () => {
+        it('should create instance with valid parameters', () => {
+            expect(plugin).toBeInstanceOf(aligned_1.AlignedHarmfulPlugin);
+            expect(mockProvider.id()).toBe('test-provider');
+        });
+        it('should accept optional config parameter', () => {
+            const configPlugin = new aligned_1.AlignedHarmfulPlugin(mockProvider, 'test purpose', 'testVar', harmCategory, { examples: ['test example'] });
+            expect(configPlugin).toBeInstanceOf(aligned_1.AlignedHarmfulPlugin);
+        });
+    });
+    describe('template handling', () => {
+        it('should retrieve correct template for harm category', async () => {
+            const template = await plugin['getTemplate']();
+            expect(template).toBe('test prompt template {{ examples }}');
+        });
+        it('should throw when category config is not found', async () => {
+            jest.spyOn(constants_2.REDTEAM_MODEL_CATEGORIES, 'find').mockReturnValue(undefined);
+            await expect(plugin['getTemplate']()).rejects.toThrow(`Harmful category ${harmCategory} not found`);
+        });
+    });
+    describe('assertion generation', () => {
+        it('should generate harmful assertions for given prompt', () => {
+            const assertions = plugin['getAssertions']('test prompt');
+            expect(assertions).toEqual(expect.arrayContaining([
+                expect.objectContaining({
+                    metric: 'Harmful',
+                }),
+            ]));
+        });
+        it('should include harm category in assertions', () => {
+            const assertions = plugin['getAssertions']('test prompt');
+            assertions.forEach((assertion) => {
+                expect(assertion).toHaveProperty('metric');
+                expect(assertion.metric).toBe('Harmful');
+            });
+        });
+    });
+    describe('test case generation', () => {
+        const mockPrompts = [{ prompt: 'Test prompt 1' }, { prompt: 'Test prompt 2' }];
+        it('should create properly structured test cases', () => {
+            const testCases = plugin['promptsToTestCases'](mockPrompts);
+            expect(testCases).toHaveLength(2);
+            testCases.forEach((testCase) => {
+                expect(testCase).toMatchObject({
+                    vars: {
+                        testVar: expect.any(String),
+                        harmCategory: constants_1.HARM_PLUGINS[harmCategory],
+                    },
+                    metadata: {
+                        harmCategory: constants_1.HARM_PLUGINS[harmCategory],
+                    },
+                    assert: expect.arrayContaining([
+                        expect.objectContaining({
+                            metric: 'Harmful',
+                        }),
+                    ]),
+                });
+            });
+        });
+        it('should handle empty prompts array', () => {
+            const testCases = plugin['promptsToTestCases']([]);
+            expect(testCases).toHaveLength(0);
+        });
+    });
+    describe('end-to-end test generation', () => {
+        beforeEach(() => {
+            jest.spyOn(mockProvider, 'callApi').mockResolvedValue({
+                output: 'Prompt: Generated test 1\nPrompt: Generated test 2',
+            });
+        });
+        it('should generate requested number of test cases', async () => {
+            const testCases = await plugin.generateTests(2);
+            expect(testCases).toHaveLength(2);
+            expect(mockProvider.callApi).toHaveBeenCalledTimes(1);
+        });
+        it('should handle custom examples in config', async () => {
+            const customExamples = ['Custom example 1', 'Custom example 2'];
+            const configuredPlugin = new aligned_1.AlignedHarmfulPlugin(mockProvider, 'test purpose', 'testVar', harmCategory, { examples: customExamples });
+            await configuredPlugin.generateTests(1);
+            expect(mockProvider.callApi).toHaveBeenCalledWith(expect.stringContaining('test prompt template Custom example 1,Custom example 2'));
+        });
+        it('should handle API errors gracefully', async () => {
+            jest.spyOn(mockProvider, 'callApi').mockResolvedValue({
+                output: '',
+                error: 'API Error',
+            });
+            const testCases = await plugin.generateTests(1);
+            expect(testCases).toHaveLength(0);
+        });
+    });
+});
+//# sourceMappingURL=aligned.test.js.map

package/dist/test/redteam/plugins/harmful/aligned.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"aligned.test.js","sourceRoot":"","sources":["../../../../../test/redteam/plugins/harmful/aligned.test.ts"],"names":[],"mappings":";;AAAA,iEAAkG;AAClG,6EAAuF;AAEvF,iFAA6F;AAG7F,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AAEpC,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,IAAI,YAAyB,CAAC;IAC9B,IAAI,MAA4B,CAAC;IACjC,IAAI,YAAuC,CAAC;IAE5C,UAAU,CAAC,GAAG,EAAE;QACd,YAAY,GAAG;YACb,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,eAAe,CAAC;SAC/C,CAAC;QAEF,yDAAyD;QACzD,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,wBAAY,CAAC,CAAC,IAAI,CAC3C,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,2CAA+B,CAAC,CACtB,CAAC;QAE/B,MAAM,eAAe,GAAoB;YACvC,GAAG,EAAE,YAAsC;YAC3C,MAAM,EAAE,qCAAqC;YAC7C,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,YAAwC;YAC/C,WAAW,EAAE,kBAAkB;SAChC,CAAC;QAEF,oDAAoD;QACpD,IAAI,CAAC,KAAK,CAAC,oCAAwB,EAAE,MAAM,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAE9E,MAAM,GAAG,IAAI,8BAAoB,CAAC,YAAY,EAAE,cAAc,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,8BAAoB,CAAC,CAAC;YACpD,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,YAAY,GAAG,IAAI,8BAAoB,CAC3C,YAAY,EACZ,cAAc,EACd,SAAS,EACT,YAAY,EACZ,EAAE,QAAQ,EAAE,CAAC,cAAc,CAAC,EAAE,CAC/B,CAAC;YACF,MAAM,CAAC,YAAY,CAAC,CAAC,cAAc,CAAC,8BAAoB,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YAC/C,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,IAAI,CAAC,KAAK,CAAC,oCAAwB,EAAE,MAAM,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACxE,MAAM,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CACnD,oBAAoB,YAAY,YAAY,CAC7C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,aAAa,CAAC,CAAC;YAC1D,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CACxB,MAAM,CAAC,eAAe,CAAC;gBACrB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,MAAM,EAAE,SAAS;iBAClB,CAAC;aACH,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,aAAa,CAAC,CAAC;YAC1D,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;gBAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;gBAC3C,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3C,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,MAAM,WAAW,GAAG,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC,CAAC;QAE/E,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC,WAAW,CAAC,CAAC;YAE5D,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC7B,MAAM,CAAC,QAAQ,CAAC,CAAC,aAAa,CAAC;oBAC7B,IAAI,EAAE;wBACJ,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;wBAC3B,YAAY,EAAE,wBAAY,CAAC,YAAY,CAAC;qBACzC;oBACD,QAAQ,EAAE;wBACR,YAAY,EAAE,wBAAY,CAAC,YAAY,CAAC;qBACzC;oBACD,MAAM,EAAE,MAAM,CAAC,eAAe,CAAC;wBAC7B,MAAM,CAAC,gBAAgB,CAAC;4BACtB,MAAM,EAAE,SAAS;yBAClB,CAAC;qBACH,CAAC;iBACH,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC,iBAAiB,CAAC;gBACpD,MAAM,EAAE,oDAAoD;aAC7D,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,cAAc,GAAG,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;YAChE,MAAM,gBAAgB,GAAG,IAAI,8BAAoB,CAC/C,YAAY,EACZ,cAAc,EACd,SAAS,EACT,YAAY,EACZ,EAAE,QAAQ,EAAE,cAAc,EAAE,CAC7B,CAAC;YAEF,MAAM,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YAExC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAC/C,MAAM,CAAC,gBAAgB,CAAC,wDAAwD,CAAC,CAClF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC,iBAAiB,CAAC;gBACpD,MAAM,EAAE,EAAE;gBACV,KAAK,EAAE,WAAW;aACnB,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YAChD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/redteam/plugins/harmful/common.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=common.test.d.ts.map

package/dist/test/redteam/plugins/harmful/common.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.test.d.ts","sourceRoot":"","sources":["../../../../../test/redteam/plugins/harmful/common.test.ts"],"names":[],"mappings":""}

package/dist/test/redteam/plugins/harmful/common.test.js

@@ -0,0 +1,105 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const envars_1 = require("../../../../src/envars");
+const constants_1 = require("../../../../src/redteam/constants");
+const constants_2 = require("../../../../src/redteam/constants");
+const common_1 = require("../../../../src/redteam/plugins/harmful/common");
+jest.mock('../../../../src/envars');
+describe('harmful plugin', () => {
+    beforeEach(() => {
+        jest.mocked(envars_1.getEnvBool).mockReset();
+        jest.mocked(envars_1.getEnvString).mockReset();
+    });
+    describe('getHarmfulAssertions', () => {
+        beforeEach(() => {
+            jest.mocked(envars_1.getEnvBool).mockReset();
+            jest.mocked(envars_1.getEnvString).mockReset();
+        });
+        it('should return basic assertion for privacy category', () => {
+            const assertions = (0, common_1.getHarmfulAssertions)('harmful:privacy');
+            expect(assertions).toEqual([
+                { metric: 'Harmful', type: 'promptfoo:redteam:harmful:privacy' },
+            ]);
+        });
+        it('should return basic assertion for non-privacy category', () => {
+            const assertions = (0, common_1.getHarmfulAssertions)('harmful:intellectual-property');
+            expect(assertions).toEqual([{ metric: 'Harmful', type: 'promptfoo:redteam:harmful' }]);
+        });
+        it('should return only basic assertions when moderation is disabled', () => {
+            jest.mocked(envars_1.getEnvBool).mockImplementation((key) => {
+                if (key === 'PROMPTFOO_DISABLE_REDTEAM_MODERATION') {
+                    return true;
+                }
+                return false;
+            });
+            jest.mocked(envars_1.getEnvString).mockReturnValue('test-key');
+            const assertions = (0, common_1.getHarmfulAssertions)('harmful:privacy');
+            expect(assertions).toHaveLength(1);
+            expect(assertions).not.toContainEqual(expect.objectContaining({ type: 'moderation' }));
+        });
+        it('should prioritize Replicate over OpenAI when both keys are present', () => {
+            jest.mocked(envars_1.getEnvString).mockImplementation((key) => {
+                if (key === 'REPLICATE_API_KEY') {
+                    return 'replicate-key';
+                }
+                if (key === 'OPENAI_API_KEY') {
+                    return 'openai-key';
+                }
+                return '';
+            });
+            const assertions = (0, common_1.getHarmfulAssertions)('harmful:privacy');
+            expect(assertions).toContainEqual({
+                metric: 'Harmful',
+                type: 'moderation',
+                value: constants_2.LLAMA_GUARD_ENABLED_CATEGORIES,
+                provider: constants_2.LLAMA_GUARD_REPLICATE_PROVIDER,
+            });
+            expect(assertions).toHaveLength(2);
+        });
+        it('should handle REPLICATE_API_TOKEN', () => {
+            jest.mocked(envars_1.getEnvString).mockImplementation((key) => {
+                if (key === 'REPLICATE_API_TOKEN') {
+                    return 'replicate-token';
+                }
+                return '';
+            });
+            const assertions = (0, common_1.getHarmfulAssertions)('harmful:privacy');
+            expect(assertions).toContainEqual({
+                metric: 'Harmful',
+                type: 'moderation',
+                value: constants_2.LLAMA_GUARD_ENABLED_CATEGORIES,
+                provider: constants_2.LLAMA_GUARD_REPLICATE_PROVIDER,
+            });
+        });
+    });
+    describe('createTestCase', () => {
+        it('should create test case with single line output', () => {
+            const testCase = (0, common_1.createTestCase)('inputVar', 'single line output', 'harmful:privacy');
+            expect(testCase).toEqual({
+                vars: {
+                    inputVar: 'single line output',
+                    harmCategory: constants_1.HARM_PLUGINS['harmful:privacy'],
+                },
+                metadata: {
+                    harmCategory: constants_1.HARM_PLUGINS['harmful:privacy'],
+                },
+                assert: expect.any(Array),
+            });
+        });
+        it('should handle multiline output', () => {
+            const testCase = (0, common_1.createTestCase)('inputVar', 'first line\nsecond line\nthird line', 'harmful:privacy');
+            expect(testCase.vars?.inputVar).toBe('first line\nsecond line\nthird line');
+        });
+        it('should handle whitespace in output', () => {
+            const testCase = (0, common_1.createTestCase)('inputVar', '  padded output  ', 'harmful:privacy');
+            expect(testCase.vars?.inputVar).toBe('padded output');
+        });
+        it('should use harm category as fallback when not in HARM_PLUGINS', () => {
+            const unknownCategory = 'harmful:unknown';
+            const testCase = (0, common_1.createTestCase)('inputVar', 'test output', unknownCategory);
+            expect(testCase.vars?.harmCategory).toBe(unknownCategory);
+            expect(testCase.metadata?.harmCategory).toBe(unknownCategory);
+        });
+    });
+});
+//# sourceMappingURL=common.test.js.map

package/dist/test/redteam/plugins/harmful/common.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.test.js","sourceRoot":"","sources":["../../../../../test/redteam/plugins/harmful/common.test.ts"],"names":[],"mappings":";;AAAA,mDAAkE;AAClE,iEAAiE;AACjE,iEAG2C;AAC3C,2EAGwD;AAExD,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AAEpC,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,MAAM,CAAC,mBAAU,CAAC,CAAC,SAAS,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,SAAS,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;QACpC,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,mBAAU,CAAC,CAAC,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,SAAS,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;YAC5D,MAAM,UAAU,GAAG,IAAA,6BAAoB,EAAC,iBAAiB,CAAC,CAAC;YAC3D,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;gBACzB,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,mCAAmC,EAAE;aACjE,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,UAAU,GAAG,IAAA,6BAAoB,EAAC,+BAA+B,CAAC,CAAC;YACzE,MAAM,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;YACzE,IAAI,CAAC,MAAM,CAAC,mBAAU,CAAC,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE;gBACjD,IAAI,GAAG,KAAK,sCAAsC,EAAE,CAAC;oBACnD,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;YAEtD,MAAM,UAAU,GAAG,IAAA,6BAAoB,EAAC,iBAAiB,CAAC,CAAC;YAC3D,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oEAAoE,EAAE,GAAG,EAAE;YAC5E,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnD,IAAI,GAAG,KAAK,mBAAmB,EAAE,CAAC;oBAChC,OAAO,eAAe,CAAC;gBACzB,CAAC;gBACD,IAAI,GAAG,KAAK,gBAAgB,EAAE,CAAC;oBAC7B,OAAO,YAAY,CAAC;gBACtB,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAA,6BAAoB,EAAC,iBAAiB,CAAC,CAAC;YAC3D,MAAM,CAAC,UAAU,CAAC,CAAC,cAAc,CAAC;gBAChC,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,0CAA8B;gBACrC,QAAQ,EAAE,0CAA8B;aACzC,CAAC,CAAC;YACH,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnD,IAAI,GAAG,KAAK,qBAAqB,EAAE,CAAC;oBAClC,OAAO,iBAAiB,CAAC;gBAC3B,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,IAAA,6BAAoB,EAAC,iBAAiB,CAAC,CAAC;YAC3D,MAAM,CAAC,UAAU,CAAC,CAAC,cAAc,CAAC;gBAChC,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,0CAA8B;gBACrC,QAAQ,EAAE,0CAA8B;aACzC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,QAAQ,GAAG,IAAA,uBAAc,EAAC,UAAU,EAAE,oBAAoB,EAAE,iBAAiB,CAAC,CAAC;YAErF,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;gBACvB,IAAI,EAAE;oBACJ,QAAQ,EAAE,oBAAoB;oBAC9B,YAAY,EAAE,wBAAY,CAAC,iBAAiB,CAAC;iBAC9C;gBACD,QAAQ,EAAE;oBACR,YAAY,EAAE,wBAAY,CAAC,iBAAiB,CAAC;iBAC9C;gBACD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;aAC1B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,QAAQ,GAAG,IAAA,uBAAc,EAC7B,UAAU,EACV,qCAAqC,EACrC,iBAAiB,CAClB,CAAC;YAEF,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,QAAQ,GAAG,IAAA,uBAAc,EAAC,UAAU,EAAE,mBAAmB,EAAE,iBAAiB,CAAC,CAAC;YACpF,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;YACvE,MAAM,eAAe,GAAG,iBAA8C,CAAC;YACvE,MAAM,QAAQ,GAAG,IAAA,uBAAc,EAAC,UAAU,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;YAE5E,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC1D,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/redteam/plugins/harmful/unaligned.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=unaligned.test.d.ts.map

package/dist/test/redteam/plugins/harmful/unaligned.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"unaligned.test.d.ts","sourceRoot":"","sources":["../../../../../test/redteam/plugins/harmful/unaligned.test.ts"],"names":[],"mappings":""}

package/dist/test/redteam/plugins/harmful/unaligned.test.js

@@ -0,0 +1,150 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const envars_1 = require("../../../../src/envars");
+const promptfoo_1 = require("../../../../src/providers/promptfoo");
+const constants_1 = require("../../../../src/redteam/constants");
+const constants_2 = require("../../../../src/redteam/constants");
+const constants_3 = require("../../../../src/redteam/plugins/harmful/constants");
+const unaligned_1 = require("../../../../src/redteam/plugins/harmful/unaligned");
+jest.mock('../../../../src/envars');
+describe('harmful plugin', () => {
+    let mockProvider;
+    let mockCallApi;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        mockProvider = {
+            callApi: jest.fn(),
+            id: jest.fn().mockReturnValue('test-provider'),
+        };
+        if (mockCallApi) {
+            mockCallApi.mockRestore();
+        }
+        mockCallApi = jest.spyOn(promptfoo_1.PromptfooHarmfulCompletionProvider.prototype, 'callApi').mockReset();
+        jest.mocked(envars_1.getEnvBool).mockReset();
+        jest.mocked(envars_1.getEnvString).mockReset();
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+        if (mockCallApi) {
+            mockCallApi.mockRestore();
+        }
+    });
+    describe('getHarmfulTests', () => {
+        it('should handle unaligned provider plugins with multiple prompts', async () => {
+            const unalignedPlugin = Object.keys(constants_1.UNALIGNED_PROVIDER_HARM_PLUGINS)[0];
+            mockCallApi.mockResolvedValueOnce({
+                output: ['Test harmful output', 'Another test output'],
+            });
+            const result = await (0, unaligned_1.getHarmfulTests)({
+                provider: mockProvider,
+                purpose: 'test purpose',
+                injectVar: 'testVar',
+                n: 2,
+                delayMs: 0,
+            }, unalignedPlugin);
+            expect(mockCallApi).toHaveBeenCalledTimes(1);
+            expect(result).toHaveLength(2);
+            const prompts = result.map((r) => r.vars?.testVar);
+            expect(prompts).toContain('Test harmful output');
+            expect(prompts).toContain('Another test output');
+        });
+        it('should retry when not enough unique prompts are returned', async () => {
+            const unalignedPlugin = Object.keys(constants_1.UNALIGNED_PROVIDER_HARM_PLUGINS)[0];
+            mockCallApi
+                .mockResolvedValueOnce({ output: ['Test output'] })
+                .mockResolvedValueOnce({ output: ['Test output'] })
+                .mockResolvedValueOnce({ output: ['Another test output'] });
+            const result = await (0, unaligned_1.getHarmfulTests)({
+                provider: mockProvider,
+                purpose: 'test purpose',
+                injectVar: 'testVar',
+                n: 2,
+                delayMs: 0,
+            }, unalignedPlugin);
+            expect(mockCallApi).toHaveBeenCalledTimes(3);
+            expect(result).toHaveLength(2);
+            const prompts = result.map((r) => r.vars?.testVar);
+            expect(prompts).toContain('Test output');
+            expect(prompts).toContain('Another test output');
+        });
+        it('should handle empty provider response', async () => {
+            mockCallApi.mockResolvedValue({ output: [] });
+            const result = await (0, unaligned_1.getHarmfulTests)({
+                provider: mockProvider,
+                purpose: 'test purpose',
+                injectVar: 'testVar',
+                n: 1,
+                delayMs: 0,
+            }, 'harmful:sex-crime');
+            expect(result).toHaveLength(0);
+        });
+        it('should respect delay parameter between API calls', async () => {
+            const unalignedPlugin = Object.keys(constants_1.UNALIGNED_PROVIDER_HARM_PLUGINS)[0];
+            mockCallApi
+                .mockResolvedValueOnce({ output: ['Test output'] })
+                .mockResolvedValueOnce({ output: ['Another output'] });
+            const startTime = Date.now();
+            await (0, unaligned_1.getHarmfulTests)({
+                provider: mockProvider,
+                purpose: 'test purpose',
+                injectVar: 'testVar',
+                n: 2,
+                delayMs: 100,
+            }, unalignedPlugin);
+            expect(Date.now() - startTime).toBeGreaterThanOrEqual(100);
+        });
+        it('should handle moderation assertions with OPENAI_API_KEY', async () => {
+            jest.mocked(envars_1.getEnvString).mockImplementation((key) => {
+                if (key === 'OPENAI_API_KEY') {
+                    return 'test-key';
+                }
+                return '';
+            });
+            mockCallApi.mockResolvedValueOnce({ output: ['Test output'] });
+            const result = await (0, unaligned_1.getHarmfulTests)({
+                provider: mockProvider,
+                purpose: 'test purpose',
+                injectVar: 'testVar',
+                n: 1,
+                delayMs: 0,
+            }, 'harmful:sex-crime');
+            expect(result).toHaveLength(1);
+            expect(result[0].vars?.testVar).toBe('Test output');
+            expect(result[0].assert).toContainEqual(expect.objectContaining({ type: 'moderation' }));
+        });
+        it('should handle moderation assertions with REPLICATE_API_KEY', async () => {
+            const mockCategory = {
+                key: 'harmful:privacy',
+                prompt: 'test prompt',
+                examples: 'example1',
+                label: 'Privacy violations',
+                description: 'test description',
+            };
+            jest.spyOn(constants_3.REDTEAM_MODEL_CATEGORIES, 'find').mockReturnValue(mockCategory);
+            jest.mocked(envars_1.getEnvString).mockImplementation((key) => {
+                if (key === 'REPLICATE_API_KEY') {
+                    return 'test-key';
+                }
+                return '';
+            });
+            const mockOutput = 'Test output';
+            jest
+                .spyOn(promptfoo_1.PromptfooHarmfulCompletionProvider.prototype, 'callApi')
+                .mockResolvedValue({ output: [mockOutput] });
+            const result = await (0, unaligned_1.getHarmfulTests)({
+                provider: mockProvider,
+                purpose: 'test purpose',
+                injectVar: 'testVar',
+                n: 1,
+                delayMs: 0,
+            }, 'harmful:sex-crime');
+            expect(result).toHaveLength(1);
+            expect(result[0].vars?.testVar).toBe('Test output');
+            expect(result[0].assert).toContainEqual(expect.objectContaining({
+                type: 'moderation',
+                provider: constants_2.LLAMA_GUARD_REPLICATE_PROVIDER,
+            }));
+        });
+    });
+});
+//# sourceMappingURL=unaligned.test.js.map

package/dist/test/redteam/plugins/harmful/unaligned.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unaligned.test.js","sourceRoot":"","sources":["../../../../../test/redteam/plugins/harmful/unaligned.test.ts"],"names":[],"mappings":";;AAAA,mDAAkE;AAClE,mEAAyF;AACzF,iEAAoF;AACpF,iEAAmF;AAEnF,iFAA6F;AAC7F,iFAAoF;AAGpF,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;AAEpC,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,YAAyB,CAAC;IAC9B,IAAI,WAA6B,CAAC;IAElC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,YAAY,GAAG;YACb,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,eAAe,CAAC;SAC/C,CAAC;QACF,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,WAAW,EAAE,CAAC;QAC5B,CAAC;QACD,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,8CAAkC,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,SAAS,EAAE,CAAC;QAE9F,IAAI,CAAC,MAAM,CAAC,mBAAU,CAAC,CAAC,SAAS,EAAE,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,SAAS,EAAE,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,WAAW,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,2CAA+B,CAAC,CAAC,CAAC,CAAC,CAAC;YAExE,WAAW,CAAC,qBAAqB,CAAC;gBAChC,MAAM,EAAE,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;aACvD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,EACD,eAA+D,CAChE,CAAC;YAEF,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;YACjD,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACxE,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,2CAA+B,CAAC,CAAC,CAAC,CAAC,CAAC;YAExE,WAAW;iBACR,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC;iBAClD,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC;iBAClD,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE9D,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,EACD,eAA+D,CAChE,CAAC;YAEF,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACzC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,WAAW,CAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAE9C,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,EACD,mBAAmB,CACpB,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,2CAA+B,CAAC,CAAC,CAAC,CAAC,CAAC;YACxE,WAAW;iBACR,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC;iBAClD,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YAEzD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,IAAA,2BAAe,EACnB;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,GAAG;aACb,EACD,eAA+D,CAChE,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnD,IAAI,GAAG,KAAK,gBAAgB,EAAE,CAAC;oBAC7B,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,WAAW,CAAC,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YAE/D,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,EACD,mBAAmB,CACpB,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,GAAoB;gBACpC,GAAG,EAAE,iBAAiB;gBACtB,MAAM,EAAE,aAAa;gBACrB,QAAQ,EAAE,UAAU;gBACpB,KAAK,EAAE,oBAAoB;gBAC3B,WAAW,EAAE,kBAAkB;aAChC,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,oCAAwB,EAAE,MAAM,CAAC,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;YAE3E,IAAI,CAAC,MAAM,CAAC,qBAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE;gBACnD,IAAI,GAAG,KAAK,mBAAmB,EAAE,CAAC;oBAChC,OAAO,UAAU,CAAC;gBACpB,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,aAAa,CAAC;YACjC,IAAI;iBACD,KAAK,CAAC,8CAAkC,CAAC,SAAS,EAAE,SAAS,CAAC;iBAC9D,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,IAAA,2BAAe,EAClC;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,cAAc;gBACvB,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,EACD,mBAAmB,CACpB,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,cAAc,CACrC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,IAAI,EAAE,YAAY;gBAClB,QAAQ,EAAE,0CAA8B;aACzC,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/redteam/plugins/index.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=index.test.d.ts.map

package/dist/test/redteam/plugins/index.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.test.d.ts","sourceRoot":"","sources":["../../../../test/redteam/plugins/index.test.ts"],"names":[],"mappings":""}

package/dist/test/redteam/plugins/index.test.js

@@ -0,0 +1,172 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cache_1 = require("../../../src/cache");
+const constants_1 = require("../../../src/constants");
+const logger_1 = __importDefault(require("../../../src/logger"));
+const constants_2 = require("../../../src/redteam/constants");
+const plugins_1 = require("../../../src/redteam/plugins");
+const util_1 = require("../../../src/redteam/util");
+jest.mock('../../../src/cache');
+jest.mock('../../../src/logger');
+jest.mock('../../../src/redteam/util', () => ({
+    shouldGenerateRemote: jest.fn().mockReturnValue(false),
+    neverGenerateRemote: jest.fn().mockReturnValue(false),
+}));
+jest.mock('../../../src/cliState', () => ({
+    __esModule: true,
+    default: { remote: false },
+}));
+describe('Plugins', () => {
+    let mockProvider;
+    beforeEach(() => {
+        mockProvider = {
+            callApi: jest.fn(),
+            id: jest.fn().mockReturnValue('test-provider'),
+        };
+        // Reset all mocks
+        jest.clearAllMocks();
+        jest.mocked(cache_1.fetchWithCache).mockReset();
+    });
+    describe('plugin registration', () => {
+        it('should register all base plugins', () => {
+            const basePluginKeys = [
+                'contracts',
+                'cross-session-leak',
+                'debug-access',
+                'excessive-agency',
+                'hallucination',
+                'imitation',
+                'intent',
+                'overreliance',
+                'politics',
+                'policy',
+                'prompt-extraction',
+                'rbac',
+                'shell-injection',
+                'sql-injection',
+            ];
+            basePluginKeys.forEach((key) => {
+                const plugin = plugins_1.Plugins.find((p) => p.key === key);
+                expect(plugin).toBeDefined();
+            });
+        });
+        it('should register all aligned harm plugins', () => {
+            Object.keys(constants_2.REDTEAM_PROVIDER_HARM_PLUGINS).forEach((key) => {
+                const plugin = plugins_1.Plugins.find((p) => p.key === key);
+                expect(plugin).toBeDefined();
+            });
+        });
+        it('should register all unaligned harm plugins', () => {
+            Object.keys(constants_2.UNALIGNED_PROVIDER_HARM_PLUGINS).forEach((key) => {
+                const plugin = plugins_1.Plugins.find((p) => p.key === key);
+                expect(plugin).toBeDefined();
+            });
+        });
+        it('should register all PII plugins', () => {
+            constants_2.PII_PLUGINS.forEach((key) => {
+                const plugin = plugins_1.Plugins.find((p) => p.key === key);
+                expect(plugin).toBeDefined();
+            });
+        });
+        it('should register all remote plugins', () => {
+            const remotePluginKeys = [
+                'ascii-smuggling',
+                'bfla',
+                'bola',
+                'competitors',
+                'hijacking',
+                'religion',
+                'ssrf',
+                'indirect-prompt-injection',
+            ];
+            remotePluginKeys.forEach((key) => {
+                const plugin = plugins_1.Plugins.find((p) => p.key === key);
+                expect(plugin).toBeDefined();
+            });
+        });
+    });
+    describe('plugin validation', () => {
+        it('should validate intent plugin config', async () => {
+            const intentPlugin = plugins_1.Plugins.find((p) => p.key === 'intent');
+            expect(() => intentPlugin?.validate?.({})).toThrow('Intent plugin requires `config.intent` to be set');
+        });
+        it('should validate policy plugin config', async () => {
+            const policyPlugin = plugins_1.Plugins.find((p) => p.key === 'policy');
+            expect(() => policyPlugin?.validate?.({})).toThrow('Policy plugin requires `config.policy` to be set');
+        });
+        it('should validate prompt extraction plugin config', async () => {
+            const promptExtractionPlugin = plugins_1.Plugins.find((p) => p.key === 'prompt-extraction');
+            expect(() => promptExtractionPlugin?.validate?.({})).toThrow('Prompt extraction plugin requires `config.systemPrompt` to be set');
+        });
+        it('should validate indirect prompt injection plugin config', async () => {
+            const indirectPlugin = plugins_1.Plugins.find((p) => p.key === 'indirect-prompt-injection');
+            expect(() => indirectPlugin?.validate?.({})).toThrow('Indirect prompt injection plugin requires `config.indirectInjectionVar` to be set');
+        });
+    });
+    describe('remote generation', () => {
+        it('should call remote generation with correct parameters', async () => {
+            jest.mocked(util_1.shouldGenerateRemote).mockReturnValue(true);
+            jest.mocked(util_1.neverGenerateRemote).mockReturnValue(false);
+            const mockResponse = {
+                data: { result: [{ test: 'case' }] },
+                cached: false,
+                status: 200,
+                statusText: 'OK',
+            };
+            jest.mocked(cache_1.fetchWithCache).mockResolvedValue(mockResponse);
+            const plugin = plugins_1.Plugins.find((p) => p.key === 'contracts');
+            const result = await plugin?.action({
+                provider: mockProvider,
+                purpose: 'test',
+                injectVar: 'testVar',
+                n: 1,
+                config: {},
+                delayMs: 0,
+            });
+            expect(cache_1.fetchWithCache).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    config: {},
+                    injectVar: 'testVar',
+                    n: 1,
+                    purpose: 'test',
+                    task: 'contracts',
+                    version: constants_1.VERSION,
+                }),
+            }), expect.any(Number));
+            expect(result).toEqual([{ test: 'case' }]);
+        });
+        it('should handle remote generation errors', async () => {
+            jest.mocked(cache_1.fetchWithCache).mockRejectedValue(new Error('Network error'));
+            const plugin = plugins_1.Plugins.find((p) => p.key === 'contracts');
+            const result = await plugin?.action({
+                provider: mockProvider,
+                purpose: 'test',
+                injectVar: 'testVar',
+                n: 1,
+                delayMs: 0,
+            });
+            expect(logger_1.default.error).toHaveBeenCalledWith(expect.stringContaining('Network error'));
+            expect(result).toEqual([]);
+        });
+    });
+    describe('unaligned harm plugins', () => {
+        it('should require remote generation', async () => {
+            jest.mocked(util_1.shouldGenerateRemote).mockReturnValue(false);
+            jest.mocked(util_1.neverGenerateRemote).mockReturnValue(true);
+            const unalignedPlugin = plugins_1.Plugins.find((p) => p.key === Object.keys(constants_2.UNALIGNED_PROVIDER_HARM_PLUGINS)[0]);
+            await expect(unalignedPlugin?.action({
+                provider: mockProvider,
+                purpose: 'test',
+                injectVar: 'testVar',
+                n: 1,
+                delayMs: 0,
+            })).rejects.toThrow('requires remote generation to be enabled');
+        });
+    });
+});
+//# sourceMappingURL=index.test.js.map

package/dist/test/redteam/plugins/index.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../../../test/redteam/plugins/index.test.ts"],"names":[],"mappings":";;;;;AACA,8CAAoD;AACpD,sDAAiD;AACjD,iEAAyC;AACzC,8DAIwC;AACxC,0DAAuD;AACvD,oDAAsF;AAGtF,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAChC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;AACjC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5C,oBAAoB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IACtD,mBAAmB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;CACtD,CAAC,CAAC,CAAC;AACJ,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAAC,CAAC;IACxC,UAAU,EAAE,IAAI;IAChB,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;CAC3B,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACvB,IAAI,YAAyB,CAAC;IAE9B,UAAU,CAAC,GAAG,EAAE;QACd,YAAY,GAAG;YACb,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,eAAe,CAAC;SAC/C,CAAC;QAEF,kBAAkB;QAClB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,sBAAc,CAAC,CAAC,SAAS,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,cAAc,GAAG;gBACrB,WAAW;gBACX,oBAAoB;gBACpB,cAAc;gBACd,kBAAkB;gBAClB,eAAe;gBACf,WAAW;gBACX,QAAQ;gBACR,cAAc;gBACd,UAAU;gBACV,QAAQ;gBACR,mBAAmB;gBACnB,MAAM;gBACN,iBAAiB;gBACjB,eAAe;aAChB,CAAC;YAEF,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC7B,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAClD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,IAAI,CAAC,yCAA6B,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBACzD,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAClD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,CAAC,IAAI,CAAC,2CAA+B,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC3D,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAClD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,uBAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC1B,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAClD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,gBAAgB,GAAG;gBACvB,iBAAiB;gBACjB,MAAM;gBACN,MAAM;gBACN,aAAa;gBACb,WAAW;gBACX,UAAU;gBACV,MAAM;gBACN,2BAA2B;aAC5B,CAAC;YAEF,gBAAgB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAClD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC;YAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAChD,kDAAkD,CACnD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC;YAC7D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAChD,kDAAkD,CACnD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,sBAAsB,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;YAClF,MAAM,CAAC,GAAG,EAAE,CAAC,sBAAsB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAC1D,mEAAmE,CACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,cAAc,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC;YAClF,MAAM,CAAC,GAAG,EAAE,CAAC,cAAc,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAClD,mFAAmF,CACpF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,IAAI,CAAC,MAAM,CAAC,2BAAoB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACxD,IAAI,CAAC,MAAM,CAAC,0BAAmB,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,YAAY,GAAkC;gBAClD,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;gBACpC,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,IAAI;aACjB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,sBAAc,CAAC,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE5D,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,WAAW,CAAC,CAAC;YAC1D,MAAM,MAAM,GAAG,MAAM,MAAM,EAAE,MAAM,CAAC;gBAClC,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;YAEH,MAAM,CAAC,sBAAc,CAAC,CAAC,oBAAoB,CACzC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAClB,MAAM,CAAC,gBAAgB,CAAC;gBACtB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,MAAM,EAAE,EAAE;oBACV,SAAS,EAAE,SAAS;oBACpB,CAAC,EAAE,CAAC;oBACJ,OAAO,EAAE,MAAM;oBACf,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,mBAAO;iBACjB,CAAC;aACH,CAAC,EACF,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CACnB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,IAAI,CAAC,MAAM,CAAC,sBAAc,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAE1E,MAAM,MAAM,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,WAAW,CAAC,CAAC;YAC1D,MAAM,MAAM,GAAG,MAAM,MAAM,EAAE,MAAM,CAAC;gBAClC,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,CAAC,CAAC;YAEH,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,IAAI,CAAC,MAAM,CAAC,2BAAoB,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC,0BAAmB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YACvD,MAAM,eAAe,GAAG,iBAAO,CAAC,IAAI,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,IAAI,CAAC,2CAA+B,CAAC,CAAC,CAAC,CAAC,CACjE,CAAC;YACF,MAAM,MAAM,CACV,eAAe,EAAE,MAAM,CAAC;gBACtB,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,SAAS;gBACpB,CAAC,EAAE,CAAC;gBACJ,OAAO,EAAE,CAAC;aACX,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}