promptfoo 0.100.0 → 0.100.2

package/dist/test/redteam/strategies/multilingual.test.js

@@ -0,0 +1,158 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cli_progress_1 = require("cli-progress");
+const cache_1 = require("../../../src/cache");
+const logger_1 = __importDefault(require("../../../src/logger"));
+const remoteGeneration_1 = require("../../../src/redteam/remoteGeneration");
+const multilingual_1 = require("../../../src/redteam/strategies/multilingual");
+jest.mock('../../../src/logger', () => ({
+    debug: jest.fn(),
+    error: jest.fn(),
+    level: 'info',
+}));
+jest.mock('../../../src/redteam/remoteGeneration', () => ({
+    getRemoteGenerationUrl: jest.fn().mockReturnValue('http://test.url'),
+    shouldGenerateRemote: jest.fn().mockReturnValue(false),
+}));
+jest.mock('../../../src/cache', () => ({
+    fetchWithCache: jest.fn().mockResolvedValue({
+        data: { result: [] },
+    }),
+}));
+jest.mock('../../../src/redteam/providers/shared', () => ({
+    loadRedteamProvider: jest.fn().mockResolvedValue({
+        callApi: jest.fn().mockResolvedValue({
+            output: '{"translation": "Hallo Welt"}',
+        }),
+    }),
+}));
+jest.mock('cli-progress', () => ({
+    Presets: {
+        shades_classic: {},
+    },
+    SingleBar: jest.fn().mockImplementation(() => ({
+        increment: jest.fn(),
+        start: jest.fn(),
+        stop: jest.fn(),
+    })),
+}));
+describe('addMultilingual', () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+    afterEach(() => {
+        jest.resetModules();
+    });
+    it('should preserve harmCategory and modify assertion metrics', async () => {
+        const testCase = {
+            assert: [{ type: 'promptfoo:redteam:contracts' }, { type: 'promptfoo:redteam:harmful' }],
+            metadata: {
+                harmCategory: 'Illegal Activities - Fraud & scams',
+                otherField: 'value',
+            },
+            vars: {
+                text: 'Hello world',
+            },
+        };
+        const result = await (0, multilingual_1.addMultilingual)([testCase], 'text', { languages: ['de'] });
+        expect(result[0].metadata).toEqual({
+            harmCategory: 'Illegal Activities - Fraud & scams',
+            otherField: 'value',
+        });
+        expect(result[0].assert).toEqual([
+            {
+                metric: 'contracts/Multilingual-DE',
+                type: 'promptfoo:redteam:contracts',
+            },
+            {
+                metric: 'harmful/Multilingual-DE',
+                type: 'promptfoo:redteam:harmful',
+            },
+        ]);
+    });
+    it('should handle test cases without metadata', async () => {
+        const testCase = {
+            vars: {
+                text: 'Hello world',
+            },
+        };
+        const result = await (0, multilingual_1.addMultilingual)([testCase], 'text', { languages: ['de'] });
+        expect(result[0].metadata).toEqual({});
+    });
+    it('should translate text and update vars correctly', async () => {
+        const testCase = {
+            vars: {
+                otherVar: 'test',
+                text: 'Hello world',
+            },
+        };
+        const result = await (0, multilingual_1.addMultilingual)([testCase], 'text', { languages: ['de'] });
+        expect(result[0].vars).toEqual({
+            otherVar: 'test',
+            text: 'Hallo Welt',
+        });
+    });
+    it('should increment progress bar when provided', async () => {
+        const testCase = {
+            vars: {
+                text: 'Hello world',
+            },
+        };
+        logger_1.default.level = 'info';
+        await (0, multilingual_1.addMultilingual)([testCase], 'text', { languages: ['de'] });
+        const mockBarInstance = jest.mocked(cli_progress_1.SingleBar).mock.results[0].value;
+        expect(mockBarInstance.increment).toHaveBeenCalledWith(1);
+        expect(mockBarInstance.stop).toHaveBeenCalledWith();
+    });
+    it('should attempt remote generation first', async () => {
+        const testCase = {
+            vars: { text: 'Hello world' },
+        };
+        jest.mocked(remoteGeneration_1.shouldGenerateRemote).mockReturnValueOnce(true);
+        jest.mocked(cache_1.fetchWithCache).mockResolvedValueOnce({
+            cached: false,
+            data: {
+                result: [
+                    {
+                        vars: { text: 'Hallo Welt' },
+                    },
+                ],
+            },
+            status: 200,
+            statusText: 'OK',
+        });
+        const result = await (0, multilingual_1.addMultilingual)([testCase], 'text', { languages: ['es'] });
+        expect(result).toHaveLength(1);
+        expect(result[0].vars?.text).toBe('Hallo Welt');
+    });
+    it('should preserve harmCategory and modify assertion metrics only for redteam types', async () => {
+        const testCase = {
+            assert: [{ type: 'promptfoo:redteam:harmful' }, { type: 'moderation' }],
+            metadata: {
+                harmCategory: 'Illegal Activities - Fraud & scams',
+                otherField: 'value',
+            },
+            vars: {
+                text: 'Hello world',
+            },
+        };
+        const result = await (0, multilingual_1.addMultilingual)([testCase], 'text', { languages: ['de'] });
+        expect(result[0].metadata).toEqual({
+            harmCategory: 'Illegal Activities - Fraud & scams',
+            otherField: 'value',
+        });
+        expect(result[0].assert).toEqual([
+            {
+                metric: 'harmful/Multilingual-DE',
+                type: 'promptfoo:redteam:harmful',
+            },
+            {
+                type: 'moderation',
+            },
+        ]);
+    });
+});
+//# sourceMappingURL=multilingual.test.js.map

package/dist/test/redteam/strategies/multilingual.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"multilingual.test.js","sourceRoot":"","sources":["../../../../test/redteam/strategies/multilingual.test.ts"],"names":[],"mappings":";;;;;AAAA,+CAAyC;AACzC,8CAAoD;AACpD,iEAAyC;AACzC,4EAA6E;AAC7E,+EAA+E;AAG/E,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACtC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;IAChB,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;IAChB,KAAK,EAAE,MAAM;CACd,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,uCAAuC,EAAE,GAAG,EAAE,CAAC,CAAC;IACxD,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,iBAAiB,CAAC;IACpE,oBAAoB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;CACvD,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;QAC1C,IAAI,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;KACrB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,uCAAuC,EAAE,GAAG,EAAE,CAAC,CAAC;IACxD,mBAAmB,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;QAC/C,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;YACnC,MAAM,EAAE,+BAA+B;SACxC,CAAC;KACH,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;IAC/B,OAAO,EAAE;QACP,cAAc,EAAE,EAAE;KACnB;IACD,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7C,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;QACpB,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE;QAChB,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;KAChB,CAAC,CAAC;CACJ,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,QAAQ,GAAmB;YAC/B,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,6BAA6B,EAAE,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC;YACxF,QAAQ,EAAE;gBACR,YAAY,EAAE,oCAAoC;gBAClD,UAAU,EAAE,OAAO;aACpB;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE,aAAa;aACpB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,8BAAe,EAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;YACjC,YAAY,EAAE,oCAAoC;YAClD,UAAU,EAAE,OAAO;SACpB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YAC/B;gBACE,MAAM,EAAE,2BAA2B;gBACnC,IAAI,EAAE,6BAA6B;aACpC;YACD;gBACE,MAAM,EAAE,yBAAyB;gBACjC,IAAI,EAAE,2BAA2B;aAClC;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,QAAQ,GAAmB;YAC/B,IAAI,EAAE;gBACJ,IAAI,EAAE,aAAa;aACpB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,8BAAe,EAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,QAAQ,GAAmB;YAC/B,IAAI,EAAE;gBACJ,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,aAAa;aACpB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,8BAAe,EAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;YAC7B,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,YAAY;SACnB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,QAAQ,GAAmB;YAC/B,IAAI,EAAE;gBACJ,IAAI,EAAE,aAAa;aACpB;SACF,CAAC;QAED,gBAAc,CAAC,KAAK,GAAG,MAAM,CAAC;QAE/B,MAAM,IAAA,8BAAe,EAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEjE,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,wBAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAErE,MAAM,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;QAC1D,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,QAAQ,GAAmB;YAC/B,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;SAC9B,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,uCAAoB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,sBAAc,CAAC,CAAC,qBAAqB,CAAC;YAChD,MAAM,EAAE,KAAK;YACb,IAAI,EAAE;gBACJ,MAAM,EAAE;oBACN;wBACE,IAAI,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;qBAC7B;iBACF;aACF;YACD,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,8BAAe,EAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kFAAkF,EAAE,KAAK,IAAI,EAAE;QAChG,MAAM,QAAQ,GAAmB;YAC/B,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC;YACvE,QAAQ,EAAE;gBACR,YAAY,EAAE,oCAAoC;gBAClD,UAAU,EAAE,OAAO;aACpB;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE,aAAa;aACpB;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,8BAAe,EAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC;YACjC,YAAY,EAAE,oCAAoC;YAClD,UAAU,EAAE,OAAO;SACpB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YAC/B;gBACE,MAAM,EAAE,yBAAyB;gBACjC,IAAI,EAAE,2BAA2B;aAClC;YACD;gBACE,IAAI,EAAE,YAAY;aACnB;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}