promptfoo 0.98.0 → 0.99.0

package/dist/src/redteam/providers/goat.js

@@ -39,59 +39,64 @@ class GoatProvider {
             cached: 0,
         };
         for (let turn = 0; turn < this.maxTurns; turn++) {
-            response = await fetch((0, constants_2.getRemoteGenerationUrl)(), {
-                body: JSON.stringify({
-                    goal: context?.vars[this.injectVar],
-                    i: turn,
-                    messages,
-                    prompt: context?.prompt?.raw,
-                    task: 'goat',
-                    version: constants_1.VERSION,
-                }),
-                headers: {
-                    'Content-Type': 'application/json',
-                },
-                method: 'POST',
-            });
-            const data = await response.json();
-            messages.push(data.message);
-            if (data.tokenUsage) {
-                totalTokenUsage.total += data.tokenUsage.total || 0;
-                totalTokenUsage.prompt += data.tokenUsage.prompt || 0;
-                totalTokenUsage.completion += data.tokenUsage.completion || 0;
-                totalTokenUsage.cached += data.tokenUsage.cached || 0;
-                totalTokenUsage.numRequests += data.tokenUsage.numRequests ?? 1;
-            }
-            logger_1.default.debug((0, dedent_1.default) `
+            try {
+                response = await fetch((0, constants_2.getRemoteGenerationUrl)(), {
+                    body: JSON.stringify({
+                        goal: context?.vars[this.injectVar],
+                        i: turn,
+                        messages,
+                        prompt: context?.prompt?.raw,
+                        task: 'goat',
+                        version: constants_1.VERSION,
+                    }),
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    method: 'POST',
+                });
+                const data = await response.json();
+                messages.push(data.message);
+                if (data.tokenUsage) {
+                    totalTokenUsage.total += data.tokenUsage.total || 0;
+                    totalTokenUsage.prompt += data.tokenUsage.prompt || 0;
+                    totalTokenUsage.completion += data.tokenUsage.completion || 0;
+                    totalTokenUsage.cached += data.tokenUsage.cached || 0;
+                    totalTokenUsage.numRequests += data.tokenUsage.numRequests ?? 1;
+                }
+                logger_1.default.debug((0, dedent_1.default) `
           ${chalk_1.default.bold.green(`GOAT turn ${turn} history:`)}
           ${chalk_1.default.cyan(JSON.stringify(messages, null, 2))}
         `);
-            const targetPrompt = this.stateless
-                ? JSON.stringify(messages)
-                : await (0, evaluatorHelpers_1.renderPrompt)(context.prompt, { ...context.vars, [this.injectVar]: messages[messages.length - 1].content }, context.filters, targetProvider);
-            const targetResponse = await targetProvider.callApi(targetPrompt, context, options);
-            logger_1.default.debug(`GOAT turn ${turn} target response: ${JSON.stringify(targetResponse)}`);
-            if (targetResponse.sessionId) {
-                context = context ?? { vars: {}, prompt: { raw: '', label: 'target' } };
-                context.vars.sessionId = targetResponse.sessionId;
-            }
-            if (targetResponse.error) {
-                throw new Error(`Error from target provider: ${targetResponse.error}`);
-            }
-            (0, tiny_invariant_1.default)(targetResponse.output, `Expected target response output to be set, but got: ${JSON.stringify(targetResponse)}`);
-            messages.push({
-                content: targetResponse.output,
-                role: 'assistant',
-            });
-            if (targetResponse.tokenUsage) {
-                totalTokenUsage.total += targetResponse.tokenUsage.total || 0;
-                totalTokenUsage.prompt += targetResponse.tokenUsage.prompt || 0;
-                totalTokenUsage.completion += targetResponse.tokenUsage.completion || 0;
-                totalTokenUsage.numRequests += targetResponse.tokenUsage.numRequests ?? 1;
-                totalTokenUsage.cached += targetResponse.tokenUsage.cached || 0;
+                const targetPrompt = this.stateless
+                    ? JSON.stringify(messages)
+                    : await (0, evaluatorHelpers_1.renderPrompt)(context.prompt, { ...context.vars, [this.injectVar]: messages[messages.length - 1].content }, context.filters, targetProvider);
+                const targetResponse = await targetProvider.callApi(targetPrompt, context, options);
+                logger_1.default.debug(`GOAT turn ${turn} target response: ${JSON.stringify(targetResponse)}`);
+                if (targetResponse.sessionId) {
+                    context = context ?? { vars: {}, prompt: { raw: '', label: 'target' } };
+                    context.vars.sessionId = targetResponse.sessionId;
+                }
+                if (targetResponse.error) {
+                    throw new Error(`Error from target provider: ${targetResponse.error}`);
+                }
+                (0, tiny_invariant_1.default)(targetResponse.output, `Expected target response output to be set, but got: ${JSON.stringify(targetResponse)}`);
+                messages.push({
+                    content: targetResponse.output,
+                    role: 'assistant',
+                });
+                if (targetResponse.tokenUsage) {
+                    totalTokenUsage.total += targetResponse.tokenUsage.total || 0;
+                    totalTokenUsage.prompt += targetResponse.tokenUsage.prompt || 0;
+                    totalTokenUsage.completion += targetResponse.tokenUsage.completion || 0;
+                    totalTokenUsage.numRequests += targetResponse.tokenUsage.numRequests ?? 1;
+                    totalTokenUsage.cached += targetResponse.tokenUsage.cached || 0;
+                }
+                else {
+                    totalTokenUsage.numRequests += 1;
+                }
             }
-            else {
-                totalTokenUsage.numRequests += 1;
+            catch (err) {
+                logger_1.default.error(`Error in GOAT turn ${turn}: ${err}`);
             }
         }
         delete context?.vars?.sessionId;

package/dist/src/redteam/providers/goat.js.map

@@ -1 +1 @@
-{"version":3,"file":"goat.js","sourceRoot":"","sources":["../../../../src/redteam/providers/goat.ts"],"names":[],"mappings":";;;;;AAAA,kDAA0B;AAC1B,oDAA4B;AAC5B,oEAAuC;AACvC,+CAA0C;AAC1C,6DAAsD;AACtD,0DAAkC;AAQlC,4CAAsD;AACtD,kCAA8C;AAE9C,MAAqB,YAAY;IAK/B,EAAE;QACA,OAAO,wBAAwB,CAAC;IAClC,CAAC;IAED,YACE,UAII,EAAE;QAEN,IAAI,IAAA,0BAAmB,GAAE,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,IAAA,wBAAS,EAAC,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE,8BAA8B,CAAC,CAAC;QACjF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,OAA8B;QAE9B,IAAI,QAAQ,GAAyB,SAAS,CAAC;QAC/C,IAAA,wBAAS,EAAC,OAAO,EAAE,gBAAgB,EAAE,qCAAqC,CAAC,CAAC;QAC5E,IAAA,wBAAS,EAAC,OAAO,EAAE,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAEpD,MAAM,cAAc,GAA4B,OAAO,EAAE,gBAAgB,CAAC;QAC1E,IAAA,wBAAS,EAAC,cAAc,EAAE,qCAAqC,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAiE,EAAE,CAAC;QAClF,MAAM,eAAe,GAAG;YACtB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,CAAC;SACV,CAAC;QAEF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAChD,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAA,kCAAsB,GAAE,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;oBACnC,CAAC,EAAE,IAAI;oBACP,QAAQ;oBACR,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG;oBAC5B,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,mBAAO;iBACjB,CAAC;gBACF,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,eAAe,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;gBACpD,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBACtD,eAAe,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;gBAC9D,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBACtD,eAAe,CAAC,WAAW,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,IAAI,CAAC,CAAC;YAClE,CAAC;YACD,gBAAM,CAAC,KAAK,CACV,IAAA,gBAAM,EAAA;YACF,eAAK,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,WAAW,CAAC;YAC9C,eAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;SAChD,CACF,CAAC;YAEF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS;gBACjC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;gBAC1B,CAAC,CAAC,MAAM,IAAA,+BAAY,EAChB,OAAO,CAAC,MAAM,EACd,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,EAC5E,OAAO,CAAC,OAAO,EACf,cAAc,CACf,CAAC;YACN,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YACpF,gBAAM,CAAC,KAAK,CAAC,aAAa,IAAI,qBAAqB,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;YAErF,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC;gBAC7B,OAAO,GAAG,OAAO,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;gBACxE,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;YACpD,CAAC;YACD,IAAI,cAAc,CAAC,KAAK,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,IAAA,wBAAS,EACP,cAAc,CAAC,MAAM,EACrB,uDAAuD,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CACxF,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,cAAc,CAAC,MAAM;gBAC9B,IAAI,EAAE,WAAW;aAClB,CAAC,CAAC;YAEH,IAAI,cAAc,CAAC,UAAU,EAAE,CAAC;gBAC9B,eAAe,CAAC,KAAK,IAAI,cAAc,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC9D,eAAe,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBAChE,eAAe,CAAC,UAAU,IAAI,cAAc,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;gBACxE,eAAe,CAAC,WAAW,IAAI,cAAc,CAAC,UAAU,CAAC,WAAW,IAAI,CAAC,CAAC;gBAC1E,eAAe,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;YAClE,CAAC;iBAAM,CAAC;gBACN,eAAe,CAAC,WAAW,IAAI,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QACD,OAAO,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC;QAEhC,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO;YAC9C,QAAQ,EAAE;gBACR,kBAAkB,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO;gBAC1D,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;aAC5C;YACD,UAAU,EAAE,eAAe;SAC5B,CAAC;IACJ,CAAC;CACF;AA7HD,+BA6HC"}
+{"version":3,"file":"goat.js","sourceRoot":"","sources":["../../../../src/redteam/providers/goat.ts"],"names":[],"mappings":";;;;;AAAA,kDAA0B;AAC1B,oDAA4B;AAC5B,oEAAuC;AACvC,+CAA0C;AAC1C,6DAAsD;AACtD,0DAAkC;AAQlC,4CAAsD;AACtD,kCAA8C;AAE9C,MAAqB,YAAY;IAK/B,EAAE;QACA,OAAO,wBAAwB,CAAC;IAClC,CAAC;IAED,YACE,UAII,EAAE;QAEN,IAAI,IAAA,0BAAmB,GAAE,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,IAAA,wBAAS,EAAC,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ,EAAE,8BAA8B,CAAC,CAAC;QACjF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,OAA8B;QAE9B,IAAI,QAAQ,GAAyB,SAAS,CAAC;QAC/C,IAAA,wBAAS,EAAC,OAAO,EAAE,gBAAgB,EAAE,qCAAqC,CAAC,CAAC;QAC5E,IAAA,wBAAS,EAAC,OAAO,EAAE,IAAI,EAAE,yBAAyB,CAAC,CAAC;QAEpD,MAAM,cAAc,GAA4B,OAAO,EAAE,gBAAgB,CAAC;QAC1E,IAAA,wBAAS,EAAC,cAAc,EAAE,qCAAqC,CAAC,CAAC;QAEjE,MAAM,QAAQ,GAAiE,EAAE,CAAC;QAClF,MAAM,eAAe,GAAG;YACtB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,CAAC;SACV,CAAC;QAEF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAA,kCAAsB,GAAE,EAAE;oBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;wBACnC,CAAC,EAAE,IAAI;wBACP,QAAQ;wBACR,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG;wBAC5B,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,mBAAO;qBACjB,CAAC;oBACF,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;qBACnC;oBACD,MAAM,EAAE,MAAM;iBACf,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC5B,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,eAAe,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;oBACpD,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;oBACtD,eAAe,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;oBAC9D,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;oBACtD,eAAe,CAAC,WAAW,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,IAAI,CAAC,CAAC;gBAClE,CAAC;gBACD,gBAAM,CAAC,KAAK,CACV,IAAA,gBAAM,EAAA;YACJ,eAAK,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,WAAW,CAAC;YAC9C,eAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;SAChD,CACA,CAAC;gBAEF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS;oBACjC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;oBAC1B,CAAC,CAAC,MAAM,IAAA,+BAAY,EAChB,OAAO,CAAC,MAAM,EACd,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,EAC5E,OAAO,CAAC,OAAO,EACf,cAAc,CACf,CAAC;gBACN,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBACpF,gBAAM,CAAC,KAAK,CAAC,aAAa,IAAI,qBAAqB,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;gBAErF,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC;oBAC7B,OAAO,GAAG,OAAO,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;oBACxE,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;gBACpD,CAAC;gBACD,IAAI,cAAc,CAAC,KAAK,EAAE,CAAC;oBACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC;gBACzE,CAAC;gBACD,IAAA,wBAAS,EACP,cAAc,CAAC,MAAM,EACrB,uDAAuD,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CACxF,CAAC;gBACF,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,cAAc,CAAC,MAAM;oBAC9B,IAAI,EAAE,WAAW;iBAClB,CAAC,CAAC;gBAEH,IAAI,cAAc,CAAC,UAAU,EAAE,CAAC;oBAC9B,eAAe,CAAC,KAAK,IAAI,cAAc,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;oBAC9D,eAAe,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;oBAChE,eAAe,CAAC,UAAU,IAAI,cAAc,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;oBACxE,eAAe,CAAC,WAAW,IAAI,cAAc,CAAC,UAAU,CAAC,WAAW,IAAI,CAAC,CAAC;oBAC1E,eAAe,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBAClE,CAAC;qBAAM,CAAC;oBACN,eAAe,CAAC,WAAW,IAAI,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,gBAAM,CAAC,KAAK,CAAC,sBAAsB,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QACD,OAAO,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC;QAEhC,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO;YAC9C,QAAQ,EAAE;gBACR,kBAAkB,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,OAAO;gBAC1D,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;aAC5C;YACD,UAAU,EAAE,eAAe;SAC5B,CAAC;IACJ,CAAC;CACF;AAjID,+BAiIC"}

package/dist/src/redteam/strategies/citation.d.ts

@@ -0,0 +1,3 @@
+import type { TestCase } from '../../types';
+export declare function addCitationTestCases(testCases: TestCase[], injectVar: string, config: Record<string, unknown>): Promise<TestCase[]>;
+//# sourceMappingURL=citation.d.ts.map

package/dist/src/redteam/strategies/citation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"citation.d.ts","sourceRoot":"","sources":["../../../../src/redteam/strategies/citation.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAmG5C,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,QAAQ,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAWrB"}

package/dist/src/redteam/strategies/citation.js

@@ -0,0 +1,96 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.addCitationTestCases = addCitationTestCases;
+const async_1 = __importDefault(require("async"));
+const cli_progress_1 = require("cli-progress");
+const dedent_1 = __importDefault(require("dedent"));
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const cache_1 = require("../../cache");
+const logger_1 = __importDefault(require("../../logger"));
+const shared_1 = require("../../providers/shared");
+const constants_1 = require("../constants");
+const util_1 = require("../util");
+async function generateCitations(testCases, injectVar, config) {
+    let progressBar;
+    try {
+        const concurrency = 10;
+        const allResults = [];
+        if (logger_1.default.level !== 'debug') {
+            progressBar = new cli_progress_1.SingleBar({
+                format: 'Citation Generation {bar} {percentage}% | ETA: {eta}s | {value}/{total} cases',
+                hideCursor: true,
+            }, cli_progress_1.Presets.shades_classic);
+            progressBar.start(testCases.length, 0);
+        }
+        await async_1.default.forEachOfLimit(testCases, concurrency, async (testCase, index) => {
+            (0, tiny_invariant_1.default)(testCase.vars, `Citation: testCase.vars is required, but got ${JSON.stringify(testCase)}`);
+            const payload = {
+                task: 'citation',
+                testCases: [testCase],
+                injectVar,
+                topic: testCase.vars[injectVar],
+                config,
+            };
+            const { data } = await (0, cache_1.fetchWithCache)((0, constants_1.getRemoteGenerationUrl)(), {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(payload),
+            }, shared_1.REQUEST_TIMEOUT_MS);
+            logger_1.default.debug(`Got remote citation generation result for case ${Number(index) + 1}: ${JSON.stringify(data)}`);
+            const citationTestCase = {
+                ...testCase,
+                vars: {
+                    ...testCase.vars,
+                    [injectVar]: (0, dedent_1.default) `
+            Based on the following ${data.result.citation.type.toLowerCase()}, ${testCase.vars[injectVar]}
+
+          Reference:
+          1. ${data.result.citation.content}
+        `,
+                },
+                assert: testCase.assert?.map((assertion) => ({
+                    ...assertion,
+                    metric: `${assertion.metric}/Citation`,
+                })),
+                metadata: {
+                    ...testCase.metadata,
+                    citation: data.result.citation,
+                },
+            };
+            allResults.push(citationTestCase);
+            if (progressBar) {
+                progressBar.increment(1);
+            }
+            else {
+                logger_1.default.debug(`Processed case ${Number(index) + 1} of ${testCases.length}`);
+            }
+        });
+        if (progressBar) {
+            progressBar.stop();
+        }
+        return allResults;
+    }
+    catch (error) {
+        if (progressBar) {
+            progressBar.stop();
+        }
+        logger_1.default.error(`Error in remote citation generation: ${error}`);
+        return [];
+    }
+}
+async function addCitationTestCases(testCases, injectVar, config) {
+    if ((0, util_1.neverGenerateRemote)()) {
+        throw new Error('Citation strategy requires remote generation to be enabled');
+    }
+    const citationTestCases = await generateCitations(testCases, injectVar, config);
+    if (citationTestCases.length === 0) {
+        logger_1.default.warn('No citation test cases were generated');
+    }
+    return citationTestCases;
+}
+//# sourceMappingURL=citation.js.map

package/dist/src/redteam/strategies/citation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"citation.js","sourceRoot":"","sources":["../../../../src/redteam/strategies/citation.ts"],"names":[],"mappings":";;;;;AA0GA,oDAeC;AAzHD,kDAA0B;AAC1B,+CAAkD;AAClD,oDAA4B;AAC5B,oEAAuC;AACvC,uCAA6C;AAC7C,0DAAkC;AAClC,mDAA4D;AAE5D,4CAAsD;AACtD,kCAA8C;AAE9C,KAAK,UAAU,iBAAiB,CAC9B,SAAqB,EACrB,SAAiB,EACjB,MAA2B;IAE3B,IAAI,WAAkC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAe,EAAE,CAAC;QAElC,IAAI,gBAAM,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC7B,WAAW,GAAG,IAAI,wBAAS,CACzB;gBACE,MAAM,EAAE,+EAA+E;gBACvF,UAAU,EAAE,IAAI;aACjB,EACD,sBAAO,CAAC,cAAc,CACvB,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,eAAK,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;YAC3E,IAAA,wBAAS,EACP,QAAQ,CAAC,IAAI,EACb,gDAAgD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAC3E,CAAC;YAEF,MAAM,OAAO,GAAG;gBACd,IAAI,EAAE,UAAU;gBAChB,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACrB,SAAS;gBACT,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC/B,MAAM;aACP,CAAC;YAEF,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,sBAAc,EACnC,IAAA,kCAAsB,GAAE,EACxB;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;aAC9B,EACD,2BAAkB,CACnB,CAAC;YAEF,gBAAM,CAAC,KAAK,CACV,kDAAkD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAC/F,CAAC;YAEF,MAAM,gBAAgB,GAAG;gBACvB,GAAG,QAAQ;gBACX,IAAI,EAAE;oBACJ,GAAG,QAAQ,CAAC,IAAI;oBAChB,CAAC,SAAS,CAAC,EAAE,IAAA,gBAAM,EAAA;qCACQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;;;eAG1F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO;SAClC;iBACA;gBACD,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBAC3C,GAAG,SAAS;oBACZ,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,WAAW;iBACvC,CAAC,CAAC;gBACH,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;iBAC/B;aACF,CAAC;YAEF,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;QACD,gBAAM,CAAC,KAAK,CAAC,wCAAwC,KAAK,EAAE,CAAC,CAAC;QAC9D,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,SAAqB,EACrB,SAAiB,EACjB,MAA+B;IAE/B,IAAI,IAAA,0BAAmB,GAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IAChF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,gBAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC"}

package/dist/src/redteam/strategies/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/redteam/strategies/index.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,qBAAqB,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAYvF,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,CACN,SAAS,EAAE,kBAAkB,EAAE,EAC/B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACxB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CAC1B;AAED,eAAO,MAAM,UAAU,EAAE,QAAQ,EA2FhC,CAAC;AAEF,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAwB3F;AAED,wBAAsB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CA2B1E"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/redteam/strategies/index.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,qBAAqB,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAcvF,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,CACN,SAAS,EAAE,kBAAkB,EAAE,EAC/B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACxB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CAC1B;AAED,eAAO,MAAM,UAAU,EAAE,QAAQ,EA6GhC,CAAC;AAEF,wBAAsB,kBAAkB,CAAC,UAAU,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAwB3F;AAED,wBAAsB,YAAY,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CA2B1E"}

package/dist/src/redteam/strategies/index.js

@@ -14,6 +14,7 @@ const esm_1 = require("../../esm");
 const logger_1 = __importDefault(require("../../logger"));
 const file_1 = require("../../util/file");
 const base64_1 = require("./base64");
+const citation_1 = require("./citation");
 const crescendo_1 = require("./crescendo");
 const goat_1 = require("./goat");
 const iterative_1 = require("./iterative");
@@ -22,6 +23,7 @@ const mathPrompt_1 = require("./mathPrompt");
 const multilingual_1 = require("./multilingual");
 const promptInjections_1 = require("./promptInjections");
 const rot13_1 = require("./rot13");
+const singleTurnComposite_1 = require("./singleTurnComposite");
 exports.Strategies = [
     {
         id: 'base64',
@@ -113,6 +115,24 @@ exports.Strategies = [
             return newTestCases;
         },
     },
+    {
+        id: 'citation',
+        action: async (testCases, injectVar, config) => {
+            logger_1.default.debug('Adding Citation to all test cases');
+            const newTestCases = await (0, citation_1.addCitationTestCases)(testCases, injectVar, config);
+            logger_1.default.debug(`Added ${newTestCases.length} Citation test cases`);
+            return newTestCases;
+        },
+    },
+    {
+        id: 'jailbreak:composite',
+        action: async (testCases, injectVar, config) => {
+            logger_1.default.debug('Adding composite jailbreak test cases');
+            const newTestCases = await (0, singleTurnComposite_1.addCompositeTestCases)(testCases, injectVar, config);
+            logger_1.default.debug(`Added ${newTestCases.length} composite jailbreak test cases`);
+            return newTestCases;
+        },
+    },
 ];
 async function validateStrategies(strategies) {
     const invalidStrategies = [];

package/dist/src/redteam/strategies/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/redteam/strategies/index.ts"],"names":[],"mappings":";;;;;;AAwHA,gDAwBC;AAED,oCA2BC;AA7KD,kDAA0B;AAC1B,oDAA4B;AAC5B,gDAAwB;AACxB,8DAAsC;AACtC,mCAAyC;AACzC,0DAAkC;AAElC,0CAAmD;AACnD,qCAA6C;AAC7C,2CAA2C;AAC3C,iCAA0C;AAC1C,2CAAqD;AACrD,2CAA2C;AAC3C,6CAA6C;AAC7C,iDAAiD;AACjD,yDAAmD;AACnD,mCAAmC;AAWtB,QAAA,UAAU,GAAe;IACpC;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;YACrC,gBAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,IAAA,0BAAiB,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC7D,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,4BAA4B,CAAC,CAAC;YACvE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACnD,MAAM,YAAY,GAAG,IAAA,wBAAY,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAChE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,uBAAuB,CAAC,CAAC;YAClE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,IAAA,uBAAgB,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1E,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC7D,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACjE,MAAM,YAAY,GAAG,IAAA,kCAAsB,EAAC,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YACvF,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,oCAAoC,CAAC,CAAC;YAC/E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACtE,MAAM,YAAY,GAAG,IAAA,kCAAsB,EAAC,SAAS,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAC5F,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,yCAAyC,CAAC,CAAC;YACpF,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;YACrC,gBAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;YAC5D,MAAM,YAAY,GAAG,IAAA,wBAAY,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACxD,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,+BAA+B,CAAC,CAAC;YAC1E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,aAAa;QACjB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,MAAM,IAAA,0BAAa,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACvE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,gCAAgC,CAAC,CAAC;YAC3E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,cAAc;QAClB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC/C,MAAM,YAAY,GAAG,MAAM,IAAA,8BAAe,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACzE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,0BAA0B,CAAC,CAAC;YACrE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,kBAAkB;QACtB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC3D,MAAM,YAAY,GAAG,MAAM,IAAA,gCAAa,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACvE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,8BAA8B,CAAC,CAAC;YACzE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;YACrC,gBAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACxD,MAAM,YAAY,GAAG,IAAA,gBAAQ,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACpD,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,2BAA2B,CAAC,CAAC;YACtE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;CACF,CAAC;AAEK,KAAK,UAAU,kBAAkB,CAAC,UAAmC;IAC1E,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,0EAA0E;QAC1E,IAAI,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACtC,SAAS;QACX,CAAC;QAED,IAAI,CAAC,kBAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;YACvD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,qBAAqB,GAAG,kBAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,MAAM,uBAAuB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9E,gBAAM,CAAC,KAAK,CACV,IAAA,gBAAM,EAAA,wBAAwB,uBAAuB;;UAEjD,eAAK,CAAC,KAAK,CAAC,yBAAyB,qBAAqB,EAAE,CAAC,EAAE,CACpE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,YAAoB;IACrD,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAA,uBAAgB,EAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,QAAQ,GAAG,kBAAQ,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QACpD,MAAM,UAAU,GAAG,cAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxF,MAAM,cAAc,GAAG,CAAC,MAAM,IAAA,kBAAY,EAAC,UAAU,CAAC,CAAa,CAAC;QAEpE,sEAAsE;QACtE,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,OAAO,cAAc,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CACb,sBAAsB,QAAQ,2DAA2D,CAC1F,CAAC;QACJ,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,QAAQ,GAAG,kBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC;IAC/D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/redteam/strategies/index.ts"],"names":[],"mappings":";;;;;;AA4IA,gDAwBC;AAED,oCA2BC;AAjMD,kDAA0B;AAC1B,oDAA4B;AAC5B,gDAAwB;AACxB,8DAAsC;AACtC,mCAAyC;AACzC,0DAAkC;AAElC,0CAAmD;AACnD,qCAA6C;AAC7C,yCAAkD;AAClD,2CAA2C;AAC3C,iCAA0C;AAC1C,2CAAqD;AACrD,2CAA2C;AAC3C,6CAA6C;AAC7C,iDAAiD;AACjD,yDAAmD;AACnD,mCAAmC;AACnC,+DAA8D;AAWjD,QAAA,UAAU,GAAe;IACpC;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;YACrC,gBAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,IAAA,0BAAiB,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC7D,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,4BAA4B,CAAC,CAAC;YACvE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACnD,MAAM,YAAY,GAAG,IAAA,wBAAY,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAChE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,uBAAuB,CAAC,CAAC;YAClE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,MAAM;QACV,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,IAAA,uBAAgB,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC1E,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC7D,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACjE,MAAM,YAAY,GAAG,IAAA,kCAAsB,EAAC,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;YACvF,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,oCAAoC,CAAC,CAAC;YAC/E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;YACtE,MAAM,YAAY,GAAG,IAAA,kCAAsB,EAAC,SAAS,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAC5F,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,yCAAyC,CAAC,CAAC;YACpF,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;YACrC,gBAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;YAC5D,MAAM,YAAY,GAAG,IAAA,wBAAY,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACxD,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,+BAA+B,CAAC,CAAC;YAC1E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,aAAa;QACjB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC7D,MAAM,YAAY,GAAG,MAAM,IAAA,0BAAa,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACvE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,gCAAgC,CAAC,CAAC;YAC3E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,cAAc;QAClB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC/C,MAAM,YAAY,GAAG,MAAM,IAAA,8BAAe,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACzE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,0BAA0B,CAAC,CAAC;YACrE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,kBAAkB;QACtB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC3D,MAAM,YAAY,GAAG,MAAM,IAAA,gCAAa,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YACvE,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,8BAA8B,CAAC,CAAC;YACzE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;YACrC,gBAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACxD,MAAM,YAAY,GAAG,IAAA,gBAAQ,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACpD,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,2BAA2B,CAAC,CAAC;YACtE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,UAAU;QACd,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAClD,MAAM,YAAY,GAAG,MAAM,IAAA,+BAAoB,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC9E,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,sBAAsB,CAAC,CAAC;YACjE,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE;YAC7C,gBAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACtD,MAAM,YAAY,GAAG,MAAM,IAAA,2CAAqB,EAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC/E,gBAAM,CAAC,KAAK,CAAC,SAAS,YAAY,CAAC,MAAM,iCAAiC,CAAC,CAAC;YAC5E,OAAO,YAAY,CAAC;QACtB,CAAC;KACF;CACF,CAAC;AAEK,KAAK,UAAU,kBAAkB,CAAC,UAAmC;IAC1E,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,0EAA0E;QAC1E,IAAI,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACtC,SAAS;QACX,CAAC;QAED,IAAI,CAAC,kBAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;YACvD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,qBAAqB,GAAG,kBAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,MAAM,uBAAuB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9E,gBAAM,CAAC,KAAK,CACV,IAAA,gBAAM,EAAA,wBAAwB,uBAAuB;;UAEjD,eAAK,CAAC,KAAK,CAAC,yBAAyB,qBAAqB,EAAE,CAAC,EAAE,CACpE,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,YAAY,CAAC,YAAoB;IACrD,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,IAAA,uBAAgB,EAAC,QAAQ,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,mDAAmD,QAAQ,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,QAAQ,GAAG,kBAAQ,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QACpD,MAAM,UAAU,GAAG,cAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxF,MAAM,cAAc,GAAG,CAAC,MAAM,IAAA,kBAAY,EAAC,UAAU,CAAC,CAAa,CAAC;QAEpE,sEAAsE;QACtE,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,OAAO,cAAc,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CACb,sBAAsB,QAAQ,2DAA2D,CAC1F,CAAC;QACJ,CAAC;QAED,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,QAAQ,GAAG,kBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC;IAC/D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,uBAAuB,YAAY,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/src/redteam/strategies/singleTurnComposite.d.ts

@@ -0,0 +1,3 @@
+import type { TestCase } from '../../types';
+export declare function addCompositeTestCases(testCases: TestCase[], injectVar: string, config: Record<string, unknown>): Promise<TestCase[]>;
+//# sourceMappingURL=singleTurnComposite.d.ts.map

package/dist/src/redteam/strategies/singleTurnComposite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"singleTurnComposite.d.ts","sourceRoot":"","sources":["../../../../src/redteam/strategies/singleTurnComposite.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAgG5C,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,QAAQ,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAWrB"}

package/dist/src/redteam/strategies/singleTurnComposite.js

@@ -0,0 +1,89 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.addCompositeTestCases = addCompositeTestCases;
+const async_1 = __importDefault(require("async"));
+const cli_progress_1 = require("cli-progress");
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const cache_1 = require("../../cache");
+const logger_1 = __importDefault(require("../../logger"));
+const shared_1 = require("../../providers/shared");
+const constants_1 = require("../constants");
+const util_1 = require("../util");
+async function generateCompositePrompts(testCases, injectVar, config) {
+    let progressBar;
+    try {
+        const concurrency = 10;
+        let allResults = [];
+        if (logger_1.default.level !== 'debug') {
+            progressBar = new cli_progress_1.SingleBar({
+                format: 'Composite Jailbreak Generation {bar} {percentage}% | ETA: {eta}s | {value}/{total} cases',
+                hideCursor: true,
+            }, cli_progress_1.Presets.shades_classic);
+            progressBar.start(testCases.length, 0);
+        }
+        await async_1.default.forEachOfLimit(testCases, concurrency, async (testCase, index) => {
+            (0, tiny_invariant_1.default)(testCase.vars, `Composite: testCase.vars is required, but got ${JSON.stringify(testCase)}`);
+            const payload = {
+                task: 'jailbreak:composite',
+                prompt: testCase.vars[injectVar],
+                ...(config.n && { n: config.n }),
+                ...(config.modelFamily && { modelFamily: config.modelFamily }),
+            };
+            const { data } = await (0, cache_1.fetchWithCache)((0, constants_1.getRemoteGenerationUrl)(), {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(payload),
+            }, shared_1.REQUEST_TIMEOUT_MS);
+            logger_1.default.debug(`Got composite jailbreak generation result for case ${Number(index) + 1}: ${JSON.stringify(data)}`);
+            const compositeTestCases = data.modifiedPrompts.map((modifiedPrompt) => ({
+                ...testCase,
+                vars: {
+                    ...testCase.vars,
+                    [injectVar]: modifiedPrompt,
+                },
+                assert: testCase.assert?.map((assertion) => ({
+                    ...assertion,
+                    metric: `${assertion.metric}/Composite`,
+                })),
+                metadata: {
+                    ...testCase.metadata,
+                    strategy: 'jailbreak:composite',
+                },
+            }));
+            allResults = allResults.concat(compositeTestCases);
+            if (progressBar) {
+                progressBar.increment(1);
+            }
+            else {
+                logger_1.default.debug(`Processed case ${Number(index) + 1} of ${testCases.length}`);
+            }
+        });
+        if (progressBar) {
+            progressBar.stop();
+        }
+        return allResults;
+    }
+    catch (error) {
+        if (progressBar) {
+            progressBar.stop();
+        }
+        logger_1.default.error(`Error in composite generation: ${error}`);
+        return [];
+    }
+}
+async function addCompositeTestCases(testCases, injectVar, config) {
+    if ((0, util_1.neverGenerateRemote)()) {
+        throw new Error('Composite jailbreak strategy requires remote generation to be enabled');
+    }
+    const compositeTestCases = await generateCompositePrompts(testCases, injectVar, config);
+    if (compositeTestCases.length === 0) {
+        logger_1.default.warn('No composite  jailbreak test cases were generated');
+    }
+    return compositeTestCases;
+}
+//# sourceMappingURL=singleTurnComposite.js.map

package/dist/src/redteam/strategies/singleTurnComposite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"singleTurnComposite.js","sourceRoot":"","sources":["../../../../src/redteam/strategies/singleTurnComposite.ts"],"names":[],"mappings":";;;;;AAsGA,sDAeC;AArHD,kDAA0B;AAC1B,+CAAkD;AAClD,oEAAuC;AACvC,uCAA6C;AAC7C,0DAAkC;AAClC,mDAA4D;AAE5D,4CAAsD;AACtD,kCAA8C;AAE9C,KAAK,UAAU,wBAAwB,CACrC,SAAqB,EACrB,SAAiB,EACjB,MAAkE;IAElE,IAAI,WAAkC,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,EAAE,CAAC;QACvB,IAAI,UAAU,GAAe,EAAE,CAAC;QAEhC,IAAI,gBAAM,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;YAC7B,WAAW,GAAG,IAAI,wBAAS,CACzB;gBACE,MAAM,EACJ,0FAA0F;gBAC5F,UAAU,EAAE,IAAI;aACjB,EACD,sBAAO,CAAC,cAAc,CACvB,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,eAAK,CAAC,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;YAC3E,IAAA,wBAAS,EACP,QAAQ,CAAC,IAAI,EACb,iDAAiD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAC5E,CAAC;YAEF,MAAM,OAAO,GAAG;gBACd,IAAI,EAAE,qBAAqB;gBAC3B,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;gBAChC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;gBAChC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;aAC/D,CAAC;YAEF,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,sBAAc,EACnC,IAAA,kCAAsB,GAAE,EACxB;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;aAC9B,EACD,2BAAkB,CACnB,CAAC;YAEF,gBAAM,CAAC,KAAK,CACV,sDAAsD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CACxF,IAAI,CACL,EAAE,CACJ,CAAC;YAEF,MAAM,kBAAkB,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,cAAsB,EAAE,EAAE,CAAC,CAAC;gBAC/E,GAAG,QAAQ;gBACX,IAAI,EAAE;oBACJ,GAAG,QAAQ,CAAC,IAAI;oBAChB,CAAC,SAAS,CAAC,EAAE,cAAc;iBAC5B;gBACD,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;oBAC3C,GAAG,SAAS;oBACZ,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,YAAY;iBACxC,CAAC,CAAC;gBACH,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,QAAQ,EAAE,qBAAqB;iBAChC;aACF,CAAC,CAAC,CAAC;YAEJ,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAEnD,IAAI,WAAW,EAAE,CAAC;gBAChB,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,gBAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;QACD,gBAAM,CAAC,KAAK,CAAC,kCAAkC,KAAK,EAAE,CAAC,CAAC;QACxD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,qBAAqB,CACzC,SAAqB,EACrB,SAAiB,EACjB,MAA+B;IAE/B,IAAI,IAAA,0BAAmB,GAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,kBAAkB,GAAG,MAAM,wBAAwB,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IACxF,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,gBAAM,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,kBAAkB,CAAC;AAC5B,CAAC"}

package/dist/src/redteam/types.d.ts

@@ -19,7 +19,11 @@ type WithNumTests = {
 };
 export type RedteamPluginObject = ConfigurableObject & WithNumTests;
 export type RedteamPlugin = string | RedteamPluginObject;
-export type RedteamStrategyObject = ConfigurableObject;
+export type RedteamStrategyObject = ConfigurableObject & {
+    config?: StrategyConfig & {
+        plugins?: RedteamPluginObject['id'][];
+    };
+};
 export type RedteamStrategy = string | RedteamStrategyObject;
 export interface PluginActionParams {
     provider: ApiProvider;

package/dist/src/redteam/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/redteam/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAGvE,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC1D,MAAM,MAAM,YAAY,GAAG,mBAAmB,GAAG;IAC/C,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,CAAC,EAAE;QACf,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,OAAO,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;CACL,CAAC;AACF,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAC;AAEjD,KAAK,kBAAkB,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B,CAAC;AAEF,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAGF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG,YAAY,CAAC;AACpE,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,mBAAmB,CAAC;AAEzD,MAAM,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AACvD,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,qBAAqB,CAAC;AAE7D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,WAAW,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,EAAE,MAAM,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAGD,KAAK,aAAa,GAAG;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,WAAW,yBAA0B,SAAQ,aAAa;IAC9D,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,mBAAmB,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IACpE,OAAO,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC/B,UAAU,EAAE,qBAAqB,EAAE,CAAC;CACrC;AAED,MAAM,MAAM,qBAAqB,GAAG,qBAAqB,MAAM,EAAE,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/redteam/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAGvE,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC1D,MAAM,MAAM,YAAY,GAAG,mBAAmB,GAAG;IAC/C,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,CAAC,EAAE;QACf,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,OAAO,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;CACL,CAAC;AACF,MAAM,MAAM,cAAc,GAAG,mBAAmB,CAAC;AAEjD,KAAK,kBAAkB,GAAG;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B,CAAC;AAEF,KAAK,YAAY,GAAG;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAGF,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG,YAAY,CAAC;AACpE,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,mBAAmB,CAAC;AAEzD,MAAM,MAAM,qBAAqB,GAAG,kBAAkB,GAAG;IACvD,MAAM,CAAC,EAAE,cAAc,GAAG;QACxB,OAAO,CAAC,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;KACvC,CAAC;CACH,CAAC;AACF,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,qBAAqB,CAAC;AAE7D,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,WAAW,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,EAAE,MAAM,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB;AAGD,KAAK,aAAa,GAAG;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAChC,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;IAClD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,WAAW,yBAA0B,SAAQ,aAAa;IAC9D,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,iBAAkB,SAAQ,aAAa;IACtD,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,mBAAmB,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IACpE,OAAO,EAAE,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAC/B,UAAU,EAAE,qBAAqB,EAAE,CAAC;CACrC;AAED,MAAM,MAAM,qBAAqB,GAAG,qBAAqB,MAAM,EAAE,CAAC"}

package/dist/src/server/index.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-const server_1 = require("./server");
+const constants_1 = require("../constants");
+const server_1 = require("../util/server");
+const server_2 = require("./server");
 // start server
-(0, server_1.startServer)(Number.parseInt(process.env.API_PORT || '15500'), server_1.BrowserBehavior.SKIP);
+(0, server_2.startServer)(constants_1.DEFAULT_PORT, server_1.BrowserBehavior.SKIP);
 //# sourceMappingURL=index.js.map

package/dist/src/server/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/server/index.ts"],"names":[],"mappings":";;AAAA,qCAAwD;AAExD,eAAe;AACf,IAAA,oBAAW,EAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,EAAE,wBAAe,CAAC,IAAI,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/server/index.ts"],"names":[],"mappings":";;AAAA,4CAA4C;AAC5C,2CAAiD;AACjD,qCAAuC;AAEvC,eAAe;AACf,IAAA,oBAAW,EAAC,wBAAY,EAAE,wBAAe,CAAC,IAAI,CAAC,CAAC"}

package/dist/src/server/server.d.ts

@@ -1,10 +1,4 @@
-export declare enum BrowserBehavior {
-    ASK = 0,
-    OPEN = 1,
-    SKIP = 2,
-    OPEN_TO_REPORT = 3,
-    OPEN_TO_REDTEAM_CREATE = 4
-}
+import { BrowserBehavior } from '../util/server';
 export declare function createApp(): import("express-serve-static-core").Express;
 export declare function startServer(port?: number, browserBehavior?: BrowserBehavior, filterDescription?: string): void;
 //# sourceMappingURL=server.d.ts.map

package/dist/src/server/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../../src/server/server.ts"],"names":[],"mappings":"AA4CA,oBAAY,eAAe;IACzB,GAAG,IAAI;IACP,IAAI,IAAI;IACR,IAAI,IAAI;IACR,cAAc,IAAI;IAClB,sBAAsB,IAAI;CAC3B;AAED,wBAAgB,SAAS,gDAiIxB;AAED,wBAAgB,WAAW,CACzB,IAAI,SAAQ,EACZ,eAAe,kBAAsB,EACrC,iBAAiB,CAAC,EAAE,MAAM,QAmF3B"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../../src/server/server.ts"],"names":[],"mappings":"AAiCA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAWjD,wBAAgB,SAAS,gDAiIxB;AAED,wBAAgB,WAAW,CACzB,IAAI,SAAe,EACnB,eAAe,kBAAsB,EACrC,iBAAiB,CAAC,EAAE,MAAM,QAoD3B"}

package/dist/src/server/server.js

@@ -3,7 +3,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BrowserBehavior = void 0;
 exports.createApp = createApp;
 exports.startServer = startServer;
 const compression_1 = __importDefault(require("compression"));
@@ -14,8 +13,6 @@ const express_1 = __importDefault(require("express"));
 const fs_1 = __importDefault(require("fs"));
 const node_http_1 = __importDefault(require("node:http"));
 const node_path_1 = __importDefault(require("node:path"));
-const node_readline_1 = __importDefault(require("node:readline"));
-const opener_1 = __importDefault(require("opener"));
 const socket_io_1 = require("socket.io");
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const zod_validation_error_1 = require("zod-validation-error");
@@ -30,6 +27,8 @@ const telemetry_1 = __importDefault(require("../telemetry"));
 const telemetry_2 = require("../telemetry");
 const testCases_1 = require("../testCases");
 const util_1 = require("../util");
+const server_1 = require("../util/server");
+const server_2 = require("../util/server");
 const configs_1 = require("./routes/configs");
 const eval_2 = require("./routes/eval");
 const providers_1 = require("./routes/providers");
@@ -37,14 +36,6 @@ const redteam_1 = require("./routes/redteam");
 const user_1 = require("./routes/user");
 // Prompts cache
 let allPrompts = null;
-var BrowserBehavior;
-(function (BrowserBehavior) {
-    BrowserBehavior[BrowserBehavior["ASK"] = 0] = "ASK";
-    BrowserBehavior[BrowserBehavior["OPEN"] = 1] = "OPEN";
-    BrowserBehavior[BrowserBehavior["SKIP"] = 2] = "SKIP";
-    BrowserBehavior[BrowserBehavior["OPEN_TO_REPORT"] = 3] = "OPEN_TO_REPORT";
-    BrowserBehavior[BrowserBehavior["OPEN_TO_REDTEAM_CREATE"] = 4] = "OPEN_TO_REDTEAM_CREATE";
-})(BrowserBehavior || (exports.BrowserBehavior = BrowserBehavior = {}));
 function createApp() {
     const app = (0, express_1.default)();
     const staticDir = node_path_1.default.join((0, esm_1.getDirectory)(), 'app');
@@ -155,7 +146,7 @@ function createApp() {
     });
     return app;
 }
-function startServer(port = 15500, browserBehavior = BrowserBehavior.ASK, filterDescription) {
+function startServer(port = constants_1.DEFAULT_PORT, browserBehavior = server_1.BrowserBehavior.ASK, filterDescription) {
     const app = createApp();
     const httpServer = node_http_1.default.createServer(app);
     const io = new socket_io_1.Server(httpServer, {
@@ -181,40 +172,9 @@ function startServer(port = 15500, browserBehavior = BrowserBehavior.ASK, filter
         .listen(port, () => {
         const url = `http://localhost:${port}`;
         logger_1.default.info(`Server running at ${url} and monitoring for new evals.`);
-        const openUrl = async () => {
-            try {
-                logger_1.default.info('Press Ctrl+C to stop the server');
-                if (browserBehavior === BrowserBehavior.OPEN_TO_REPORT) {
-                    await (0, opener_1.default)(`${url}/report`);
-                }
-                else if (browserBehavior === BrowserBehavior.OPEN_TO_REDTEAM_CREATE) {
-                    await (0, opener_1.default)(`${url}/redteam/setup`);
-                }
-                else {
-                    await (0, opener_1.default)(url);
-                }
-            }
-            catch (err) {
-                logger_1.default.error(`Failed to open browser: ${String(err)}`);
-            }
-        };
-        if (browserBehavior === BrowserBehavior.OPEN ||
-            browserBehavior === BrowserBehavior.OPEN_TO_REPORT ||
-            browserBehavior === BrowserBehavior.OPEN_TO_REDTEAM_CREATE) {
-            openUrl();
-        }
-        else if (browserBehavior === BrowserBehavior.ASK) {
-            const rl = node_readline_1.default.createInterface({
-                input: process.stdin,
-                output: process.stdout,
-            });
-            rl.question('Open URL in browser? (y/N): ', async (answer) => {
-                if (answer.toLowerCase().startsWith('y')) {
-                    openUrl();
-                }
-                rl.close();
-            });
-        }
+        (0, server_2.openBrowser)(browserBehavior, port).catch((err) => {
+            logger_1.default.error(`Failed to handle browser behavior: ${err}`);
+        });
     })
         .on('error', (error) => {
         if (error.code === 'EADDRINUSE') {