promptfoo 0.96.1 → 0.97.0

package/dist/src/providers/promptfoo.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"promptfoo.d.ts","sourceRoot":"","sources":["../../../src/providers/promptfoo.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EAEb,MAAM,UAAU,CAAC;AAGlB,UAAU,iCAAiC;IACzC,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,kCAAmC,YAAW,WAAW;IACpE,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;gBAET,OAAO,EAAE,iCAAiC;IAKtD,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,cAAc,CAAC,EAAE,oBAAoB,GACpC,OAAO,CAAC,gBAAgB,CAAC;CAyC7B;AAED,UAAU,8BAA8B;IACtC,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;IAClB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,IAAI,EAAE,WAAW,GAAG,MAAM,GAAG,WAAW,GAAG,iBAAiB,GAAG,gBAAgB,CAAC;CACjF;AAED,qBAAa,+BAAgC,YAAW,WAAW;IACjE,OAAO,CAAC,OAAO,CAAiC;gBAEpC,OAAO,EAAE,8BAA8B;IAInD,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,cAAc,CAAC,EAAE,oBAAoB,GACpC,OAAO,CAAC,gBAAgB,CAAC;CAuC7B;AAED,UAAU,qBAAqB;IAC7B,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,8BAA+B,YAAW,WAAW;IAChE,OAAO,CAAC,OAAO,CAAwB;gBAE3B,OAAO,GAAE,qBAA0B;IAI/C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,cAAc,CAAC,EAAE,oBAAoB,GACpC,OAAO,CAAC,gBAAgB,CAAC;CA2C7B"}
+{"version":3,"file":"promptfoo.d.ts","sourceRoot":"","sources":["../../../src/providers/promptfoo.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,oBAAoB,EACpB,oBAAoB,EACpB,YAAY,EAEb,MAAM,UAAU,CAAC;AAGlB,UAAU,iCAAiC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,EAAE,MAAM,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,kCAAmC,YAAW,WAAW;IACpE,YAAY,EAAE,MAAM,CAAC;IACrB,CAAC,EAAE,MAAM,CAAC;IACV,OAAO,EAAE,MAAM,CAAC;gBAEJ,OAAO,EAAE,iCAAiC;IAMtD,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,cAAc,CAAC,EAAE,oBAAoB,GACpC,OAAO,CAAC,gBAAgB,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CAyCrD;AAED,UAAU,8BAA8B;IACtC,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,OAAO,CAAC;IAClB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,IAAI,EAAE,WAAW,GAAG,MAAM,GAAG,WAAW,GAAG,iBAAiB,GAAG,gBAAgB,CAAC;CACjF;AAED,qBAAa,+BAAgC,YAAW,WAAW;IACjE,OAAO,CAAC,OAAO,CAAiC;gBAEpC,OAAO,EAAE,8BAA8B;IAInD,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,cAAc,CAAC,EAAE,oBAAoB,GACpC,OAAO,CAAC,gBAAgB,CAAC;CAuC7B;AAED,UAAU,qBAAqB;IAC7B,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,8BAA+B,YAAW,WAAW;IAChE,OAAO,CAAC,OAAO,CAAwB;gBAE3B,OAAO,GAAE,qBAA0B;IAI/C,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,cAAc,CAAC,EAAE,oBAAoB,GACpC,OAAO,CAAC,gBAAgB,CAAC;CA2C7B"}

package/dist/src/providers/promptfoo.js

@@ -14,8 +14,9 @@ const constants_2 = require("../redteam/constants");
 const shared_1 = require("./shared");
 class PromptfooHarmfulCompletionProvider {
     constructor(options) {
-        this.purpose = options.purpose;
         this.harmCategory = options.harmCategory;
+        this.n = options.n;
+        this.purpose = options.purpose;
     }
     id() {
         return `promptfoo:redteam:${this.harmCategory}`;
@@ -25,16 +26,15 @@ class PromptfooHarmfulCompletionProvider {
     }
     async callApi(prompt, context, callApiOptions) {
         const body = {
-            purpose: this.purpose,
-            harmCategory: this.harmCategory,
             email: (0, accounts_1.getUserEmail)(),
+            harmCategory: this.harmCategory,
+            n: this.n,
+            purpose: this.purpose,
             version: constants_1.VERSION,
         };
         try {
             logger_1.default.debug(`Calling promptfoo generate harmful API with body: ${JSON.stringify(body)}`);
             // We're using the promptfoo API to avoid having users provide their own unaligned model.
-            // See here for a prompt you can use with Llama 3 base to host your own inference endpoint:
-            // https://gist.github.com/typpo/3815d97a638f1a41d28634293aff33a0
             const response = await (0, fetch_1.fetchWithRetries)((0, envars_1.getEnvString)('PROMPTFOO_UNALIGNED_INFERENCE_ENDPOINT') ||
                 'https://api.promptfoo.dev/redteam/generateHarmful', {
                 method: 'POST',
@@ -42,14 +42,14 @@ class PromptfooHarmfulCompletionProvider {
                     'Content-Type': 'application/json',
                 },
                 body: JSON.stringify(body),
-            }, 10000);
+            }, 10000, 2);
             if (!response.ok) {
                 throw new Error(`API call failed with status ${response.status}: ${await response.text()}`);
             }
             const data = await response.json();
             logger_1.default.debug(`promptfoo API call response: ${JSON.stringify(data)}`);
             return {
-                output: data.output,
+                output: [data.output].flat(),
             };
         }
         catch (err) {

package/dist/src/providers/promptfoo.js.map

@@ -1 +1 @@
-{"version":3,"file":"promptfoo.js","sourceRoot":"","sources":["../../../src/providers/promptfoo.ts"],"names":[],"mappings":";;;;;;AAAA,oCAA0C;AAC1C,4CAAuC;AACvC,sCAAyC;AACzC,oCAA4C;AAC5C,uDAAwD;AACxD,uDAA+B;AAC/B,oDAA8D;AAS9D,qCAA8C;AAO9C,MAAa,kCAAkC;IAI7C,YAAY,OAA0C;QACpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,EAAE;QACA,OAAO,qBAAqB,IAAI,CAAC,YAAY,EAAE,CAAC;IAClD,CAAC;IAED,QAAQ;QACN,OAAO,0CAA0C,IAAI,CAAC,OAAO,MAAM,IAAI,CAAC,YAAY,GAAG,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,cAAqC;QAErC,MAAM,IAAI,GAAG;YACX,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,KAAK,EAAE,IAAA,uBAAY,GAAE;YACrB,OAAO,EAAE,mBAAO;SACjB,CAAC;QAEF,IAAI,CAAC;YACH,gBAAM,CAAC,KAAK,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1F,yFAAyF;YACzF,2FAA2F;YAC3F,iEAAiE;YACjE,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EACrC,IAAA,qBAAY,EAAC,wCAAwC,CAAC;gBACpD,mDAAmD,EACrD;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,KAAK,CACN,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,KAAK,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,gBAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA9DD,gFA8DC;AAUD,MAAa,+BAA+B;IAG1C,YAAY,OAAuC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,0BAA0B,CAAC;IACvD,CAAC;IAED,QAAQ;QACN,OAAO,sCAAsC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,cAAqC;QAErC,MAAM,IAAI,GAAG;YACX,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;YAC/B,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB;YAC/C,MAAM;YACN,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK;YAC3B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACvB,KAAK,EAAE,IAAA,uBAAY,GAAE;SACtB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,sBAAc,EACnC,IAAA,kCAAsB,GAAE,EACxB;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;YAEF,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;YAEpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAClD,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,UAAU;aACX,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA1DD,0EA0DC;AAQD,MAAa,8BAA8B;IAGzC,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,iBAAiB,CAAC;IAC9C,CAAC;IAED,QAAQ;QACN,OAAO,4BAA4B,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,cAAqC;QAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG;YACX,IAAI,EAAE,KAAK;YACX,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;YACvC,OAAO,EAAE,QAAQ;YACjB,KAAK,EAAE,IAAA,uBAAY,GAAE;YACrB,OAAO,EAAE,mBAAO;SACjB,CAAC;QAEF,gBAAM,CAAC,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/E,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EACrC,IAAA,kCAAsB,GAAE,EACxB;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,KAAK,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA9DD,wEA8DC"}
+{"version":3,"file":"promptfoo.js","sourceRoot":"","sources":["../../../src/providers/promptfoo.ts"],"names":[],"mappings":";;;;;;AAAA,oCAA0C;AAC1C,4CAAuC;AACvC,sCAAyC;AACzC,oCAA4C;AAC5C,uDAAwD;AACxD,uDAA+B;AAC/B,oDAA8D;AAS9D,qCAA8C;AAQ9C,MAAa,kCAAkC;IAK7C,YAAY,OAA0C;QACpD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACjC,CAAC;IAED,EAAE;QACA,OAAO,qBAAqB,IAAI,CAAC,YAAY,EAAE,CAAC;IAClD,CAAC;IAED,QAAQ;QACN,OAAO,0CAA0C,IAAI,CAAC,OAAO,MAAM,IAAI,CAAC,YAAY,GAAG,CAAC;IAC1F,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,cAAqC;QAErC,MAAM,IAAI,GAAG;YACX,KAAK,EAAE,IAAA,uBAAY,GAAE;YACrB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,mBAAO;SACjB,CAAC;QAEF,IAAI,CAAC;YACH,gBAAM,CAAC,KAAK,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1F,yFAAyF;YACzF,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EACrC,IAAA,qBAAY,EAAC,wCAAwC,CAAC;gBACpD,mDAAmD,EACrD;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,KAAK,EACL,CAAC,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,KAAK,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,gBAAM,CAAC,KAAK,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrE,OAAO;gBACL,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE;aAC7B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAhED,gFAgEC;AAUD,MAAa,+BAA+B;IAG1C,YAAY,OAAuC;QACjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,0BAA0B,CAAC;IACvD,CAAC;IAED,QAAQ;QACN,OAAO,sCAAsC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,cAAqC;QAErC,MAAM,IAAI,GAAG;YACX,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ;YAC/B,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB;YAC/C,MAAM;YACN,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK;YAC3B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACvB,KAAK,EAAE,IAAA,uBAAY,GAAE;SACtB,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAA,sBAAc,EACnC,IAAA,kCAAsB,GAAE,EACxB;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;YAEF,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;YAEpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;YAClD,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,MAAM;gBACd,UAAU;aACX,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA1DD,0EA0DC;AAQD,MAAa,8BAA8B;IAGzC,YAAY,UAAiC,EAAE;QAC7C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,iBAAiB,CAAC;IAC9C,CAAC;IAED,QAAQ;QACN,OAAO,4BAA4B,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,cAAqC;QAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG;YACX,IAAI,EAAE,KAAK;YACX,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;YACvC,OAAO,EAAE,QAAQ;YACjB,KAAK,EAAE,IAAA,uBAAY,GAAE;YACrB,OAAO,EAAE,mBAAO;SACjB,CAAC;QAEF,gBAAM,CAAC,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/E,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EACrC,IAAA,kCAAsB,GAAE,EACxB;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,KAAK,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC9F,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIlC,CAAC;YACF,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AA9DD,wEA8DC"}

package/dist/src/providers/simulatedUser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"simulatedUser.d.ts","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,WAAW,EACX,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,gBAAgB,EACjB,MAAM,UAAU,CAAC;AAIlB,MAAM,MAAM,sBAAsB,GAAG;IACnC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,eAAe,GAAG;IACnD,MAAM,CAAC,EAAE;QACP,YAAY,CAAC,EAAE,eAAe,CAAC;QAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH,CAAC;AAEF,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;gBAE7B,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,oBAAoB;IAMvD,EAAE;YAIY,iBAAiB;YAgBjB,kBAAkB;IAS1B,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,eAAe,CAAC,EAAE,oBAAoB,GACrC,OAAO,CAAC,gBAAgB,CAAC;IA8B5B,QAAQ;CAGT"}
+{"version":3,"file":"simulatedUser.d.ts","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,WAAW,EACX,oBAAoB,EACpB,oBAAoB,EACpB,eAAe,EACf,gBAAgB,EACjB,MAAM,UAAU,CAAC;AAIlB,MAAM,MAAM,sBAAsB,GAAG;IACnC,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG,eAAe,GAAG;IACnD,MAAM,CAAC,EAAE;QACP,YAAY,CAAC,EAAE,eAAe,CAAC;QAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;CACH,CAAC;AAEF,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;gBAE7B,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,oBAAoB;IAMvD,EAAE;YAIY,iBAAiB;YAgBjB,kBAAkB;IAS1B,OAAO,CACX,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,oBAAoB,EAC9B,eAAe,CAAC,EAAE,oBAAoB,GACrC,OAAO,CAAC,gBAAgB,CAAC;IAmC5B,QAAQ;CAGT"}

package/dist/src/providers/simulatedUser.js

@@ -42,9 +42,11 @@ class SimulatedUser {
         logger_1.default.debug(`Formatted user instructions: ${instructions}`);
         let messages = [];
         const maxTurns = this.maxTurns;
+        let numRequests = 0;
         for (let i = 0; i < maxTurns; i++) {
             messages = await this.sendMessageToUser(messages, userProvider);
             messages = await this.sendMessageToAgent(messages, context.originalProvider);
+            numRequests += 1; // Only count the request to the agent.
             const lastMessage = messages[messages.length - 1];
             if (lastMessage.content.includes('###STOP###')) {
                 break;
@@ -54,6 +56,9 @@ class SimulatedUser {
             output: messages
                 .map((message) => `${message.role === 'assistant' ? 'Assistant' : 'User'}: ${message.content}`)
                 .join('\n---\n'),
+            tokenUsage: {
+                numRequests,
+            },
         };
     }
     toString() {

package/dist/src/providers/simulatedUser.js.map

@@ -1 +1 @@
-{"version":3,"file":"simulatedUser.js","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,uDAA+B;AAQ/B,iDAAsD;AACtD,2CAA6D;AAmB7D,MAAa,aAAa;IAKxB,YAAY,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAwB;QACrD,IAAI,CAAC,UAAU,GAAG,EAAE,IAAI,KAAK,IAAI,gBAAgB,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,YAAY,IAAI,kBAAkB,CAAC;IACnE,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,QAAmB,EACnB,YAA4C;QAE5C,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,OAAO;gBACL,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM;gBACpD,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;QAC7E,gBAAM,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAmB,EACnB,cAA2B;QAE3B,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxE,gBAAM,CAAC,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,eAAsC;QAEtC,IAAA,wBAAS,EAAC,OAAO,EAAE,gBAAgB,EAAE,qCAAqC,CAAC,CAAC;QAE5E,MAAM,YAAY,GAAG,IAAA,6BAAiB,GAAE,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC3F,MAAM,YAAY,GAAG,IAAI,0CAA8B,CAAC;YACtD,YAAY;SACb,CAAC,CAAC;QAEH,gBAAM,CAAC,KAAK,CAAC,gCAAgC,YAAY,EAAE,CAAC,CAAC;QAC7D,IAAI,QAAQ,GAAc,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAChE,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAE7E,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAClD,IAAI,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/C,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,QAAQ;iBACb,GAAG,CACF,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE,CAC1F;iBACA,IAAI,CAAC,SAAS,CAAC;SACnB,CAAC;IACJ,CAAC;IAED,QAAQ;QACN,OAAO,eAAe,CAAC;IACzB,CAAC;CACF;AA7ED,sCA6EC"}
+{"version":3,"file":"simulatedUser.js","sourceRoot":"","sources":["../../../src/providers/simulatedUser.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,uDAA+B;AAQ/B,iDAAsD;AACtD,2CAA6D;AAmB7D,MAAa,aAAa;IAKxB,YAAY,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAwB;QACrD,IAAI,CAAC,UAAU,GAAG,EAAE,IAAI,KAAK,IAAI,gBAAgB,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,YAAY,IAAI,kBAAkB,CAAC;IACnE,CAAC;IAED,EAAE;QACA,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,QAAmB,EACnB,YAA4C;QAE5C,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/C,OAAO;gBACL,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM;gBACpD,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC;QAC7E,gBAAM,CAAC,KAAK,CAAC,SAAS,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAmB,EACnB,cAA2B;QAE3B,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxE,gBAAM,CAAC,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAc,EACd,OAA8B,EAC9B,eAAsC;QAEtC,IAAA,wBAAS,EAAC,OAAO,EAAE,gBAAgB,EAAE,qCAAqC,CAAC,CAAC;QAE5E,MAAM,YAAY,GAAG,IAAA,6BAAiB,GAAE,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;QAC3F,MAAM,YAAY,GAAG,IAAI,0CAA8B,CAAC;YACtD,YAAY;SACb,CAAC,CAAC;QAEH,gBAAM,CAAC,KAAK,CAAC,gCAAgC,YAAY,EAAE,CAAC,CAAC;QAC7D,IAAI,QAAQ,GAAc,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,QAAQ,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;YAChE,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAC7E,WAAW,IAAI,CAAC,CAAC,CAAC,uCAAuC;YAEzD,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAClD,IAAI,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/C,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,QAAQ;iBACb,GAAG,CACF,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,OAAO,EAAE,CAC1F;iBACA,IAAI,CAAC,SAAS,CAAC;YAClB,UAAU,EAAE;gBACV,WAAW;aACZ;SACF,CAAC;IACJ,CAAC;IAED,QAAQ;QACN,OAAO,eAAe,CAAC;IACzB,CAAC;CACF;AAlFD,sCAkFC"}

package/dist/src/redteam/graders.d.ts

@@ -1,80 +1,5 @@
-import { AsciiSmugglingGrader } from './plugins/asciiSmuggling';
 import type { RedteamGraderBase } from './plugins/base';
-import { BflaGrader } from './plugins/bfla';
-import { BolaGrader } from './plugins/bola';
-import { CompetitorsGrader } from './plugins/competitors';
-import { ContractsGrader } from './plugins/contracts';
-import { CrossSessionLeakGrader } from './plugins/crossSessionLeak';
-import { DebugAccessGrader } from './plugins/debugAccess';
-import { ExcessiveAgencyGrader } from './plugins/excessiveAgency';
-import { HallucinationGrader } from './plugins/hallucination';
-import { HarmfulGrader, HarmfulPrivacyGrader } from './plugins/harmful';
-import { HijackingGrader } from './plugins/hijacking';
-import { ImitationGrader } from './plugins/imitation';
-import { IndirectPromptInjectionGrader } from './plugins/indirectPromptInjection';
-import { IntentGrader } from './plugins/intent';
-import { OverrelianceGrader } from './plugins/overreliance';
-import { PiiGrader } from './plugins/pii';
-import { PolicyViolationGrader } from './plugins/policy';
-import { PoliticsGrader } from './plugins/politics';
-import { PromptExtractionGrader } from './plugins/promptExtraction';
-import { RbacGrader } from './plugins/rbac';
-import { ReligionGrader } from './plugins/religion';
-import { ShellInjectionGrader } from './plugins/shellInjection';
-import { SqlInjectionGrader } from './plugins/sqlInjection';
-import { SsrfGrader } from './plugins/ssrf';
-export declare const GRADERS: {
-    'promptfoo:redteam:ascii-smuggling': AsciiSmugglingGrader;
-    'promptfoo:redteam:bfla': BflaGrader;
-    'promptfoo:redteam:bola': BolaGrader;
-    'promptfoo:redteam:competitors': CompetitorsGrader;
-    'promptfoo:redteam:contracts': ContractsGrader;
-    'promptfoo:redteam:cross-session-leak': CrossSessionLeakGrader;
-    'promptfoo:redteam:debug-access': DebugAccessGrader;
-    'promptfoo:redteam:excessive-agency': ExcessiveAgencyGrader;
-    'promptfoo:redteam:hallucination': HallucinationGrader;
-    'promptfoo:redteam:harmful:chemical-biological-weapons': HarmfulGrader;
-    'promptfoo:redteam:harmful:child-exploitation': HarmfulGrader;
-    'promptfoo:redteam:harmful:copyright-violations': HarmfulGrader;
-    'promptfoo:redteam:harmful:cybercrime': HarmfulGrader;
-    'promptfoo:redteam:harmful:graphic-content': HarmfulGrader;
-    'promptfoo:redteam:harmful:harassment-bullying': HarmfulGrader;
-    'promptfoo:redteam:harmful:hate': HarmfulGrader;
-    'promptfoo:redteam:harmful:illegal-activities': HarmfulGrader;
-    'promptfoo:redteam:harmful:illegal-drugs': HarmfulGrader;
-    'promptfoo:redteam:harmful:indiscriminate-weapons': HarmfulGrader;
-    'promptfoo:redteam:harmful:insults': HarmfulGrader;
-    'promptfoo:redteam:harmful:intellectual-property': HarmfulGrader;
-    'promptfoo:redteam:harmful:misinformation-disinformation': HarmfulGrader;
-    'promptfoo:redteam:harmful:non-violent-crime': HarmfulGrader;
-    'promptfoo:redteam:harmful:privacy': HarmfulPrivacyGrader;
-    'promptfoo:redteam:harmful:profanity': HarmfulGrader;
-    'promptfoo:redteam:harmful:radicalization': HarmfulGrader;
-    'promptfoo:redteam:harmful:self-harm': HarmfulGrader;
-    'promptfoo:redteam:harmful:sex-crime': HarmfulGrader;
-    'promptfoo:redteam:harmful:sexual-content': HarmfulGrader;
-    'promptfoo:redteam:harmful:specialized-advice': HarmfulGrader;
-    'promptfoo:redteam:harmful:unsafe-practices': HarmfulGrader;
-    'promptfoo:redteam:harmful:violent-crime': HarmfulGrader;
-    'promptfoo:redteam:harmful': HarmfulGrader;
-    'promptfoo:redteam:hijacking': HijackingGrader;
-    'promptfoo:redteam:imitation': ImitationGrader;
-    'promptfoo:redteam:indirect-prompt-injection': IndirectPromptInjectionGrader;
-    'promptfoo:redteam:intent': IntentGrader;
-    'promptfoo:redteam:overreliance': OverrelianceGrader;
-    'promptfoo:redteam:pii:api-db': PiiGrader;
-    'promptfoo:redteam:pii:direct': PiiGrader;
-    'promptfoo:redteam:pii:session': PiiGrader;
-    'promptfoo:redteam:pii:social': PiiGrader;
-    'promptfoo:redteam:pii': PiiGrader;
-    'promptfoo:redteam:policy': PolicyViolationGrader;
-    'promptfoo:redteam:politics': PoliticsGrader;
-    'promptfoo:redteam:prompt-extraction': PromptExtractionGrader;
-    'promptfoo:redteam:rbac': RbacGrader;
-    'promptfoo:redteam:religion': ReligionGrader;
-    'promptfoo:redteam:shell-injection': ShellInjectionGrader;
-    'promptfoo:redteam:sql-injection': SqlInjectionGrader;
-    'promptfoo:redteam:ssrf': SsrfGrader;
-};
+import type { RedteamAssertionTypes } from './types';
+export declare const GRADERS: Record<RedteamAssertionTypes, RedteamGraderBase>;
 export declare function getGraderById(id: string): RedteamGraderBase | undefined;
 //# sourceMappingURL=graders.d.ts.map

package/dist/src/redteam/graders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../src/redteam/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoDnB,CAAC;AAEF,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAEvE"}
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../src/redteam/graders.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAwBxD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAErD,eAAO,MAAM,OAAO,EAAE,MAAM,CAAC,qBAAqB,EAAE,iBAAiB,CAoDpE,CAAC;AAEF,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAEvE"}

package/dist/src/redteam/graders.js

@@ -11,7 +11,7 @@ const crossSessionLeak_1 = require("./plugins/crossSessionLeak");
 const debugAccess_1 = require("./plugins/debugAccess");
 const excessiveAgency_1 = require("./plugins/excessiveAgency");
 const hallucination_1 = require("./plugins/hallucination");
-const harmful_1 = require("./plugins/harmful");
+const graders_1 = require("./plugins/harmful/graders");
 const hijacking_1 = require("./plugins/hijacking");
 const imitation_1 = require("./plugins/imitation");
 const indirectPromptInjection_1 = require("./plugins/indirectPromptInjection");
@@ -36,30 +36,30 @@ exports.GRADERS = {
     'promptfoo:redteam:debug-access': new debugAccess_1.DebugAccessGrader(),
     'promptfoo:redteam:excessive-agency': new excessiveAgency_1.ExcessiveAgencyGrader(),
     'promptfoo:redteam:hallucination': new hallucination_1.HallucinationGrader(),
-    'promptfoo:redteam:harmful:chemical-biological-weapons': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:child-exploitation': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:copyright-violations': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:cybercrime': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:graphic-content': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:harassment-bullying': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:hate': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:illegal-activities': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:illegal-drugs': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:indiscriminate-weapons': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:insults': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:intellectual-property': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:misinformation-disinformation': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:non-violent-crime': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:privacy': new harmful_1.HarmfulPrivacyGrader(),
-    'promptfoo:redteam:harmful:profanity': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:radicalization': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:self-harm': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:sex-crime': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:sexual-content': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:specialized-advice': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:unsafe-practices': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful:violent-crime': new harmful_1.HarmfulGrader(),
-    'promptfoo:redteam:harmful': new harmful_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:chemical-biological-weapons': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:child-exploitation': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:copyright-violations': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:cybercrime': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:graphic-content': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:harassment-bullying': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:hate': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:illegal-activities': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:illegal-drugs': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:indiscriminate-weapons': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:insults': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:intellectual-property': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:misinformation-disinformation': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:non-violent-crime': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:privacy': new graders_1.HarmfulPrivacyGrader(),
+    'promptfoo:redteam:harmful:profanity': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:radicalization': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:self-harm': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:sex-crime': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:sexual-content': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:specialized-advice': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:unsafe-practices': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful:violent-crime': new graders_1.HarmfulGrader(),
+    'promptfoo:redteam:harmful': new graders_1.HarmfulGrader(),
     'promptfoo:redteam:hijacking': new hijacking_1.HijackingGrader(),
     'promptfoo:redteam:imitation': new imitation_1.ImitationGrader(),
     'promptfoo:redteam:indirect-prompt-injection': new indirectPromptInjection_1.IndirectPromptInjectionGrader(),

package/dist/src/redteam/graders.js.map

@@ -1 +1 @@
-{"version":3,"file":"graders.js","sourceRoot":"","sources":["../../../src/redteam/graders.ts"],"names":[],"mappings":";;;AAgFA,sCAEC;AAlFD,6DAAgE;AAEhE,yCAA4C;AAC5C,yCAA4C;AAC5C,uDAA0D;AAC1D,mDAAsD;AACtD,iEAAoE;AACpE,uDAA0D;AAC1D,+DAAkE;AAClE,2DAA8D;AAC9D,+CAAwE;AACxE,mDAAsD;AACtD,mDAAsD;AACtD,+EAAkF;AAClF,6CAAgD;AAChD,yDAA4D;AAC5D,uCAA0C;AAC1C,6CAAyD;AACzD,iDAAoD;AACpD,iEAAoE;AACpE,yCAA4C;AAC5C,iDAAoD;AACpD,6DAAgE;AAChE,yDAA4D;AAC5D,yCAA4C;AAE/B,QAAA,OAAO,GAAG;IACrB,mCAAmC,EAAE,IAAI,qCAAoB,EAAE;IAC/D,wBAAwB,EAAE,IAAI,iBAAU,EAAE;IAC1C,wBAAwB,EAAE,IAAI,iBAAU,EAAE;IAC1C,+BAA+B,EAAE,IAAI,+BAAiB,EAAE;IACxD,6BAA6B,EAAE,IAAI,2BAAe,EAAE;IACpD,sCAAsC,EAAE,IAAI,yCAAsB,EAAE;IACpE,gCAAgC,EAAE,IAAI,+BAAiB,EAAE;IACzD,oCAAoC,EAAE,IAAI,uCAAqB,EAAE;IACjE,iCAAiC,EAAE,IAAI,mCAAmB,EAAE;IAC5D,uDAAuD,EAAE,IAAI,uBAAa,EAAE;IAC5E,8CAA8C,EAAE,IAAI,uBAAa,EAAE;IACnE,gDAAgD,EAAE,IAAI,uBAAa,EAAE;IACrE,sCAAsC,EAAE,IAAI,uBAAa,EAAE;IAC3D,2CAA2C,EAAE,IAAI,uBAAa,EAAE;IAChE,+CAA+C,EAAE,IAAI,uBAAa,EAAE;IACpE,gCAAgC,EAAE,IAAI,uBAAa,EAAE;IACrD,8CAA8C,EAAE,IAAI,uBAAa,EAAE;IACnE,yCAAyC,EAAE,IAAI,uBAAa,EAAE;IAC9D,kDAAkD,EAAE,IAAI,uBAAa,EAAE;IACvE,mCAAmC,EAAE,IAAI,uBAAa,EAAE;IACxD,iDAAiD,EAAE,IAAI,uBAAa,EAAE;IACtE,yDAAyD,EAAE,IAAI,uBAAa,EAAE;IAC9E,6CAA6C,EAAE,IAAI,uBAAa,EAAE;IAClE,mCAAmC,EAAE,IAAI,8BAAoB,EAAE;IAC/D,qCAAqC,EAAE,IAAI,uBAAa,EAAE;IAC1D,0CAA0C,EAAE,IAAI,uBAAa,EAAE;IAC/D,qCAAqC,EAAE,IAAI,uBAAa,EAAE;IAC1D,qCAAqC,EAAE,IAAI,uBAAa,EAAE;IAC1D,0CAA0C,EAAE,IAAI,uBAAa,EAAE;IAC/D,8CAA8C,EAAE,IAAI,uBAAa,EAAE;IACnE,4CAA4C,EAAE,IAAI,uBAAa,EAAE;IACjE,yCAAyC,EAAE,IAAI,uBAAa,EAAE;IAC9D,2BAA2B,EAAE,IAAI,uBAAa,EAAE;IAChD,6BAA6B,EAAE,IAAI,2BAAe,EAAE;IACpD,6BAA6B,EAAE,IAAI,2BAAe,EAAE;IACpD,6CAA6C,EAAE,IAAI,uDAA6B,EAAE;IAClF,0BAA0B,EAAE,IAAI,qBAAY,EAAE;IAC9C,gCAAgC,EAAE,IAAI,iCAAkB,EAAE;IAC1D,8BAA8B,EAAE,IAAI,eAAS,EAAE;IAC/C,8BAA8B,EAAE,IAAI,eAAS,EAAE;IAC/C,+BAA+B,EAAE,IAAI,eAAS,EAAE;IAChD,8BAA8B,EAAE,IAAI,eAAS,EAAE;IAC/C,uBAAuB,EAAE,IAAI,eAAS,EAAE;IACxC,0BAA0B,EAAE,IAAI,8BAAqB,EAAE;IACvD,4BAA4B,EAAE,IAAI,yBAAc,EAAE;IAClD,qCAAqC,EAAE,IAAI,yCAAsB,EAAE;IACnE,wBAAwB,EAAE,IAAI,iBAAU,EAAE;IAC1C,4BAA4B,EAAE,IAAI,yBAAc,EAAE;IAClD,mCAAmC,EAAE,IAAI,qCAAoB,EAAE;IAC/D,iCAAiC,EAAE,IAAI,iCAAkB,EAAE;IAC3D,wBAAwB,EAAE,IAAI,iBAAU,EAAE;CAC3C,CAAC;AAEF,SAAgB,aAAa,CAAC,EAAU;IACtC,OAAO,eAAO,CAAC,EAA0B,CAAC,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"graders.js","sourceRoot":"","sources":["../../../src/redteam/graders.ts"],"names":[],"mappings":";;;AAiFA,sCAEC;AAnFD,6DAAgE;AAEhE,yCAA4C;AAC5C,yCAA4C;AAC5C,uDAA0D;AAC1D,mDAAsD;AACtD,iEAAoE;AACpE,uDAA0D;AAC1D,+DAAkE;AAClE,2DAA8D;AAC9D,uDAAgF;AAChF,mDAAsD;AACtD,mDAAsD;AACtD,+EAAkF;AAClF,6CAAgD;AAChD,yDAA4D;AAC5D,uCAA0C;AAC1C,6CAAyD;AACzD,iDAAoD;AACpD,iEAAoE;AACpE,yCAA4C;AAC5C,iDAAoD;AACpD,6DAAgE;AAChE,yDAA4D;AAC5D,yCAA4C;AAG/B,QAAA,OAAO,GAAqD;IACvE,mCAAmC,EAAE,IAAI,qCAAoB,EAAE;IAC/D,wBAAwB,EAAE,IAAI,iBAAU,EAAE;IAC1C,wBAAwB,EAAE,IAAI,iBAAU,EAAE;IAC1C,+BAA+B,EAAE,IAAI,+BAAiB,EAAE;IACxD,6BAA6B,EAAE,IAAI,2BAAe,EAAE;IACpD,sCAAsC,EAAE,IAAI,yCAAsB,EAAE;IACpE,gCAAgC,EAAE,IAAI,+BAAiB,EAAE;IACzD,oCAAoC,EAAE,IAAI,uCAAqB,EAAE;IACjE,iCAAiC,EAAE,IAAI,mCAAmB,EAAE;IAC5D,uDAAuD,EAAE,IAAI,uBAAa,EAAE;IAC5E,8CAA8C,EAAE,IAAI,uBAAa,EAAE;IACnE,gDAAgD,EAAE,IAAI,uBAAa,EAAE;IACrE,sCAAsC,EAAE,IAAI,uBAAa,EAAE;IAC3D,2CAA2C,EAAE,IAAI,uBAAa,EAAE;IAChE,+CAA+C,EAAE,IAAI,uBAAa,EAAE;IACpE,gCAAgC,EAAE,IAAI,uBAAa,EAAE;IACrD,8CAA8C,EAAE,IAAI,uBAAa,EAAE;IACnE,yCAAyC,EAAE,IAAI,uBAAa,EAAE;IAC9D,kDAAkD,EAAE,IAAI,uBAAa,EAAE;IACvE,mCAAmC,EAAE,IAAI,uBAAa,EAAE;IACxD,iDAAiD,EAAE,IAAI,uBAAa,EAAE;IACtE,yDAAyD,EAAE,IAAI,uBAAa,EAAE;IAC9E,6CAA6C,EAAE,IAAI,uBAAa,EAAE;IAClE,mCAAmC,EAAE,IAAI,8BAAoB,EAAE;IAC/D,qCAAqC,EAAE,IAAI,uBAAa,EAAE;IAC1D,0CAA0C,EAAE,IAAI,uBAAa,EAAE;IAC/D,qCAAqC,EAAE,IAAI,uBAAa,EAAE;IAC1D,qCAAqC,EAAE,IAAI,uBAAa,EAAE;IAC1D,0CAA0C,EAAE,IAAI,uBAAa,EAAE;IAC/D,8CAA8C,EAAE,IAAI,uBAAa,EAAE;IACnE,4CAA4C,EAAE,IAAI,uBAAa,EAAE;IACjE,yCAAyC,EAAE,IAAI,uBAAa,EAAE;IAC9D,2BAA2B,EAAE,IAAI,uBAAa,EAAE;IAChD,6BAA6B,EAAE,IAAI,2BAAe,EAAE;IACpD,6BAA6B,EAAE,IAAI,2BAAe,EAAE;IACpD,6CAA6C,EAAE,IAAI,uDAA6B,EAAE;IAClF,0BAA0B,EAAE,IAAI,qBAAY,EAAE;IAC9C,gCAAgC,EAAE,IAAI,iCAAkB,EAAE;IAC1D,8BAA8B,EAAE,IAAI,eAAS,EAAE;IAC/C,8BAA8B,EAAE,IAAI,eAAS,EAAE;IAC/C,+BAA+B,EAAE,IAAI,eAAS,EAAE;IAChD,8BAA8B,EAAE,IAAI,eAAS,EAAE;IAC/C,uBAAuB,EAAE,IAAI,eAAS,EAAE;IACxC,0BAA0B,EAAE,IAAI,8BAAqB,EAAE;IACvD,4BAA4B,EAAE,IAAI,yBAAc,EAAE;IAClD,qCAAqC,EAAE,IAAI,yCAAsB,EAAE;IACnE,wBAAwB,EAAE,IAAI,iBAAU,EAAE;IAC1C,4BAA4B,EAAE,IAAI,yBAAc,EAAE;IAClD,mCAAmC,EAAE,IAAI,qCAAoB,EAAE;IAC/D,iCAAiC,EAAE,IAAI,iCAAkB,EAAE;IAC3D,wBAAwB,EAAE,IAAI,iBAAU,EAAE;CAC3C,CAAC;AAEF,SAAgB,aAAa,CAAC,EAAU;IACtC,OAAO,eAAO,CAAC,EAA0B,CAAC,CAAC;AAC7C,CAAC"}

package/dist/src/redteam/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/redteam/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AASnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAyDjD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAwBhG;AAeD;;;;GAIG;AACH,wBAAsB,UAAU,CAAC,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EACT,QAAQ,EACR,cAAkB,EAClB,OAAO,EACP,OAAO,EACP,QAAQ,EACR,OAAO,EAAE,eAAe,EACxB,UAAU,EACV,KAAK,GACN,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE,kBAAkB,EAAE,CAAC;CACjC,CAAC,CA8PD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/redteam/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAY,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAS7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAyDjD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAwBhG;AAeD;;;;GAIG;AACH,wBAAsB,UAAU,CAAC,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EACT,QAAQ,EACR,cAAkB,EAClB,OAAO,EACP,OAAO,EACP,QAAQ,EACR,OAAO,EAAE,eAAe,EACxB,UAAU,EACV,KAAK,GACN,EAAE,iBAAiB,GAAG,OAAO,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,EAAE,kBAAkB,EAAE,CAAC;CACjC,CAAC,CA4QD"}

package/dist/src/redteam/index.js

@@ -232,9 +232,16 @@ async function synthesize({ entities: entitiesOverride, injectVar, language, max
         const { action } = plugins_1.Plugins.find((p) => p.key === plugin.id) || {};
         if (action) {
             logger_1.default.debug(`Generating tests for ${plugin.id}...`);
-            let pluginTests = await action(redteamProvider, purpose, injectVar, plugin.numTests, delay || 0, {
-                language,
-                ...resolvePluginConfig(plugin.config),
+            let pluginTests = await action({
+                provider: redteamProvider,
+                purpose,
+                injectVar,
+                n: plugin.numTests,
+                delayMs: delay || 0,
+                config: {
+                    language,
+                    ...resolvePluginConfig(plugin.config),
+                },
             });
             if (!Array.isArray(pluginTests) || pluginTests.length === 0) {
                 logger_1.default.warn(`Failed to generate tests for ${plugin.id}`);
@@ -245,6 +252,7 @@ async function synthesize({ entities: entitiesOverride, injectVar, language, max
                 metadata: {
                     ...(t.metadata || {}),
                     pluginId: plugin.id,
+                    pluginConfig: resolvePluginConfig(plugin.config),
                 },
             })));
             progressBar?.increment(plugin.numTests);
@@ -263,6 +271,7 @@ async function synthesize({ entities: entitiesOverride, injectVar, language, max
                     metadata: {
                         ...(t.metadata || {}),
                         pluginId: plugin.id,
+                        pluginConfig: resolvePluginConfig(plugin.config),
                     },
                 })));
                 logger_1.default.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
@@ -296,14 +305,23 @@ async function synthesize({ entities: entitiesOverride, injectVar, language, max
             progressBar?.update({ task: `Applying strategy: ${key}` });
             logger_1.default.debug(`Generating ${key} tests`);
             const strategyTestCases = await action(testCases, injectVar, strategy.config || {});
-            newTestCases.push(...strategyTestCases.map((t) => ({
-                ...t,
-                metadata: {
-                    ...(t.metadata || {}),
-                    pluginId: t.metadata?.pluginId,
-                    strategyId: strategy.id,
-                },
-            })));
+            try {
+                newTestCases.push(...strategyTestCases
+                    .filter((t) => t !== null && t !== undefined)
+                    .map((t) => ({
+                    ...t,
+                    metadata: {
+                        ...(t?.metadata || {}),
+                        strategyId: strategy.id,
+                        ...(t?.metadata?.pluginId && { pluginId: t.metadata.pluginId }),
+                        ...(t?.metadata?.pluginConfig && { pluginConfig: t.metadata.pluginConfig }),
+                        ...(strategy.config && { strategyConfig: strategy.config }),
+                    },
+                })));
+            }
+            catch (e) {
+                logger_1.default.warn(`Strategy ${key} did not return valid test cases: ${e}`);
+            }
             strategyResults[key] = {
                 requested: testCases.length,
                 generated: strategyTestCases.length,

package/dist/src/redteam/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/redteam/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+EA,kDAwBC;AAoBD,gCA6QC;AAxYD,kDAA0B;AAC1B,kDAA0B;AAC1B,gEAAuC;AACvC,4DAA+B;AAC/B,uCAAyB;AACzB,sDAA2B;AAC3B,oEAAuC;AACvC,oDAAgD;AAEhD,iDAAkE;AAClE,2CAAiF;AACjF,oDAAwD;AACxD,kDAA4D;AAC5D,uCAAoC;AACpC,6CAAgD;AAChD,+CAAyD;AACzD,6CAA8D;AAG9D;;;;;GAKG;AACH,SAAS,SAAS,CAAC,SAAiB,EAAE,SAAiB;IACrD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,eAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IACD,IAAI,SAAS,GAAG,SAAS,EAAE,CAAC;QAC1B,OAAO,eAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAChC,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CACrB,aAAuE,EACvE,eAAyE;IAEzE,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC;QAC7D,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;KACnC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;QAC1C,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAChG,CAAC,CAAC,CAAC;IAEL,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;SAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;QAC1C,KAAK,CAAC,IAAI,CAAC;YACT,QAAQ,EAAE;YACV,UAAU;YACV,EAAE;YACF,SAAS;YACT,SAAS;YACT,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEL,OAAO,8BAA8B,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,SAAgB,mBAAmB,CAAC,MAAuC;IACzE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,GAAG;IACjB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,wBAAY,CAAC;IAClC,GAAG,EAAE,uBAAW;CACR,CAAC;AAEX;;;;GAIG;AACH,MAAM,eAAe,GAAG,CAAC,QAAgB,EAAU,EAAE,CACnD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,QAAQ,CAAC;AAElD;;;;GAIG;AACI,KAAK,UAAU,UAAU,CAAC,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EACT,QAAQ,EACR,cAAc,GAAG,CAAC,EAClB,OAAO,EACP,OAAO,EACP,QAAQ,EACR,OAAO,EAAE,eAAe,EACxB,UAAU,EACV,KAAK,GACa;IAKlB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QAChC,cAAc,GAAG,CAAC,CAAC;QACnB,gBAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC;IACD,IAAA,+BAAkB,EAAC,UAAU,CAAC,CAAC;IAE/B,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAmB,EAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEhE,gBAAM,CAAC,IAAI,CACT,+BAA+B,OAAO,CAAC,MAAM,IAC3C,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SACpC,0BAA0B,eAAK,CAAC,MAAM,CACpC,OAAO;SACJ,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,GAAG,CAAC,CAAC,EAAE,KAAK,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAChG;SACA,IAAI,EAAE;SACN,IAAI,CAAC,IAAI,CAAC,CACd,IAAI,CACN,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,gBAAM,CAAC,IAAI,CACT,wBAAwB,eAAK,CAAC,MAAM,CAClC,UAAU;aACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC;aAC5D,IAAI,EAAE;aACN,IAAI,CAAC,IAAI,CAAC,CACd,IAAI,CACN,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GACd,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEnF,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhF,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QACpC,oBAAoB,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;QAC5C,qBAAqB,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE;QACnD,gBAAgB,eAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QAC5C,mBAAmB,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;QAClD,wBAAwB,eAAK,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI;QACtD,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,eAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CACnD,CAAC;IAEF,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,IAAA,yCAA6B,EAAC,OAAO,CAAC,CAAC;QAC1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,gBAAM,CAAC,IAAI,CACT,0CAA0C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,6CAA6C,CACvK,CAAC;QACJ,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,gBAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;QACtF,CAAC;QACD,+GAA+G;QAC/G,SAAS,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QACzD,IAAA,wBAAS,EAAC,OAAO,SAAS,KAAK,QAAQ,EAAE,oCAAoC,SAAS,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,IAAI,WAAW,GAAiC,IAAI,CAAC;IACrD,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,OAAO,IAAI,IAAA,oBAAW,GAAE,KAAK,OAAO,EAAE,CAAC;QACnE,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CACrC;YACE,MAAM,EAAE,+DAA+D;SACxE,EACD,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAmB,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,CACnB,MAA2B,EAC3B,OAAoD,EACpD,EAAE;QACF,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAS,EAAE,EAAE,CACpC,eAAe,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAC3D,CAAC;QACF,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC;IAEF,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QACzB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,mCAAuB,CAAC,CAAC,IAAI,CAC1D,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CAC9D,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,OAAO,GACX,mCAAuB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,MAAM,CAAC,mCAAuB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,UAAU,GAAG,CAAC,CACvC,CAAC;YACJ,IAAI,OAAO,EAAE,CAAC;gBACZ,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,eAAe,CAAC;IAE1B,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9F,WAAW,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,eAAe,IAAI,CAAC,MAAM,IAAA,8BAAoB,EAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC;IAE1F,WAAW,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAa,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC;QACxD,CAAC,CAAC,gBAAgB;QAClB,CAAC,CAAC,MAAM,IAAA,0BAAe,EAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IAEpD,gBAAM,CAAC,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC;IAE3C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,EAAE,QAAQ,EAAE,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACpE,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC;gBACP,QAAQ;gBACR,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC;aACtC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAA6D,EAAE,CAAC;IACnF,MAAM,SAAS,GAAyB,EAAE,CAAC;IAC3C,MAAM,eAAK,CAAC,YAAY,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACjE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,MAAM,EAAE,CAAC;YACX,gBAAM,CAAC,KAAK,CAAC,wBAAwB,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;YACrD,IAAI,WAAW,GAAG,MAAM,MAAM,CAC5B,eAAe,EACf,OAAO,EACP,SAAS,EACT,MAAM,CAAC,QAAQ,EACf,KAAK,IAAI,CAAC,EACV;gBACE,QAAQ;gBACR,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC;aACtC,CACF,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5D,gBAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzD,WAAW,GAAG,EAAE,CAAC;YACnB,CAAC;YACD,SAAS,CAAC,IAAI,CACZ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,GAAG,CAAC;gBACJ,QAAQ,EAAE;oBACR,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACrB,QAAQ,EAAE,MAAM,CAAC,EAAE;iBACpB;aACF,CAAC,CAAC,CACJ,CAAC;YACF,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACxC,gBAAM,CAAC,KAAK,CAAC,SAAS,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC;YACpE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG;gBACzB,SAAS,EAAE,MAAM,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ;gBACxE,SAAS,EAAE,WAAW,CAAC,MAAM;aAC9B,CAAC;QACJ,CAAC;aAAM,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,IAAI,qBAAY,CAAC,eAAe,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBACtF,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAC7E,SAAS,CAAC,IAAI,CACZ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACzB,GAAG,CAAC;oBACJ,QAAQ,EAAE;wBACR,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;wBACrB,QAAQ,EAAE,MAAM,CAAC,EAAE;qBACpB;iBACF,CAAC,CAAC,CACJ,CAAC;gBACF,gBAAM,CAAC,KAAK,CAAC,SAAS,WAAW,CAAC,MAAM,2BAA2B,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;gBAChF,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC;YAC3F,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,gBAAM,CAAC,KAAK,CAAC,4CAA4C,MAAM,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC5E,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,2BAA2B,CAAC,CAAC;YAC5D,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YACxE,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAyB,EAAE,CAAC;IAE9C,MAAM,eAAe,GAA6D,EAAE,CAAC;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3C,MAAM,kBAAkB,GAAG,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjE,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,IAAI,CACR,uCAAuC,UAAU,CAAC,MAAM,WAAW,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAC5G;YACC,uBAAuB,eAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACtD,2BAA2B,eAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;YAC3D,6BAA6B,eAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,kBAAkB,CAAC,EAAE,CACpF,CAAC;QAEF,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,uBAAU,EAAE,CAAC;YACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;YACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YACD,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,sBAAsB,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3D,gBAAM,CAAC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,CAAC;YACxC,MAAM,iBAAiB,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YACpF,YAAY,CAAC,IAAI,CACf,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,CAAC;gBACJ,QAAQ,EAAE;oBACR,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACrB,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,QAAQ;oBAC9B,UAAU,EAAE,QAAQ,CAAC,EAAE;iBACxB;aACF,CAAC,CAAC,CACJ,CAAC;YACF,eAAe,CAAC,GAAG,CAAC,GAAG;gBACrB,SAAS,EAAE,SAAS,CAAC,MAAM;gBAC3B,SAAS,EAAE,iBAAiB,CAAC,MAAM;aACpC,CAAC;QACJ,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAClC,CAAC;IAED,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACvC,WAAW,EAAE,IAAI,EAAE,CAAC;IACpB,IAAI,WAAW,EAAE,CAAC;QAChB,8CAA8C;QAC9C,gBAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,gBAAM,CAAC,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC;IAE5D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AAC1C,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/redteam/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+EA,kDAwBC;AAoBD,gCA2RC;AAtZD,kDAA0B;AAC1B,kDAA0B;AAC1B,gEAAuC;AACvC,4DAA+B;AAC/B,uCAAyB;AACzB,sDAA2B;AAC3B,oEAAuC;AACvC,oDAAgD;AAEhD,iDAAkE;AAClE,2CAAiF;AACjF,oDAAwD;AACxD,kDAA4D;AAC5D,uCAAoC;AACpC,6CAAgD;AAChD,+CAAyD;AACzD,6CAA8D;AAG9D;;;;;GAKG;AACH,SAAS,SAAS,CAAC,SAAiB,EAAE,SAAiB;IACrD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,eAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IACD,IAAI,SAAS,GAAG,SAAS,EAAE,CAAC;QAC1B,OAAO,eAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;AAChC,CAAC;AAED;;;;;GAKG;AACH,SAAS,cAAc,CACrB,aAAuE,EACvE,eAAyE;IAEzE,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,CAAC;QAC7D,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;KACnC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;QAC1C,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IAChG,CAAC,CAAC,CAAC;IAEL,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC;SAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;QAC1C,KAAK,CAAC,IAAI,CAAC;YACT,QAAQ,EAAE;YACV,UAAU;YACV,EAAE;YACF,SAAS;YACT,SAAS;YACT,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;SAChC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEL,OAAO,8BAA8B,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC1D,CAAC;AAED;;;;GAIG;AACH,SAAgB,mBAAmB,CAAC,MAAuC;IACzE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,GAAG;IACjB,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,wBAAY,CAAC;IAClC,GAAG,EAAE,uBAAW;CACR,CAAC;AAEX;;;;GAIG;AACH,MAAM,eAAe,GAAG,CAAC,QAAgB,EAAU,EAAE,CACnD,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,QAAQ,CAAC;AAElD;;;;GAIG;AACI,KAAK,UAAU,UAAU,CAAC,EAC/B,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EACT,QAAQ,EACR,cAAc,GAAG,CAAC,EAClB,OAAO,EACP,OAAO,EACP,QAAQ,EACR,OAAO,EAAE,eAAe,EACxB,UAAU,EACV,KAAK,GACa;IAKlB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,KAAK,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QAChC,cAAc,GAAG,CAAC,CAAC;QACnB,gBAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;IACjE,CAAC;IACD,IAAA,+BAAkB,EAAC,UAAU,CAAC,CAAC;IAE/B,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAmB,EAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IAEhE,gBAAM,CAAC,IAAI,CACT,+BAA+B,OAAO,CAAC,MAAM,IAC3C,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SACpC,0BAA0B,eAAK,CAAC,MAAM,CACpC,OAAO;SACJ,GAAG,CACF,CAAC,CAAC,EAAE,EAAE,CACJ,GAAG,CAAC,CAAC,EAAE,KAAK,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAChG;SACA,IAAI,EAAE;SACN,IAAI,CAAC,IAAI,CAAC,CACd,IAAI,CACN,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,gBAAM,CAAC,IAAI,CACT,wBAAwB,eAAK,CAAC,MAAM,CAClC,UAAU;aACP,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,eAAe,CAAC,gBAAgB,CAAC,GAAG,CAAC;aAC5D,IAAI,EAAE;aACN,IAAI,CAAC,IAAI,CAAC,CACd,IAAI,CACN,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GACd,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEnF,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhF,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC;QACpC,oBAAoB,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;QAC5C,qBAAqB,eAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE;QACnD,gBAAgB,eAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QAC5C,mBAAmB,eAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;QAClD,wBAAwB,eAAK,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI;QACtD,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,eAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CACnD,CAAC;IAEF,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,IAAA,yCAA6B,EAAC,OAAO,CAAC,CAAC;QAC1D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,gBAAM,CAAC,IAAI,CACT,0CAA0C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,6CAA6C,CACvK,CAAC;QACJ,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,gBAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;QACtF,CAAC;QACD,+GAA+G;QAC/G,SAAS,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QACzD,IAAA,wBAAS,EAAC,OAAO,SAAS,KAAK,QAAQ,EAAE,oCAAoC,SAAS,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,IAAI,WAAW,GAAiC,IAAI,CAAC;IACrD,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,OAAO,IAAI,IAAA,oBAAW,GAAE,KAAK,OAAO,EAAE,CAAC;QACnE,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CACrC;YACE,MAAM,EAAE,+DAA+D;SACxE,EACD,sBAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;QACF,WAAW,CAAC,KAAK,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;QACtF,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAmB,EAAE,CAAC;IAC3C,MAAM,YAAY,GAAG,CACnB,MAA2B,EAC3B,OAAoD,EACpD,EAAE;QACF,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAS,EAAE,EAAE,CACpC,eAAe,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAC3D,CAAC;QACF,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC;IAEF,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QACzB,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,mCAAuB,CAAC,CAAC,IAAI,CAC1D,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,CAC9D,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,OAAO,GACX,mCAAuB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,MAAM,CAAC,mCAAuB,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,UAAU,GAAG,CAAC,CACvC,CAAC;YACJ,IAAI,OAAO,EAAE,CAAC;gBACZ,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,eAAe,CAAC;IAE1B,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9F,WAAW,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,eAAe,IAAI,CAAC,MAAM,IAAA,8BAAoB,EAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC;IAE1F,WAAW,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAa,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC;QACxD,CAAC,CAAC,gBAAgB;QAClB,CAAC,CAAC,MAAM,IAAA,0BAAe,EAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IAEpD,gBAAM,CAAC,KAAK,CAAC,mBAAmB,OAAO,EAAE,CAAC,CAAC;IAE3C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,EAAE,QAAQ,EAAE,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QACpE,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC;gBACP,QAAQ;gBACR,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC;aACtC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAA6D,EAAE,CAAC;IACnF,MAAM,SAAS,GAAyB,EAAE,CAAC;IAC3C,MAAM,eAAK,CAAC,YAAY,CAAC,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACjE,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,MAAM,EAAE,CAAC;YACX,gBAAM,CAAC,KAAK,CAAC,wBAAwB,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;YACrD,IAAI,WAAW,GAAG,MAAM,MAAM,CAAC;gBAC7B,QAAQ,EAAE,eAAe;gBACzB,OAAO;gBACP,SAAS;gBACT,CAAC,EAAE,MAAM,CAAC,QAAQ;gBAClB,OAAO,EAAE,KAAK,IAAI,CAAC;gBACnB,MAAM,EAAE;oBACN,QAAQ;oBACR,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC;iBACtC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5D,gBAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzD,WAAW,GAAG,EAAE,CAAC;YACnB,CAAC;YACD,SAAS,CAAC,IAAI,CACZ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzB,GAAG,CAAC;gBACJ,QAAQ,EAAE;oBACR,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACrB,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,YAAY,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC;iBACjD;aACF,CAAC,CAAC,CACJ,CAAC;YACF,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACxC,gBAAM,CAAC,KAAK,CAAC,SAAS,WAAW,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,aAAa,CAAC,CAAC;YACpE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG;gBACzB,SAAS,EAAE,MAAM,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ;gBACxE,SAAS,EAAE,WAAW,CAAC,MAAM;aAC9B,CAAC;QACJ,CAAC;aAAM,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,IAAI,qBAAY,CAAC,eAAe,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBACtF,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAC7E,SAAS,CAAC,IAAI,CACZ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACzB,GAAG,CAAC;oBACJ,QAAQ,EAAE;wBACR,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;wBACrB,QAAQ,EAAE,MAAM,CAAC,EAAE;wBACnB,YAAY,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC;qBACjD;iBACF,CAAC,CAAC,CACJ,CAAC;gBACF,gBAAM,CAAC,KAAK,CAAC,SAAS,WAAW,CAAC,MAAM,2BAA2B,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;gBAChF,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC;YAC3F,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,gBAAM,CAAC,KAAK,CAAC,4CAA4C,MAAM,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC5E,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YAC1E,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gBAAM,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,EAAE,2BAA2B,CAAC,CAAC;YAC5D,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YACxE,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAyB,EAAE,CAAC;IAE9C,MAAM,eAAe,GAA6D,EAAE,CAAC;IACrF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3C,MAAM,kBAAkB,GAAG,iBAAiB,GAAG,UAAU,CAAC,MAAM,CAAC;QAEjE,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,IAAI,CACR,uCAAuC,UAAU,CAAC,MAAM,WAAW,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,CAC5G;YACC,uBAAuB,eAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;YACtD,2BAA2B,eAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;YAC3D,6BAA6B,eAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,kBAAkB,CAAC,EAAE,CACpF,CAAC;QAEF,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,uBAAU,EAAE,CAAC;YACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;YACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YACD,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,sBAAsB,GAAG,EAAE,EAAE,CAAC,CAAC;YAC3D,gBAAM,CAAC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,CAAC;YACxC,MAAM,iBAAiB,GAAe,MAAM,MAAM,CAChD,SAAS,EACT,SAAS,EACT,QAAQ,CAAC,MAAM,IAAI,EAAE,CACtB,CAAC;YACF,IAAI,CAAC;gBACH,YAAY,CAAC,IAAI,CACf,GAAG,iBAAiB;qBACjB,MAAM,CAAC,CAAC,CAAC,EAA8B,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,CAAC;qBACxE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACX,GAAG,CAAC;oBACJ,QAAQ,EAAE;wBACR,GAAG,CAAC,CAAC,EAAE,QAAQ,IAAI,EAAE,CAAC;wBACtB,UAAU,EAAE,QAAQ,CAAC,EAAE;wBACvB,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;wBAC/D,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,YAAY,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;wBAC3E,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,cAAc,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;qBAC5D;iBACF,CAAC,CAAC,CACN,CAAC;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,gBAAM,CAAC,IAAI,CAAC,YAAY,GAAG,qCAAqC,CAAC,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,eAAe,CAAC,GAAG,CAAC,GAAG;gBACrB,SAAS,EAAE,SAAS,CAAC,MAAM;gBAC3B,SAAS,EAAE,iBAAiB,CAAC,MAAM;aACpC,CAAC;QACJ,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAClC,CAAC;IAED,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACvC,WAAW,EAAE,IAAI,EAAE,CAAC;IACpB,IAAI,WAAW,EAAE,CAAC;QAChB,8CAA8C;QAC9C,gBAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,gBAAM,CAAC,IAAI,CAAC,cAAc,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC;IAE5D,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AAC1C,CAAC"}

package/dist/src/redteam/plugins/base.d.ts

@@ -1,5 +1,14 @@
 import type { ApiProvider, Assertion, AssertionValue, PluginConfig, ResultSuggestion, TestCase } from '../../types';
 import type { AtomicTestCase, GradingResult } from '../../types';
+/**
+ * Parses the LLM response of generated prompts into an array of objects.
+ *
+ * @param generatedPrompts - The LLM response of generated prompts.
+ * @returns An array of { prompt: string } objects. Each of these objects represents a test case.
+ */
+export declare function parseGeneratedPrompts(generatedPrompts: string): {
+    prompt: string;
+}[];
 /**
  * Abstract base class for creating plugins that generate test cases.
  */
@@ -42,14 +51,10 @@ export declare abstract class RedteamPluginBase {
         prompt: string;
     }[]): TestCase[];
     /**
-     * Parses the LLM response of generated prompts into an array of objects.
-     *
-     * @param generatedPrompts - The LLM response of generated prompts.
-     * @returns An array of { prompt: string } objects. Each of these objects represents a test case.
+     * Appends modifiers to the template.
+     * @param template - The template to append modifiers to.
+     * @returns The modified template.
      */
-    protected parseGeneratedPrompts(generatedPrompts: string): {
-        prompt: string;
-    }[];
     private appendModifiers;
 }
 /**

package/dist/src/redteam/plugins/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/base.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,QAAQ,EACT,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAQjE;;GAEG;AACH,8BAAsB,iBAAiB;IASnC,SAAS,CAAC,QAAQ,EAAE,WAAW;IAC/B,SAAS,CAAC,OAAO,EAAE,MAAM;IACzB,SAAS,CAAC,SAAS,EAAE,MAAM;IAC3B,SAAS,CAAC,MAAM,EAAE,YAAY;IAXhC;;;;;;OAMG;gBAES,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,YAAiB;IAKrC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAEjD;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;IAE7D;;;;;OAKG;IACG,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE,MAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAkDxE;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,QAAQ,EAAE;IASvE;;;;;OAKG;IACH,SAAS,CAAC,qBAAqB,CAAC,gBAAgB,EAAE,MAAM,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE;IAyB/E,OAAO,CAAC,eAAe;CAmCxB;AAED;;;;;;;GAOG;AACH,8BAAsB,iBAAiB;IACrC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM;IAkB/C,cAAc,CAAC,EACb,IAAI,EACJ,SAAS,EACT,aAAa,GACd,EAAE;QACD,IAAI,EAAE,cAAc,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,cAAc,CAAC;KAChC,GAAG,gBAAgB,EAAE;IAIhB,SAAS,CACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,EACpB,QAAQ,EAAE,WAAW,GAAG,SAAS,EACjC,aAAa,EAAE,cAAc,GAAG,SAAS,GACxC,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,gBAAgB,EAAE,CAAA;KAAE,CAAC;CAgCvF"}
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/base.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,QAAQ,EACT,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAQjE;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,gBAAgB,EAAE,MAAM,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,EAAE,CAuBpF;AAED;;GAEG;AACH,8BAAsB,iBAAiB;IASnC,SAAS,CAAC,QAAQ,EAAE,WAAW;IAC/B,SAAS,CAAC,OAAO,EAAE,MAAM;IACzB,SAAS,CAAC,SAAS,EAAE,MAAM;IAC3B,SAAS,CAAC,MAAM,EAAE,YAAY;IAXhC;;;;;;OAMG;gBAES,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,YAAiB;IAKrC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAEjD;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;IAE7D;;;;;OAKG;IACG,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE,MAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAmDxE;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,QAAQ,EAAE;IASvE;;;;OAIG;IACH,OAAO,CAAC,eAAe;CAmCxB;AAED;;;;;;;GAOG;AACH,8BAAsB,iBAAiB;IACrC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM;IAkB/C,cAAc,CAAC,EACb,IAAI,EACJ,SAAS,EACT,aAAa,GACd,EAAE;QACD,IAAI,EAAE,cAAc,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,cAAc,CAAC;KAChC,GAAG,gBAAgB,EAAE;IAIhB,SAAS,CACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,EACpB,QAAQ,EAAE,WAAW,GAAG,SAAS,EACjC,aAAa,EAAE,cAAc,GAAG,SAAS,GACxC,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,gBAAgB,EAAE,CAAA;KAAE,CAAC;CAyCvF"}

package/dist/src/redteam/plugins/base.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RedteamGraderBase = exports.RedteamPluginBase = void 0;
+exports.parseGeneratedPrompts = parseGeneratedPrompts;
 const dedent_1 = __importDefault(require("dedent"));
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const cliState_1 = __importDefault(require("../../cliState"));
@@ -15,6 +16,34 @@ const templates_1 = require("../../util/templates");
 const time_1 = require("../../util/time");
 const shared_1 = require("../providers/shared");
 const util_2 = require("../util");
+/**
+ * Parses the LLM response of generated prompts into an array of objects.
+ *
+ * @param generatedPrompts - The LLM response of generated prompts.
+ * @returns An array of { prompt: string } objects. Each of these objects represents a test case.
+ */
+function parseGeneratedPrompts(generatedPrompts) {
+    const parsePrompt = (line) => {
+        if (!line.toLowerCase().includes('prompt:')) {
+            return null;
+        }
+        let prompt = (0, util_2.removePrefix)(line, 'Prompt');
+        // Handle numbered lists with various formats
+        prompt = prompt.replace(/^\d+[\.\)\-]?\s*-?\s*/, '');
+        // Handle quotes
+        prompt = prompt.replace(/^["'](.*)["']$/, '$1');
+        // Handle nested quotes
+        prompt = prompt.replace(/^'([^']*(?:'{2}[^']*)*)'$/, (_, p1) => p1.replace(/''/g, "'"));
+        prompt = prompt.replace(/^"([^"]*(?:"{2}[^"]*)*)"$/, (_, p1) => p1.replace(/""/g, '"'));
+        return prompt.trim();
+    };
+    // Split by newline or semicolon
+    const promptLines = generatedPrompts.split(/[\n;]+/);
+    return promptLines
+        .map(parsePrompt)
+        .filter((prompt) => prompt !== null)
+        .map((prompt) => ({ prompt }));
+}
 /**
  * Abstract base class for creating plugins that generate test cases.
  */
@@ -55,6 +84,7 @@ class RedteamPluginBase {
             const renderedTemplate = nunjucks.renderString(await this.getTemplate(), {
                 purpose: this.purpose,
                 n: currentBatchSize,
+                examples: this.config.examples,
             });
             const finalTemplate = this.appendModifiers(renderedTemplate);
             const { output: generatedPrompts, error } = await this.provider.callApi(finalTemplate);
@@ -70,7 +100,7 @@ class RedteamPluginBase {
                 logger_1.default.error(`Malformed response from API provider: Expected generatedPrompts to be a string, got ${typeof generatedPrompts}: ${JSON.stringify(generatedPrompts)}`);
                 return [];
             }
-            return this.parseGeneratedPrompts(generatedPrompts);
+            return parseGeneratedPrompts(generatedPrompts);
         };
         const allPrompts = await (0, generation_1.retryWithDeduplication)(generatePrompts, n);
         const prompts = (0, generation_1.sampleArray)(allPrompts, n);
@@ -91,37 +121,14 @@ class RedteamPluginBase {
         }));
     }
     /**
-     * Parses the LLM response of generated prompts into an array of objects.
-     *
-     * @param generatedPrompts - The LLM response of generated prompts.
-     * @returns An array of { prompt: string } objects. Each of these objects represents a test case.
+     * Appends modifiers to the template.
+     * @param template - The template to append modifiers to.
+     * @returns The modified template.
      */
-    parseGeneratedPrompts(generatedPrompts) {
-        const parsePrompt = (line) => {
-            if (!line.toLowerCase().includes('prompt:')) {
-                return null;
-            }
-            let prompt = (0, util_2.removePrefix)(line, 'Prompt');
-            // Handle numbered lists with various formats
-            prompt = prompt.replace(/^\d+[\.\)\-]?\s*-?\s*/, '');
-            // Handle quotes
-            prompt = prompt.replace(/^["'](.*)["']$/, '$1');
-            // Handle nested quotes
-            prompt = prompt.replace(/^'([^']*(?:'{2}[^']*)*)'$/, (_, p1) => p1.replace(/''/g, "'"));
-            prompt = prompt.replace(/^"([^"]*(?:"{2}[^"]*)*)"$/, (_, p1) => p1.replace(/""/g, '"'));
-            return prompt.trim();
-        };
-        // Split by newline or semicolon
-        const promptLines = generatedPrompts.split(/[\n;]+/);
-        return promptLines
-            .map(parsePrompt)
-            .filter((prompt) => prompt !== null)
-            .map((prompt) => ({ prompt }));
-    }
     appendModifiers(template) {
         // Take everything under "modifiers" config key
         const modifiers = this.config.modifiers ?? {};
-        // Right now, only pre-configured modifier is language
+        // `language` is a special top-level config field
         if (this.config.language) {
             (0, tiny_invariant_1.default)(typeof this.config.language === 'string', 'language must be a string');
             modifiers.language = this.config.language;
@@ -186,7 +193,15 @@ class RedteamGraderBase {
             tools: (0, util_1.maybeLoadFromExternalFile)(provider?.config?.tools),
             value: renderedValue,
         };
-        const finalRubric = this.renderRubric(vars);
+        // Grader examples are appended to all rubrics if present.
+        const graderExamples = test.metadata?.pluginConfig?.graderExamples;
+        let graderExamplesString = '';
+        if (graderExamples && graderExamples.length > 0) {
+            graderExamplesString =
+                '\n\n' +
+                    graderExamples.map((example) => `EXAMPLE OUTPUT: ${JSON.stringify(example)}`).join('\n');
+        }
+        const finalRubric = this.renderRubric(vars) + graderExamplesString;
         const grade = await (0, matchers_1.matchesLlmRubric)(finalRubric, llmOutput, {
             ...test.options,
             provider: await (0, shared_1.loadRedteamProvider)({