promptfoo 0.90.3 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +4 -4
- package/dist/src/app/assets/{index-Nu_Jrwyo.js → index-Dvv-swGm.js} +161 -161
- package/dist/src/app/assets/{index.es-Cwq4F6NP.js → index.es-DAIlgY6E.js} +1 -1
- package/dist/src/app/assets/{sync-CE7pFgME.js → sync-CABaZxhX.js} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/checkNodeVersion.d.ts.map +1 -1
- package/dist/src/checkNodeVersion.js +4 -37
- package/dist/src/checkNodeVersion.js.map +1 -1
- package/dist/src/commands/eval.d.ts +1 -1
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/list.js +2 -1
- package/dist/src/commands/list.js.map +1 -1
- package/dist/src/database/tables.d.ts +5 -0
- package/dist/src/database/tables.d.ts.map +1 -1
- package/dist/src/envars.d.ts +1 -0
- package/dist/src/envars.d.ts.map +1 -1
- package/dist/src/envars.js.map +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +12 -2
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/index.d.ts +9 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +11 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/main.js +9 -9
- package/dist/src/main.js.map +1 -1
- package/dist/src/models/prompt.d.ts +6 -0
- package/dist/src/models/prompt.d.ts.map +1 -0
- package/dist/src/models/prompt.js +10 -0
- package/dist/src/models/prompt.js.map +1 -0
- package/dist/src/prompts/index.d.ts +1 -1
- package/dist/src/prompts/index.d.ts.map +1 -1
- package/dist/src/prompts/index.js.map +1 -1
- package/dist/src/providers/browser.d.ts.map +1 -1
- package/dist/src/providers/browser.js +24 -1
- package/dist/src/providers/browser.js.map +1 -1
- package/dist/src/providers/golangCompletion.d.ts.map +1 -1
- package/dist/src/providers/golangCompletion.js +2 -1
- package/dist/src/providers/golangCompletion.js.map +1 -1
- package/dist/src/providers/openai.js +1 -1
- package/dist/src/providers/pythonCompletion.d.ts.map +1 -1
- package/dist/src/providers/pythonCompletion.js +2 -1
- package/dist/src/providers/pythonCompletion.js.map +1 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +23 -5
- package/dist/src/providers.js.map +1 -1
- package/dist/src/redteam/commands/generate.d.ts +2 -2
- package/dist/src/redteam/commands/generate.d.ts.map +1 -1
- package/dist/src/redteam/commands/generate.js +3 -3
- package/dist/src/redteam/commands/generate.js.map +1 -1
- package/dist/src/redteam/commands/init.d.ts.map +1 -1
- package/dist/src/redteam/commands/init.js +3 -4
- package/dist/src/redteam/commands/init.js.map +1 -1
- package/dist/src/redteam/commands/report.d.ts +3 -0
- package/dist/src/redteam/commands/report.d.ts.map +1 -0
- package/dist/src/redteam/commands/report.js +31 -0
- package/dist/src/redteam/commands/report.js.map +1 -0
- package/dist/src/redteam/commands/run.d.ts +3 -0
- package/dist/src/redteam/commands/run.d.ts.map +1 -0
- package/dist/src/redteam/commands/run.js +137 -0
- package/dist/src/redteam/commands/run.js.map +1 -0
- package/dist/src/redteam/graders.d.ts +2 -2
- package/dist/src/redteam/graders.d.ts.map +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.d.ts +2 -2
- package/dist/src/redteam/plugins/asciiSmuggling.d.ts.map +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.js +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts +3 -3
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +7 -7
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/bfla.d.ts +2 -2
- package/dist/src/redteam/plugins/bfla.d.ts.map +1 -1
- package/dist/src/redteam/plugins/bfla.js +1 -1
- package/dist/src/redteam/plugins/bfla.js.map +1 -1
- package/dist/src/redteam/plugins/bola.d.ts +2 -2
- package/dist/src/redteam/plugins/bola.d.ts.map +1 -1
- package/dist/src/redteam/plugins/bola.js +1 -1
- package/dist/src/redteam/plugins/bola.js.map +1 -1
- package/dist/src/redteam/plugins/competitors.d.ts +2 -2
- package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
- package/dist/src/redteam/plugins/competitors.js +1 -1
- package/dist/src/redteam/plugins/competitors.js.map +1 -1
- package/dist/src/redteam/plugins/contracts.d.ts +3 -3
- package/dist/src/redteam/plugins/contracts.d.ts.map +1 -1
- package/dist/src/redteam/plugins/contracts.js +2 -2
- package/dist/src/redteam/plugins/contracts.js.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts +3 -3
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js +2 -2
- package/dist/src/redteam/plugins/crossSessionLeak.js.map +1 -1
- package/dist/src/redteam/plugins/custom.d.ts +2 -2
- package/dist/src/redteam/plugins/custom.d.ts.map +1 -1
- package/dist/src/redteam/plugins/custom.js +1 -1
- package/dist/src/redteam/plugins/custom.js.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.d.ts +3 -3
- package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.js +2 -2
- package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.d.ts +3 -3
- package/dist/src/redteam/plugins/excessiveAgency.d.ts.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.js +2 -2
- package/dist/src/redteam/plugins/excessiveAgency.js.map +1 -1
- package/dist/src/redteam/plugins/hallucination.d.ts +3 -3
- package/dist/src/redteam/plugins/hallucination.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hallucination.js +2 -2
- package/dist/src/redteam/plugins/hallucination.js.map +1 -1
- package/dist/src/redteam/plugins/harmful.d.ts +2 -2
- package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful.js +4 -7
- package/dist/src/redteam/plugins/harmful.js.map +1 -1
- package/dist/src/redteam/plugins/hijacking.d.ts +2 -2
- package/dist/src/redteam/plugins/hijacking.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hijacking.js +1 -1
- package/dist/src/redteam/plugins/hijacking.js.map +1 -1
- package/dist/src/redteam/plugins/imitation.d.ts +3 -3
- package/dist/src/redteam/plugins/imitation.d.ts.map +1 -1
- package/dist/src/redteam/plugins/imitation.js +2 -2
- package/dist/src/redteam/plugins/imitation.js.map +1 -1
- package/dist/src/redteam/plugins/indirectPromptInjection.d.ts +3 -19
- package/dist/src/redteam/plugins/indirectPromptInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/indirectPromptInjection.js +2 -73
- package/dist/src/redteam/plugins/indirectPromptInjection.js.map +1 -1
- package/dist/src/redteam/plugins/overreliance.d.ts +3 -3
- package/dist/src/redteam/plugins/overreliance.d.ts.map +1 -1
- package/dist/src/redteam/plugins/overreliance.js +2 -2
- package/dist/src/redteam/plugins/overreliance.js.map +1 -1
- package/dist/src/redteam/plugins/pii.d.ts +2 -2
- package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
- package/dist/src/redteam/plugins/pii.js +3 -4
- package/dist/src/redteam/plugins/pii.js.map +1 -1
- package/dist/src/redteam/plugins/policy.d.ts +3 -3
- package/dist/src/redteam/plugins/policy.d.ts.map +1 -1
- package/dist/src/redteam/plugins/policy.js +2 -2
- package/dist/src/redteam/plugins/policy.js.map +1 -1
- package/dist/src/redteam/plugins/politics.d.ts +3 -3
- package/dist/src/redteam/plugins/politics.d.ts.map +1 -1
- package/dist/src/redteam/plugins/politics.js +2 -2
- package/dist/src/redteam/plugins/politics.js.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.d.ts +3 -3
- package/dist/src/redteam/plugins/promptExtraction.d.ts.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.js +2 -2
- package/dist/src/redteam/plugins/promptExtraction.js.map +1 -1
- package/dist/src/redteam/plugins/rbac.d.ts +3 -3
- package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
- package/dist/src/redteam/plugins/rbac.js +2 -2
- package/dist/src/redteam/plugins/rbac.js.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.d.ts +3 -3
- package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.js +2 -2
- package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.d.ts +3 -3
- package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.js +2 -2
- package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
- package/dist/src/redteam/plugins/ssrf.d.ts +2 -2
- package/dist/src/redteam/plugins/ssrf.d.ts.map +1 -1
- package/dist/src/redteam/plugins/ssrf.js +1 -1
- package/dist/src/redteam/plugins/ssrf.js.map +1 -1
- package/dist/src/redteam/providers/constants.d.ts +1 -1
- package/dist/src/redteam/providers/constants.d.ts.map +1 -1
- package/dist/src/redteam/providers/constants.js +1 -1
- package/dist/src/redteam/providers/constants.js.map +1 -1
- package/dist/src/server/server.d.ts +2 -1
- package/dist/src/server/server.d.ts.map +1 -1
- package/dist/src/server/server.js +15 -7
- package/dist/src/server/server.js.map +1 -1
- package/dist/src/share.d.ts.map +1 -1
- package/dist/src/share.js +16 -1
- package/dist/src/share.js.map +1 -1
- package/dist/src/types/index.d.ts +135 -0
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js +1 -0
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/util/createHash.d.ts +2 -0
- package/dist/src/util/createHash.d.ts.map +1 -0
- package/dist/src/util/createHash.js +8 -0
- package/dist/src/util/createHash.js.map +1 -0
- package/dist/src/util/index.d.ts +4 -2
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +16 -18
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/transform.d.ts +5 -1
- package/dist/src/util/transform.d.ts.map +1 -1
- package/dist/src/util/transform.js +13 -7
- package/dist/src/util/transform.js.map +1 -1
- package/dist/src/validators/redteam.d.ts.map +1 -1
- package/dist/src/validators/redteam.js +70 -33
- package/dist/src/validators/redteam.js.map +1 -1
- package/dist/test/checkNodeVersion.test.js +11 -58
- package/dist/test/checkNodeVersion.test.js.map +1 -1
- package/dist/test/evaluator.test.js +53 -0
- package/dist/test/evaluator.test.js.map +1 -1
- package/dist/test/index.test.js +170 -8
- package/dist/test/index.test.js.map +1 -1
- package/dist/test/providers.test.js +36 -1
- package/dist/test/providers.test.js.map +1 -1
- package/dist/test/redteam/plugins/base.test.js +6 -6
- package/dist/test/redteam/plugins/base.test.js.map +1 -1
- package/dist/test/redteam/validators.test.js +165 -7
- package/dist/test/redteam/validators.test.js.map +1 -1
- package/dist/test/util.transform.test.js +17 -0
- package/dist/test/util.transform.test.js.map +1 -1
- package/package.json +4 -4
- package/dist/src/commands/version.d.ts +0 -3
- package/dist/src/commands/version.d.ts.map +0 -1
- package/dist/src/commands/version.js +0 -18
- package/dist/src/commands/version.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../src/redteam/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../../src/redteam/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmDnB,CAAC;AAEF,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAEvE"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AtomicTestCase, GradingResult } from '../../types';
|
|
2
|
-
import {
|
|
2
|
+
import { RedteamGraderBase } from './base';
|
|
3
3
|
export declare const PLUGIN_ID = "promptfoo:redteam:ascii-smuggling";
|
|
4
|
-
export declare class AsciiSmugglingGrader extends
|
|
4
|
+
export declare class AsciiSmugglingGrader extends RedteamGraderBase {
|
|
5
5
|
id: string;
|
|
6
6
|
rubric: string;
|
|
7
7
|
getResult(prompt: string, llmOutput: string, test: AtomicTestCase): Promise<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"asciiSmuggling.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/asciiSmuggling.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"asciiSmuggling.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/asciiSmuggling.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,sCAAsC,CAAC;AAE7D,qBAAa,oBAAqB,SAAQ,iBAAiB;IACzD,EAAE,SAAa;IACf,MAAM,SAAM;IACN,SAAS,CACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAarD"}
|
|
@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.AsciiSmugglingGrader = exports.PLUGIN_ID = void 0;
|
|
4
4
|
const base_1 = require("./base");
|
|
5
5
|
exports.PLUGIN_ID = 'promptfoo:redteam:ascii-smuggling';
|
|
6
|
-
class AsciiSmugglingGrader extends base_1.
|
|
6
|
+
class AsciiSmugglingGrader extends base_1.RedteamGraderBase {
|
|
7
7
|
constructor() {
|
|
8
8
|
super(...arguments);
|
|
9
9
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"asciiSmuggling.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/asciiSmuggling.ts"],"names":[],"mappings":";;;AACA,
|
|
1
|
+
{"version":3,"file":"asciiSmuggling.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/asciiSmuggling.ts"],"names":[],"mappings":";;;AACA,iCAA2C;AAE9B,QAAA,SAAS,GAAG,mCAAmC,CAAC;AAE7D,MAAa,oBAAqB,SAAQ,wBAAiB;IAA3D;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,EAAE,CAAC;IAkBd,CAAC;IAjBC,KAAK,CAAC,SAAS,CACb,MAAc,EACd,SAAiB,EACjB,IAAoB;QAEpB,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,wBAAwB,CAAC,CAAC;QAC1E,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,CAAC,KAAK;gBACZ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpB,MAAM,EAAE,KAAK;oBACX,CAAC,CAAC,sCAAsC;oBACxC,CAAC,CAAC,qCAAqC;aAC1C;YACD,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;CACF;AApBD,oDAoBC"}
|
|
@@ -3,13 +3,13 @@ import type { AtomicTestCase, GradingResult } from '../../types';
|
|
|
3
3
|
/**
|
|
4
4
|
* Abstract base class for creating plugins that generate test cases.
|
|
5
5
|
*/
|
|
6
|
-
export declare abstract class
|
|
6
|
+
export declare abstract class RedteamPluginBase {
|
|
7
7
|
protected provider: ApiProvider;
|
|
8
8
|
protected purpose: string;
|
|
9
9
|
protected injectVar: string;
|
|
10
10
|
protected config: PluginConfig;
|
|
11
11
|
/**
|
|
12
|
-
* Creates an instance of
|
|
12
|
+
* Creates an instance of RedteamPluginBase.
|
|
13
13
|
* @param provider - The API provider used for generating prompts.
|
|
14
14
|
* @param purpose - The purpose of the plugin.
|
|
15
15
|
* @param injectVar - The variable name to inject the generated prompt into.
|
|
@@ -60,7 +60,7 @@ export declare abstract class PluginBase {
|
|
|
60
60
|
*
|
|
61
61
|
* But if you'd like, you can override the `getResult` method to use a different grading method.
|
|
62
62
|
*/
|
|
63
|
-
export declare abstract class
|
|
63
|
+
export declare abstract class RedteamGraderBase {
|
|
64
64
|
abstract id: string;
|
|
65
65
|
abstract rubric: string;
|
|
66
66
|
renderRubric(vars: Record<string, any>): string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/base.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,QAAQ,EACT,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAOjE;;GAEG;AACH,8BAAsB,
|
|
1
|
+
{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/base.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,QAAQ,EACT,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAOjE;;GAEG;AACH,8BAAsB,iBAAiB;IASnC,SAAS,CAAC,QAAQ,EAAE,WAAW;IAC/B,SAAS,CAAC,OAAO,EAAE,MAAM;IACzB,SAAS,CAAC,SAAS,EAAE,MAAM;IAC3B,SAAS,CAAC,MAAM,EAAE,YAAY;IAXhC;;;;;;OAMG;gBAES,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,YAAiB;IAKrC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAEjD;;;;OAIG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;IAE7D;;;;;OAKG;IACG,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE,MAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAkDxE;;;;OAIG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,QAAQ,EAAE;IASvE;;;;;OAKG;IACH,SAAS,CAAC,qBAAqB,CAAC,gBAAgB,EAAE,MAAM,GAAG;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE;IAyB/E,OAAO,CAAC,eAAe;CAmCxB;AAED;;;;;;;GAOG;AACH,8BAAsB,iBAAiB;IACrC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM;IAkB/C,cAAc,CAAC,EACb,IAAI,EACJ,SAAS,EACT,aAAa,GACd,EAAE;QACD,IAAI,EAAE,cAAc,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,CAAC,EAAE,cAAc,CAAC;KAChC,GAAG,gBAAgB,EAAE;IAIhB,SAAS,CACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,EACpB,QAAQ,EAAE,WAAW,GAAG,SAAS,EACjC,aAAa,EAAE,cAAc,GAAG,SAAS,GACxC,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,gBAAgB,EAAE,CAAA;KAAE,CAAC;CAgCvF"}
|
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.
|
|
6
|
+
exports.RedteamGraderBase = exports.RedteamPluginBase = void 0;
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
9
9
|
const cliState_1 = __importDefault(require("../../cliState"));
|
|
@@ -17,9 +17,9 @@ const util_2 = require("../util");
|
|
|
17
17
|
/**
|
|
18
18
|
* Abstract base class for creating plugins that generate test cases.
|
|
19
19
|
*/
|
|
20
|
-
class
|
|
20
|
+
class RedteamPluginBase {
|
|
21
21
|
/**
|
|
22
|
-
* Creates an instance of
|
|
22
|
+
* Creates an instance of RedteamPluginBase.
|
|
23
23
|
* @param provider - The API provider used for generating prompts.
|
|
24
24
|
* @param purpose - The purpose of the plugin.
|
|
25
25
|
* @param injectVar - The variable name to inject the generated prompt into.
|
|
@@ -30,7 +30,7 @@ class PluginBase {
|
|
|
30
30
|
this.purpose = purpose;
|
|
31
31
|
this.injectVar = injectVar;
|
|
32
32
|
this.config = config;
|
|
33
|
-
logger_1.default.debug(`
|
|
33
|
+
logger_1.default.debug(`RedteamPluginBase initialized with purpose: ${purpose}, injectVar: ${injectVar}`);
|
|
34
34
|
}
|
|
35
35
|
/**
|
|
36
36
|
* Generates test cases based on the plugin's configuration.
|
|
@@ -146,7 +146,7 @@ class PluginBase {
|
|
|
146
146
|
`.trim();
|
|
147
147
|
}
|
|
148
148
|
}
|
|
149
|
-
exports.
|
|
149
|
+
exports.RedteamPluginBase = RedteamPluginBase;
|
|
150
150
|
/**
|
|
151
151
|
* Base class for all redteam graders.
|
|
152
152
|
*
|
|
@@ -155,7 +155,7 @@ exports.PluginBase = PluginBase;
|
|
|
155
155
|
*
|
|
156
156
|
* But if you'd like, you can override the `getResult` method to use a different grading method.
|
|
157
157
|
*/
|
|
158
|
-
class
|
|
158
|
+
class RedteamGraderBase {
|
|
159
159
|
renderRubric(vars) {
|
|
160
160
|
const nunjucks = (0, templates_1.getNunjucksEngine)(undefined, true /* throwOnUndefined */);
|
|
161
161
|
try {
|
|
@@ -206,5 +206,5 @@ class RedteamModelGrader {
|
|
|
206
206
|
return { grade, rubric: finalRubric, suggestions };
|
|
207
207
|
}
|
|
208
208
|
}
|
|
209
|
-
exports.
|
|
209
|
+
exports.RedteamGraderBase = RedteamGraderBase;
|
|
210
210
|
//# sourceMappingURL=base.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/base.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,oEAAuC;AACvC,8DAAsC;AACtC,0DAAkC;AAClC,6CAAkD;AAUlD,qCAAuD;AACvD,sDAA4E;AAC5E,oDAAyD;AACzD,gDAA0D;AAC1D,kCAAuC;AAEvC;;GAEG;AACH,MAAsB,
|
|
1
|
+
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/base.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,oEAAuC;AACvC,8DAAsC;AACtC,0DAAkC;AAClC,6CAAkD;AAUlD,qCAAuD;AACvD,sDAA4E;AAC5E,oDAAyD;AACzD,gDAA0D;AAC1D,kCAAuC;AAEvC;;GAEG;AACH,MAAsB,iBAAiB;IACrC;;;;;;OAMG;IACH,YACY,QAAqB,EACrB,OAAe,EACf,SAAiB,EACjB,SAAuB,EAAE;QAHzB,aAAQ,GAAR,QAAQ,CAAa;QACrB,YAAO,GAAP,OAAO,CAAQ;QACf,cAAS,GAAT,SAAS,CAAQ;QACjB,WAAM,GAAN,MAAM,CAAmB;QAEnC,gBAAM,CAAC,KAAK,CAAC,+CAA+C,OAAO,gBAAgB,SAAS,EAAE,CAAC,CAAC;IAClG,CAAC;IAcD;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CAAC,CAAS,EAAE,UAAkB,CAAC;QAChD,gBAAM,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,EAAE,CAAC;QAErB;;;;WAIG;QACH,MAAM,eAAe,GAAG,KAAK,EAC3B,cAAoC,EACL,EAAE;YACjC,MAAM,cAAc,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;YACjD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAE7D,gBAAM,CAAC,KAAK,CAAC,uBAAuB,gBAAgB,UAAU,CAAC,CAAC;YAChE,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;YACrC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,EAAE;gBACvE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,CAAC,EAAE,gBAAgB;aACpB,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC7D,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YACvF,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,gBAAM,CAAC,KAAK,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;gBAC1C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAC/D,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,gBAAM,CAAC,KAAK,CACV,oDAAoD,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,KAAK,EAAE,CACtF,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,IAAI,OAAO,gBAAgB,KAAK,QAAQ,EAAE,CAAC;gBACzC,gBAAM,CAAC,KAAK,CACV,uFAAuF,OAAO,gBAAgB,KAAK,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAAE,CACtJ,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,OAAO,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,CAAC;QACtD,CAAC,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,IAAA,mCAAsB,EAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,IAAA,wBAAW,EAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC3C,gBAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,+BAA+B,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;QAC9F,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACO,kBAAkB,CAAC,OAA6B;QACxD,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,EAAE;gBACJ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM;aAChC;YACD,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;SAC1C,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACO,qBAAqB,CAAC,gBAAwB;QACtD,MAAM,WAAW,GAAG,CAAC,IAAY,EAAiB,EAAE;YAClD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC5C,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,MAAM,GAAG,IAAA,mBAAY,EAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1C,6CAA6C;YAC7C,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;YACrD,gBAAgB;YAChB,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YAChD,uBAAuB;YACvB,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,2BAA2B,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxF,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,2BAA2B,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxF,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;QACvB,CAAC,CAAC;QAEF,gCAAgC;QAChC,MAAM,WAAW,GAAG,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAErD,OAAO,WAAW;aACf,GAAG,CAAC,WAAW,CAAC;aAChB,MAAM,CAAC,CAAC,MAAM,EAAoB,EAAE,CAAC,MAAM,KAAK,IAAI,CAAC;aACrD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACnC,CAAC;IAEO,eAAe,CAAC,QAAgB;QACtC,+CAA+C;QAC/C,MAAM,SAAS,GACZ,IAAI,CAAC,MAAM,CAAC,SAAoC,IAAI,EAAE,CAAC;QAE1D,sDAAsD;QACtD,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzB,IAAA,wBAAS,EAAC,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,2BAA2B,CAAC,CAAC;YACjF,SAAS,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC;QAC5C,CAAC;QAED,eAAe;QACf,IACE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,EAAE,CAAC,EACvF,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,uBAAuB;QACvB,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;aAC9C,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,EAAE,CAAC;aACtE,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,KAAK,EAAE,CAAC;aACzC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,OAAO,IAAA,gBAAM,EAAA;QACT,QAAQ,CAAC,IAAI,EAAE;;;;QAIf,eAAe;;;KAGlB,CAAC,IAAI,EAAE,CAAC;IACX,CAAC;CACF;AArKD,8CAqKC;AAED;;;;;;;GAOG;AACH,MAAsB,iBAAiB;IAIrC,YAAY,CAAC,IAAyB;QACpC,MAAM,QAAQ,GAAG,IAAA,6BAAiB,EAAC,SAAS,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QAE3E,IAAI,CAAC;YACH,OAAO,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,gBAAM,CAAC,KAAK,CAAC,oCAAoC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAChE,gBAAM,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACzC,gBAAM,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,IAAA,gBAAM,EAAA;2CACe,GAAG,CAAC,OAAO;;qBAEjC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;OAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,cAAc,CAAC,EACb,IAAI,EACJ,SAAS,EACT,aAAa,GAKd;QACC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,MAAc,EACd,SAAiB,EACjB,IAAoB,EACpB,QAAiC,EACjC,aAAyC;QAEzC,IAAA,wBAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,kCAAkC,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG;YACX,GAAG,IAAI,CAAC,QAAQ;YAChB,MAAM;YACN,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE;YACvC,KAAK,EAAE,IAAA,gCAAyB,EAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC;YACzD,KAAK,EAAE,aAAa;SACrB,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAE5C,MAAM,KAAK,GAAG,MAAM,IAAA,2BAAgB,EAAC,WAAW,EAAE,SAAS,EAAE;YAC3D,GAAG,IAAI,CAAC,OAAO;YACf,QAAQ,EAAE,MAAM,IAAA,4BAAmB,EAAC;gBAClC,QAAQ;gBACN,yGAAyG;gBACzG,kBAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ;oBACtC,kBAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI;oBACrD,kBAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ;gBACjD,QAAQ,EAAE,IAAI;aACf,CAAC;SACH,CAAC,CAAC;QACH,gBAAM,CAAC,KAAK,CAAC,8BAA8B,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAElF,IAAI,WAAW,CAAC;QAChB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAChB,iDAAiD;YACjD,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC;IACrD,CAAC;CACF;AAxED,8CAwEC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bfla.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/bfla.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"bfla.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/bfla.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAI3C,qBAAa,UAAW,SAAQ,iBAAiB;IAC/C,EAAE,SAAa;IACf,MAAM,SAqCJ;CACH"}
|
|
@@ -7,7 +7,7 @@ exports.BflaGrader = void 0;
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
const PLUGIN_ID = 'promptfoo:redteam:bfla';
|
|
10
|
-
class BflaGrader extends base_1.
|
|
10
|
+
class BflaGrader extends base_1.RedteamGraderBase {
|
|
11
11
|
constructor() {
|
|
12
12
|
super(...arguments);
|
|
13
13
|
this.id = PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bfla.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/bfla.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,
|
|
1
|
+
{"version":3,"file":"bfla.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/bfla.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,iCAA2C;AAE3C,MAAM,SAAS,GAAG,wBAAwB,CAAC;AAE3C,MAAa,UAAW,SAAQ,wBAAiB;IAAjD;;QACE,OAAE,GAAG,SAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCd,CAAC;IACJ,CAAC;CAAA;AAxCD,gCAwCC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RedteamGraderBase } from './base';
|
|
2
2
|
export declare const PLUGIN_ID = "promptfoo:redteam:bola";
|
|
3
|
-
export declare class BolaGrader extends
|
|
3
|
+
export declare class BolaGrader extends RedteamGraderBase {
|
|
4
4
|
id: string;
|
|
5
5
|
rubric: string;
|
|
6
6
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bola.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/bola.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"bola.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/bola.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,2BAA2B,CAAC;AAElD,qBAAa,UAAW,SAAQ,iBAAiB;IAC/C,EAAE,SAAa;IACf,MAAM,SA+CJ;CACH"}
|
|
@@ -7,7 +7,7 @@ exports.BolaGrader = exports.PLUGIN_ID = void 0;
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
exports.PLUGIN_ID = 'promptfoo:redteam:bola';
|
|
10
|
-
class BolaGrader extends base_1.
|
|
10
|
+
class BolaGrader extends base_1.RedteamGraderBase {
|
|
11
11
|
constructor() {
|
|
12
12
|
super(...arguments);
|
|
13
13
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bola.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/bola.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,
|
|
1
|
+
{"version":3,"file":"bola.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/bola.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,iCAA2C;AAE9B,QAAA,SAAS,GAAG,wBAAwB,CAAC;AAElD,MAAa,UAAW,SAAQ,wBAAiB;IAAjD;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+Cd,CAAC;IACJ,CAAC;CAAA;AAlDD,gCAkDC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RedteamGraderBase } from './base';
|
|
2
2
|
export declare const PLUGIN_ID = "promptfoo:redteam:competitors";
|
|
3
|
-
export declare class CompetitorsGrader extends
|
|
3
|
+
export declare class CompetitorsGrader extends RedteamGraderBase {
|
|
4
4
|
id: string;
|
|
5
5
|
rubric: string;
|
|
6
6
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"competitors.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/competitors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"competitors.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/competitors.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE3C,eAAO,MAAM,SAAS,kCAAkC,CAAC;AAEzD,qBAAa,iBAAkB,SAAQ,iBAAiB;IACtD,EAAE,SAAa;IACf,MAAM,SA2BJ;CACH"}
|
|
@@ -7,7 +7,7 @@ exports.CompetitorsGrader = exports.PLUGIN_ID = void 0;
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
exports.PLUGIN_ID = 'promptfoo:redteam:competitors';
|
|
10
|
-
class CompetitorsGrader extends base_1.
|
|
10
|
+
class CompetitorsGrader extends base_1.RedteamGraderBase {
|
|
11
11
|
constructor() {
|
|
12
12
|
super(...arguments);
|
|
13
13
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"competitors.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/competitors.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,
|
|
1
|
+
{"version":3,"file":"competitors.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/competitors.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,iCAA2C;AAE9B,QAAA,SAAS,GAAG,+BAA+B,CAAC;AAEzD,MAAa,iBAAkB,SAAQ,wBAAiB;IAAxD;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2Bd,CAAC;IACJ,CAAC;CAAA;AA9BD,8CA8BC"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { Assertion } from '../../types';
|
|
2
|
-
import {
|
|
2
|
+
import { RedteamPluginBase, RedteamGraderBase } from './base';
|
|
3
3
|
export declare const PLUGIN_ID = "promptfoo:redteam:contracts";
|
|
4
|
-
export declare class ContractPlugin extends
|
|
4
|
+
export declare class ContractPlugin extends RedteamPluginBase {
|
|
5
5
|
protected getTemplate(): Promise<string>;
|
|
6
6
|
protected getAssertions(prompt: string): Assertion[];
|
|
7
7
|
}
|
|
8
|
-
export declare class ContractsGrader extends
|
|
8
|
+
export declare class ContractsGrader extends RedteamGraderBase {
|
|
9
9
|
id: string;
|
|
10
10
|
rubric: string;
|
|
11
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contracts.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/contracts.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"contracts.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/contracts.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE9D,eAAO,MAAM,SAAS,gCAAgC,CAAC;AAEvD,qBAAa,cAAe,SAAQ,iBAAiB;cACnC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IA6B9C,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;CAQrD;AAED,qBAAa,eAAgB,SAAQ,iBAAiB;IACpD,EAAE,SAAa;IACf,MAAM,SA4BJ;CACH"}
|
|
@@ -7,7 +7,7 @@ exports.ContractsGrader = exports.ContractPlugin = exports.PLUGIN_ID = void 0;
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
exports.PLUGIN_ID = 'promptfoo:redteam:contracts';
|
|
10
|
-
class ContractPlugin extends base_1.
|
|
10
|
+
class ContractPlugin extends base_1.RedteamPluginBase {
|
|
11
11
|
async getTemplate() {
|
|
12
12
|
return (0, dedent_1.default) `
|
|
13
13
|
In order to test whether an AI can be influenced to enter into unintended business contracts, we propose a "contract test".
|
|
@@ -46,7 +46,7 @@ class ContractPlugin extends base_1.PluginBase {
|
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
exports.ContractPlugin = ContractPlugin;
|
|
49
|
-
class ContractsGrader extends base_1.
|
|
49
|
+
class ContractsGrader extends base_1.RedteamGraderBase {
|
|
50
50
|
constructor() {
|
|
51
51
|
super(...arguments);
|
|
52
52
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contracts.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/contracts.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,
|
|
1
|
+
{"version":3,"file":"contracts.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/contracts.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,iCAA8D;AAEjD,QAAA,SAAS,GAAG,6BAA6B,CAAC;AAEvD,MAAa,cAAe,SAAQ,wBAAiB;IACzC,KAAK,CAAC,WAAW;QACzB,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBd,CAAC;IACF,CAAC;IAES,aAAa,CAAC,MAAc;QACpC,OAAO;YACL;gBACE,IAAI,EAAE,iBAAS;gBACf,MAAM,EAAE,uBAAuB;aAChC;SACF,CAAC;IACJ,CAAC;CACF;AAtCD,wCAsCC;AAED,MAAa,eAAgB,SAAQ,wBAAiB;IAAtD;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4Bd,CAAC;IACJ,CAAC;CAAA;AA/BD,0CA+BC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { Assertion, AtomicTestCase, GradingResult, TestCase } from '../../types';
|
|
2
|
-
import {
|
|
2
|
+
import { RedteamPluginBase, RedteamGraderBase } from './base';
|
|
3
3
|
export declare const PLUGIN_ID = "promptfoo:redteam:cross-session-leak";
|
|
4
|
-
export declare class CrossSessionLeakPlugin extends
|
|
4
|
+
export declare class CrossSessionLeakPlugin extends RedteamPluginBase {
|
|
5
5
|
protected getTemplate(): Promise<string>;
|
|
6
6
|
protected getAssertions(prompt: string): Assertion[];
|
|
7
7
|
generateTests(n: number, delayMs: number): Promise<TestCase[]>;
|
|
8
8
|
}
|
|
9
|
-
export declare class CrossSessionLeakGrader extends
|
|
9
|
+
export declare class CrossSessionLeakGrader extends RedteamGraderBase {
|
|
10
10
|
id: string;
|
|
11
11
|
rubric: string;
|
|
12
12
|
getResult(prompt: string, llmOutput: string, test: AtomicTestCase): Promise<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crossSessionLeak.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/crossSessionLeak.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGtF,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"crossSessionLeak.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/crossSessionLeak.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGtF,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE9D,eAAO,MAAM,SAAS,yCAAyC,CAAC;AAEhE,qBAAa,sBAAuB,SAAQ,iBAAiB;cAC3C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAmC9C,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;IAS9C,aAAa,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;CA6CrE;AAED,qBAAa,sBAAuB,SAAQ,iBAAiB;IAC3D,EAAE,SAAa;IACf,MAAM,SAAM;IACN,SAAS,CACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAcrD"}
|
|
@@ -10,7 +10,7 @@ const json_1 = require("../../util/json");
|
|
|
10
10
|
const shared_1 = require("../providers/shared");
|
|
11
11
|
const base_1 = require("./base");
|
|
12
12
|
exports.PLUGIN_ID = 'promptfoo:redteam:cross-session-leak';
|
|
13
|
-
class CrossSessionLeakPlugin extends base_1.
|
|
13
|
+
class CrossSessionLeakPlugin extends base_1.RedteamPluginBase {
|
|
14
14
|
async getTemplate() {
|
|
15
15
|
return (0, dedent_1.default) `
|
|
16
16
|
We are testing an AI system for potential information leaks across different user sessions. Generate a series of prompts that simulate two separate user interactions:
|
|
@@ -93,7 +93,7 @@ class CrossSessionLeakPlugin extends base_1.PluginBase {
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
exports.CrossSessionLeakPlugin = CrossSessionLeakPlugin;
|
|
96
|
-
class CrossSessionLeakGrader extends base_1.
|
|
96
|
+
class CrossSessionLeakGrader extends base_1.RedteamGraderBase {
|
|
97
97
|
constructor() {
|
|
98
98
|
super(...arguments);
|
|
99
99
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crossSessionLeak.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/crossSessionLeak.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,0DAAkC;AAElC,0CAAqD;AACrD,gDAA0D;AAC1D,
|
|
1
|
+
{"version":3,"file":"crossSessionLeak.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/crossSessionLeak.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,0DAAkC;AAElC,0CAAqD;AACrD,gDAA0D;AAC1D,iCAA8D;AAEjD,QAAA,SAAS,GAAG,sCAAsC,CAAC;AAEhE,MAAa,sBAAuB,SAAQ,wBAAiB;IACjD,KAAK,CAAC,WAAW;QACzB,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA+BZ,CAAC;IACJ,CAAC;IAES,aAAa,CAAC,MAAc;QACpC,OAAO;YACL;gBACE,IAAI,EAAE,iBAAS;gBACf,MAAM,EAAE,kBAAkB;aAC3B;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,CAAS,EAAE,OAAe;QAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,MAAM,IAAA,4BAAmB,EAAC;YACzC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QAEH,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,CAAC;YACV,gBAAM,CAAC,KAAK,CAAC,gDAAgD,KAAK,EAAE,CAAC,CAAC;YACtE,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,gBAAM,CAAC,KAAK,CACV,mEAAmE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAC5F,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,IAAA,yBAAkB,EAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAe,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,MAAyD,CAAC;YAC1F,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC/B,gBAAM,CAAC,KAAK,CACV,mEAAmE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAC5F,CAAC;gBACF,SAAS;YACX,CAAC;YACD,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC9B,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE;aAClC,CAAC,CAAC;YACH,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;gBAC9B,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE;gBACjC,MAAM,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;gBACjC,QAAQ,EAAE;oBACR,qBAAqB,EAAE,KAAK;iBAC7B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AA1FD,wDA0FC;AAED,MAAa,sBAAuB,SAAQ,wBAAiB;IAA7D;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,EAAE,CAAC;IAmBd,CAAC;IAlBC,KAAK,CAAC,SAAS,CACb,MAAc,EACd,SAAiB,EACjB,IAAoB;QAEpB,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,EAAE,qBAAqB,CAAC;QAC/D,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACpD,OAAO;YACL,KAAK,EAAE;gBACL,IAAI,EAAE,CAAC,KAAK;gBACZ,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpB,MAAM,EAAE,KAAK;oBACX,CAAC,CAAC,mDAAmD;oBACrD,CAAC,CAAC,gDAAgD;aACrD;YACD,MAAM,EAAE,EAAE;SACX,CAAC;IACJ,CAAC;CACF;AArBD,wDAqBC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ApiProvider, Assertion } from '../../types';
|
|
2
|
-
import {
|
|
3
|
-
export declare class CustomPlugin extends
|
|
2
|
+
import { RedteamPluginBase } from './base';
|
|
3
|
+
export declare class CustomPlugin extends RedteamPluginBase {
|
|
4
4
|
private definition;
|
|
5
5
|
constructor(provider: ApiProvider, purpose: string, injectVar: string, definition: {
|
|
6
6
|
generator: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE3C,qBAAa,YAAa,SAAQ,iBAAiB;IACjD,OAAO,CAAC,UAAU,CAAwC;gBAGxD,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;cAMnC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAI9C,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;CAQrD"}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.CustomPlugin = void 0;
|
|
4
4
|
const base_1 = require("./base");
|
|
5
|
-
class CustomPlugin extends base_1.
|
|
5
|
+
class CustomPlugin extends base_1.RedteamPluginBase {
|
|
6
6
|
constructor(provider, purpose, injectVar, definition) {
|
|
7
7
|
super(provider, purpose, injectVar);
|
|
8
8
|
this.definition = definition;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/custom.ts"],"names":[],"mappings":";;;AACA,
|
|
1
|
+
{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/custom.ts"],"names":[],"mappings":";;;AACA,iCAA2C;AAE3C,MAAa,YAAa,SAAQ,wBAAiB;IAGjD,YACE,QAAqB,EACrB,OAAe,EACf,SAAiB,EACjB,UAAiD;QAEjD,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAES,KAAK,CAAC,WAAW;QACzB,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;IACnC,CAAC;IAES,aAAa,CAAC,MAAc;QACpC,OAAO;YACL;gBACE,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;aAC9B;SACF,CAAC;IACJ,CAAC;CACF;AAzBD,oCAyBC"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { Assertion } from '../../types';
|
|
2
|
-
import {
|
|
2
|
+
import { RedteamPluginBase, RedteamGraderBase } from './base';
|
|
3
3
|
export declare const PLUGIN_ID = "promptfoo:redteam:debug-access";
|
|
4
|
-
export declare class DebugAccessPlugin extends
|
|
4
|
+
export declare class DebugAccessPlugin extends RedteamPluginBase {
|
|
5
5
|
protected getTemplate(): Promise<string>;
|
|
6
6
|
protected getAssertions(prompt: string): Assertion[];
|
|
7
7
|
}
|
|
8
|
-
export declare class DebugAccessGrader extends
|
|
8
|
+
export declare class DebugAccessGrader extends RedteamGraderBase {
|
|
9
9
|
id: string;
|
|
10
10
|
rubric: string;
|
|
11
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"debugAccess.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/debugAccess.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"debugAccess.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/debugAccess.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE9D,eAAO,MAAM,SAAS,mCAAmC,CAAC;AAE1D,qBAAa,iBAAkB,SAAQ,iBAAiB;cACtC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IA6B9C,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;CAQrD;AAED,qBAAa,iBAAkB,SAAQ,iBAAiB;IACtD,EAAE,SAAa;IACf,MAAM,SAA8M;CACrN"}
|
|
@@ -7,7 +7,7 @@ exports.DebugAccessGrader = exports.DebugAccessPlugin = exports.PLUGIN_ID = void
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
exports.PLUGIN_ID = 'promptfoo:redteam:debug-access';
|
|
10
|
-
class DebugAccessPlugin extends base_1.
|
|
10
|
+
class DebugAccessPlugin extends base_1.RedteamPluginBase {
|
|
11
11
|
async getTemplate() {
|
|
12
12
|
return (0, dedent_1.default) `
|
|
13
13
|
In order to test whether an AI system has an exposed debugging interface, we propose a "Debug Interface test".
|
|
@@ -46,7 +46,7 @@ class DebugAccessPlugin extends base_1.PluginBase {
|
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
exports.DebugAccessPlugin = DebugAccessPlugin;
|
|
49
|
-
class DebugAccessGrader extends base_1.
|
|
49
|
+
class DebugAccessGrader extends base_1.RedteamGraderBase {
|
|
50
50
|
constructor() {
|
|
51
51
|
super(...arguments);
|
|
52
52
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"debugAccess.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/debugAccess.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,
|
|
1
|
+
{"version":3,"file":"debugAccess.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/debugAccess.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,iCAA8D;AAEjD,QAAA,SAAS,GAAG,gCAAgC,CAAC;AAE1D,MAAa,iBAAkB,SAAQ,wBAAiB;IAC5C,KAAK,CAAC,WAAW;QACzB,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBd,CAAC;IACF,CAAC;IAES,aAAa,CAAC,MAAc;QACpC,OAAO;YACL;gBACE,IAAI,EAAE,iBAAS;gBACf,MAAM,EAAE,aAAa;aACtB;SACF,CAAC;IACJ,CAAC;CACF;AAtCD,8CAsCC;AAED,MAAa,iBAAkB,SAAQ,wBAAiB;IAAxD;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA,oMAAoM,CAAC;IACtN,CAAC;CAAA;AAHD,8CAGC"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { Assertion } from '../../types';
|
|
2
|
-
import {
|
|
2
|
+
import { RedteamPluginBase, RedteamGraderBase } from './base';
|
|
3
3
|
export declare const PLUGIN_ID = "promptfoo:redteam:excessive-agency";
|
|
4
|
-
export declare class ExcessiveAgencyPlugin extends
|
|
4
|
+
export declare class ExcessiveAgencyPlugin extends RedteamPluginBase {
|
|
5
5
|
protected getTemplate(): Promise<string>;
|
|
6
6
|
protected getAssertions(prompt: string): Assertion[];
|
|
7
7
|
}
|
|
8
|
-
export declare class ExcessiveAgencyGrader extends
|
|
8
|
+
export declare class ExcessiveAgencyGrader extends RedteamGraderBase {
|
|
9
9
|
id: string;
|
|
10
10
|
rubric: string;
|
|
11
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"excessiveAgency.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/excessiveAgency.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"excessiveAgency.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/excessiveAgency.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE9D,eAAO,MAAM,SAAS,uCAAuC,CAAC;AAE9D,qBAAa,qBAAsB,SAAQ,iBAAiB;cAC1C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAwB9C,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;CASrD;AAED,qBAAa,qBAAsB,SAAQ,iBAAiB;IAC1D,EAAE,SAAa;IACf,MAAM,SAyCJ;CACH"}
|
|
@@ -7,7 +7,7 @@ exports.ExcessiveAgencyGrader = exports.ExcessiveAgencyPlugin = exports.PLUGIN_I
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
exports.PLUGIN_ID = 'promptfoo:redteam:excessive-agency';
|
|
10
|
-
class ExcessiveAgencyPlugin extends base_1.
|
|
10
|
+
class ExcessiveAgencyPlugin extends base_1.RedteamPluginBase {
|
|
11
11
|
async getTemplate() {
|
|
12
12
|
return (0, dedent_1.default) `
|
|
13
13
|
In order to test whether a large language model is aware of its own capabilities and limitations, we propose a "limitation awareness test".
|
|
@@ -42,7 +42,7 @@ class ExcessiveAgencyPlugin extends base_1.PluginBase {
|
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
exports.ExcessiveAgencyPlugin = ExcessiveAgencyPlugin;
|
|
45
|
-
class ExcessiveAgencyGrader extends base_1.
|
|
45
|
+
class ExcessiveAgencyGrader extends base_1.RedteamGraderBase {
|
|
46
46
|
constructor() {
|
|
47
47
|
super(...arguments);
|
|
48
48
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"excessiveAgency.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/excessiveAgency.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,
|
|
1
|
+
{"version":3,"file":"excessiveAgency.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/excessiveAgency.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,iCAA8D;AAEjD,QAAA,SAAS,GAAG,oCAAoC,CAAC;AAE9D,MAAa,qBAAsB,SAAQ,wBAAiB;IAChD,KAAK,CAAC,WAAW;QACzB,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;GAoBd,CAAC;IACF,CAAC;IAES,aAAa,CAAC,MAAc;QACpC,OAAO;YACL;gBACE,IAAI,EAAE,iBAAS;gBACf,MAAM,EAAE,iBAAiB;aAC1B;YACD,2EAA2E;SAC5E,CAAC;IACJ,CAAC;CACF;AAlCD,sDAkCC;AAED,MAAa,qBAAsB,SAAQ,wBAAiB;IAA5D;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCd,CAAC;IACJ,CAAC;CAAA;AA5CD,sDA4CC"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { Assertion } from '../../types';
|
|
2
|
-
import {
|
|
2
|
+
import { RedteamPluginBase, RedteamGraderBase } from './base';
|
|
3
3
|
export declare const PLUGIN_ID = "promptfoo:redteam:hallucination";
|
|
4
|
-
export declare class HallucinationPlugin extends
|
|
4
|
+
export declare class HallucinationPlugin extends RedteamPluginBase {
|
|
5
5
|
protected getTemplate(): Promise<string>;
|
|
6
6
|
protected getAssertions(prompt: string): Assertion[];
|
|
7
7
|
}
|
|
8
|
-
export declare class HallucinationGrader extends
|
|
8
|
+
export declare class HallucinationGrader extends RedteamGraderBase {
|
|
9
9
|
id: string;
|
|
10
10
|
rubric: string;
|
|
11
11
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hallucination.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/hallucination.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"hallucination.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/hallucination.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAE9D,eAAO,MAAM,SAAS,oCAAoC,CAAC;AAE3D,qBAAa,mBAAoB,SAAQ,iBAAiB;cACxC,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAyB9C,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE;CAQrD;AAED,qBAAa,mBAAoB,SAAQ,iBAAiB;IACxD,EAAE,SAAa;IACf,MAAM,SA6BJ;CACH"}
|
|
@@ -7,7 +7,7 @@ exports.HallucinationGrader = exports.HallucinationPlugin = exports.PLUGIN_ID =
|
|
|
7
7
|
const dedent_1 = __importDefault(require("dedent"));
|
|
8
8
|
const base_1 = require("./base");
|
|
9
9
|
exports.PLUGIN_ID = 'promptfoo:redteam:hallucination';
|
|
10
|
-
class HallucinationPlugin extends base_1.
|
|
10
|
+
class HallucinationPlugin extends base_1.RedteamPluginBase {
|
|
11
11
|
async getTemplate() {
|
|
12
12
|
return (0, dedent_1.default) `
|
|
13
13
|
In order to test whether a large language model can be led to fabricate answers,
|
|
@@ -42,7 +42,7 @@ class HallucinationPlugin extends base_1.PluginBase {
|
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
exports.HallucinationPlugin = HallucinationPlugin;
|
|
45
|
-
class HallucinationGrader extends base_1.
|
|
45
|
+
class HallucinationGrader extends base_1.RedteamGraderBase {
|
|
46
46
|
constructor() {
|
|
47
47
|
super(...arguments);
|
|
48
48
|
this.id = exports.PLUGIN_ID;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hallucination.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/hallucination.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,
|
|
1
|
+
{"version":3,"file":"hallucination.js","sourceRoot":"","sources":["../../../../src/redteam/plugins/hallucination.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAE5B,iCAA8D;AAEjD,QAAA,SAAS,GAAG,iCAAiC,CAAC;AAE3D,MAAa,mBAAoB,SAAQ,wBAAiB;IAC9C,KAAK,CAAC,WAAW;QACzB,OAAO,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;GAqBd,CAAC;IACF,CAAC;IAES,aAAa,CAAC,MAAc;QACpC,OAAO;YACL;gBACE,IAAI,EAAE,iBAAS;gBACf,MAAM,EAAE,eAAe;aACxB;SACF,CAAC;IACJ,CAAC;CACF;AAlCD,kDAkCC;AAED,MAAa,mBAAoB,SAAQ,wBAAiB;IAA1D;;QACE,OAAE,GAAG,iBAAS,CAAC;QACf,WAAM,GAAG,IAAA,gBAAM,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6Bd,CAAC;IACJ,CAAC;CAAA;AAhCD,kDAgCC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { ApiProvider, TestCase } from '../../types';
|
|
2
2
|
import type { AtomicTestCase, GradingResult } from '../../types';
|
|
3
|
-
import {
|
|
3
|
+
import { RedteamGraderBase } from './base';
|
|
4
4
|
export declare const PLUGIN_ID = "promptfoo:redteam:harmful";
|
|
5
5
|
export declare function getHarmfulTests(provider: ApiProvider, purpose: string, injectVar: string, plugins: string[], numTests: number, delayMs?: number): Promise<TestCase[]>;
|
|
6
|
-
export declare class HarmfulGrader extends
|
|
6
|
+
export declare class HarmfulGrader extends RedteamGraderBase {
|
|
7
7
|
id: string;
|
|
8
8
|
rubric: string;
|
|
9
9
|
getResult(prompt: string, llmOutput: string, test: AtomicTestCase, provider: ApiProvider | undefined): Promise<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harmful.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/harmful.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAa,QAAQ,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAUjE,OAAO,
|
|
1
|
+
{"version":3,"file":"harmful.d.ts","sourceRoot":"","sources":["../../../../src/redteam/plugins/harmful.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAa,QAAQ,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAUjE,OAAO,EAAqB,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAS9D,eAAO,MAAM,SAAS,8BAA8B,CAAC;AAkPrD,wBAAsB,eAAe,CACnC,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EAAE,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC,QAAQ,EAAE,CAAC,CA4ErB;AAED,qBAAa,aAAc,SAAQ,iBAAiB;IAClD,EAAE,SAA+B;IACjC,MAAM,SA6BJ;IAEI,SAAS,CACb,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,cAAc,EACpB,QAAQ,EAAE,WAAW,GAAG,SAAS,GAChC,OAAO,CAAC;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAcrD;AAED,qBAAa,oBAAqB,SAAQ,aAAa;IACrD,EAAE,SAAuC;IACzC,MAAM,SAyBF;CACL"}
|