promptfoo 0.90.2 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +4 -4
- package/dist/src/app/assets/{index-Nu_Jrwyo.js → index-Dvv-swGm.js} +161 -161
- package/dist/src/app/assets/{index.es-Cwq4F6NP.js → index.es-DAIlgY6E.js} +1 -1
- package/dist/src/app/assets/{sync-CE7pFgME.js → sync-CABaZxhX.js} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/checkNodeVersion.d.ts.map +1 -1
- package/dist/src/checkNodeVersion.js +4 -37
- package/dist/src/checkNodeVersion.js.map +1 -1
- package/dist/src/commands/eval.d.ts +1 -1
- package/dist/src/commands/eval.d.ts.map +1 -1
- package/dist/src/commands/eval.js.map +1 -1
- package/dist/src/commands/list.d.ts.map +1 -1
- package/dist/src/commands/list.js +2 -1
- package/dist/src/commands/list.js.map +1 -1
- package/dist/src/database/tables.d.ts +5 -0
- package/dist/src/database/tables.d.ts.map +1 -1
- package/dist/src/envars.d.ts +1 -0
- package/dist/src/envars.d.ts.map +1 -1
- package/dist/src/envars.js.map +1 -1
- package/dist/src/evaluator.d.ts.map +1 -1
- package/dist/src/evaluator.js +12 -2
- package/dist/src/evaluator.js.map +1 -1
- package/dist/src/index.d.ts +9 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +11 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/main.js +9 -9
- package/dist/src/main.js.map +1 -1
- package/dist/src/models/prompt.d.ts +6 -0
- package/dist/src/models/prompt.d.ts.map +1 -0
- package/dist/src/models/prompt.js +10 -0
- package/dist/src/models/prompt.js.map +1 -0
- package/dist/src/prompts/index.d.ts +1 -1
- package/dist/src/prompts/index.d.ts.map +1 -1
- package/dist/src/prompts/index.js.map +1 -1
- package/dist/src/providers/browser.d.ts.map +1 -1
- package/dist/src/providers/browser.js +25 -1
- package/dist/src/providers/browser.js.map +1 -1
- package/dist/src/providers/golangCompletion.d.ts.map +1 -1
- package/dist/src/providers/golangCompletion.js +2 -1
- package/dist/src/providers/golangCompletion.js.map +1 -1
- package/dist/src/providers/openai.js +1 -1
- package/dist/src/providers/pythonCompletion.d.ts.map +1 -1
- package/dist/src/providers/pythonCompletion.js +2 -1
- package/dist/src/providers/pythonCompletion.js.map +1 -1
- package/dist/src/providers.d.ts.map +1 -1
- package/dist/src/providers.js +23 -5
- package/dist/src/providers.js.map +1 -1
- package/dist/src/redteam/commands/generate.d.ts +2 -2
- package/dist/src/redteam/commands/generate.d.ts.map +1 -1
- package/dist/src/redteam/commands/generate.js +3 -3
- package/dist/src/redteam/commands/generate.js.map +1 -1
- package/dist/src/redteam/commands/init.d.ts.map +1 -1
- package/dist/src/redteam/commands/init.js +3 -4
- package/dist/src/redteam/commands/init.js.map +1 -1
- package/dist/src/redteam/commands/report.d.ts +3 -0
- package/dist/src/redteam/commands/report.d.ts.map +1 -0
- package/dist/src/redteam/commands/report.js +31 -0
- package/dist/src/redteam/commands/report.js.map +1 -0
- package/dist/src/redteam/commands/run.d.ts +3 -0
- package/dist/src/redteam/commands/run.d.ts.map +1 -0
- package/dist/src/redteam/commands/run.js +137 -0
- package/dist/src/redteam/commands/run.js.map +1 -0
- package/dist/src/redteam/graders.d.ts +2 -2
- package/dist/src/redteam/graders.d.ts.map +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.d.ts +2 -2
- package/dist/src/redteam/plugins/asciiSmuggling.d.ts.map +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.js +1 -1
- package/dist/src/redteam/plugins/asciiSmuggling.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts +3 -3
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +7 -7
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/bfla.d.ts +2 -2
- package/dist/src/redteam/plugins/bfla.d.ts.map +1 -1
- package/dist/src/redteam/plugins/bfla.js +1 -1
- package/dist/src/redteam/plugins/bfla.js.map +1 -1
- package/dist/src/redteam/plugins/bola.d.ts +2 -2
- package/dist/src/redteam/plugins/bola.d.ts.map +1 -1
- package/dist/src/redteam/plugins/bola.js +1 -1
- package/dist/src/redteam/plugins/bola.js.map +1 -1
- package/dist/src/redteam/plugins/competitors.d.ts +2 -2
- package/dist/src/redteam/plugins/competitors.d.ts.map +1 -1
- package/dist/src/redteam/plugins/competitors.js +1 -1
- package/dist/src/redteam/plugins/competitors.js.map +1 -1
- package/dist/src/redteam/plugins/contracts.d.ts +3 -3
- package/dist/src/redteam/plugins/contracts.d.ts.map +1 -1
- package/dist/src/redteam/plugins/contracts.js +2 -2
- package/dist/src/redteam/plugins/contracts.js.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts +3 -3
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js +2 -2
- package/dist/src/redteam/plugins/crossSessionLeak.js.map +1 -1
- package/dist/src/redteam/plugins/custom.d.ts +2 -2
- package/dist/src/redteam/plugins/custom.d.ts.map +1 -1
- package/dist/src/redteam/plugins/custom.js +1 -1
- package/dist/src/redteam/plugins/custom.js.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.d.ts +3 -3
- package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.js +2 -2
- package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.d.ts +3 -3
- package/dist/src/redteam/plugins/excessiveAgency.d.ts.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.js +2 -2
- package/dist/src/redteam/plugins/excessiveAgency.js.map +1 -1
- package/dist/src/redteam/plugins/hallucination.d.ts +3 -3
- package/dist/src/redteam/plugins/hallucination.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hallucination.js +2 -2
- package/dist/src/redteam/plugins/hallucination.js.map +1 -1
- package/dist/src/redteam/plugins/harmful.d.ts +2 -2
- package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful.js +4 -7
- package/dist/src/redteam/plugins/harmful.js.map +1 -1
- package/dist/src/redteam/plugins/hijacking.d.ts +2 -2
- package/dist/src/redteam/plugins/hijacking.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hijacking.js +1 -1
- package/dist/src/redteam/plugins/hijacking.js.map +1 -1
- package/dist/src/redteam/plugins/imitation.d.ts +3 -3
- package/dist/src/redteam/plugins/imitation.d.ts.map +1 -1
- package/dist/src/redteam/plugins/imitation.js +2 -2
- package/dist/src/redteam/plugins/imitation.js.map +1 -1
- package/dist/src/redteam/plugins/indirectPromptInjection.d.ts +3 -19
- package/dist/src/redteam/plugins/indirectPromptInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/indirectPromptInjection.js +2 -73
- package/dist/src/redteam/plugins/indirectPromptInjection.js.map +1 -1
- package/dist/src/redteam/plugins/overreliance.d.ts +3 -3
- package/dist/src/redteam/plugins/overreliance.d.ts.map +1 -1
- package/dist/src/redteam/plugins/overreliance.js +2 -2
- package/dist/src/redteam/plugins/overreliance.js.map +1 -1
- package/dist/src/redteam/plugins/pii.d.ts +2 -2
- package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
- package/dist/src/redteam/plugins/pii.js +3 -4
- package/dist/src/redteam/plugins/pii.js.map +1 -1
- package/dist/src/redteam/plugins/policy.d.ts +3 -3
- package/dist/src/redteam/plugins/policy.d.ts.map +1 -1
- package/dist/src/redteam/plugins/policy.js +2 -2
- package/dist/src/redteam/plugins/policy.js.map +1 -1
- package/dist/src/redteam/plugins/politics.d.ts +3 -3
- package/dist/src/redteam/plugins/politics.d.ts.map +1 -1
- package/dist/src/redteam/plugins/politics.js +2 -2
- package/dist/src/redteam/plugins/politics.js.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.d.ts +3 -3
- package/dist/src/redteam/plugins/promptExtraction.d.ts.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.js +2 -2
- package/dist/src/redteam/plugins/promptExtraction.js.map +1 -1
- package/dist/src/redteam/plugins/rbac.d.ts +3 -3
- package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
- package/dist/src/redteam/plugins/rbac.js +2 -2
- package/dist/src/redteam/plugins/rbac.js.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.d.ts +3 -3
- package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.js +2 -2
- package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.d.ts +3 -3
- package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.js +2 -2
- package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
- package/dist/src/redteam/plugins/ssrf.d.ts +2 -2
- package/dist/src/redteam/plugins/ssrf.d.ts.map +1 -1
- package/dist/src/redteam/plugins/ssrf.js +1 -1
- package/dist/src/redteam/plugins/ssrf.js.map +1 -1
- package/dist/src/redteam/providers/constants.d.ts +1 -1
- package/dist/src/redteam/providers/constants.d.ts.map +1 -1
- package/dist/src/redteam/providers/constants.js +1 -1
- package/dist/src/redteam/providers/constants.js.map +1 -1
- package/dist/src/server/server.d.ts +2 -1
- package/dist/src/server/server.d.ts.map +1 -1
- package/dist/src/server/server.js +15 -7
- package/dist/src/server/server.js.map +1 -1
- package/dist/src/share.d.ts.map +1 -1
- package/dist/src/share.js +16 -1
- package/dist/src/share.js.map +1 -1
- package/dist/src/types/index.d.ts +135 -0
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js +1 -0
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/util/createHash.d.ts +2 -0
- package/dist/src/util/createHash.d.ts.map +1 -0
- package/dist/src/util/createHash.js +8 -0
- package/dist/src/util/createHash.js.map +1 -0
- package/dist/src/util/index.d.ts +4 -2
- package/dist/src/util/index.d.ts.map +1 -1
- package/dist/src/util/index.js +16 -18
- package/dist/src/util/index.js.map +1 -1
- package/dist/src/util/transform.d.ts +5 -1
- package/dist/src/util/transform.d.ts.map +1 -1
- package/dist/src/util/transform.js +13 -7
- package/dist/src/util/transform.js.map +1 -1
- package/dist/src/validators/redteam.d.ts.map +1 -1
- package/dist/src/validators/redteam.js +70 -33
- package/dist/src/validators/redteam.js.map +1 -1
- package/dist/test/checkNodeVersion.test.js +11 -58
- package/dist/test/checkNodeVersion.test.js.map +1 -1
- package/dist/test/evaluator.test.js +53 -0
- package/dist/test/evaluator.test.js.map +1 -1
- package/dist/test/index.test.js +170 -8
- package/dist/test/index.test.js.map +1 -1
- package/dist/test/providers.test.js +36 -1
- package/dist/test/providers.test.js.map +1 -1
- package/dist/test/redteam/plugins/base.test.js +6 -6
- package/dist/test/redteam/plugins/base.test.js.map +1 -1
- package/dist/test/redteam/validators.test.js +165 -7
- package/dist/test/redteam/validators.test.js.map +1 -1
- package/dist/test/util.transform.test.js +17 -0
- package/dist/test/util.transform.test.js.map +1 -1
- package/package.json +4 -4
- package/dist/src/commands/version.d.ts +0 -3
- package/dist/src/commands/version.d.ts.map +0 -1
- package/dist/src/commands/version.js +0 -18
- package/dist/src/commands/version.js.map +0 -1
|
@@ -3,6 +3,10 @@ export type TransformContext = {
|
|
|
3
3
|
vars?: Vars;
|
|
4
4
|
prompt: Partial<Prompt>;
|
|
5
5
|
};
|
|
6
|
+
export declare enum TransformInputType {
|
|
7
|
+
OUTPUT = "output",
|
|
8
|
+
VARS = "vars"
|
|
9
|
+
}
|
|
6
10
|
/**
|
|
7
11
|
* Transforms the output using a specified function or file.
|
|
8
12
|
*
|
|
@@ -18,5 +22,5 @@ export type TransformContext = {
|
|
|
18
22
|
* @throws Error if the file format is unsupported or if the transform function
|
|
19
23
|
* doesn't return a value (unless validateReturn is false).
|
|
20
24
|
*/
|
|
21
|
-
export declare function transform(codeOrFilepath: string, transformInput: string | object | undefined, context: TransformContext, validateReturn?: boolean): Promise<string | object | undefined>;
|
|
25
|
+
export declare function transform(codeOrFilepath: string, transformInput: string | object | undefined, context: TransformContext, validateReturn?: boolean, inputType?: TransformInputType): Promise<string | object | undefined>;
|
|
22
26
|
//# sourceMappingURL=transform.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../../src/util/transform.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../../src/util/transform.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,CAAC,EAAE,IAAI,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;CACzB,CAAC;AAEF,oBAAY,kBAAkB;IAC5B,MAAM,WAAW;IACjB,IAAI,SAAS;CACd;AA+FD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,cAAc,EAAE,MAAM,EACtB,cAAc,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,EAC3C,OAAO,EAAE,gBAAgB,EACzB,cAAc,GAAE,OAAc,EAC9B,SAAS,GAAE,kBAA8C,GACxD,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAUtC"}
|
|
@@ -3,12 +3,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.TransformInputType = void 0;
|
|
6
7
|
exports.transform = transform;
|
|
7
8
|
const path_1 = __importDefault(require("path"));
|
|
8
|
-
const _1 = require(".");
|
|
9
9
|
const cliState_1 = __importDefault(require("../cliState"));
|
|
10
10
|
const esm_1 = require("../esm");
|
|
11
11
|
const pythonUtils_1 = require("../python/pythonUtils");
|
|
12
|
+
const _1 = require("./");
|
|
13
|
+
var TransformInputType;
|
|
14
|
+
(function (TransformInputType) {
|
|
15
|
+
TransformInputType["OUTPUT"] = "output";
|
|
16
|
+
TransformInputType["VARS"] = "vars";
|
|
17
|
+
})(TransformInputType || (exports.TransformInputType = TransformInputType = {}));
|
|
12
18
|
/**
|
|
13
19
|
* Parses a file path string to extract the file path and function name.
|
|
14
20
|
* @param filePath - The file path string, potentially including a function name.
|
|
@@ -71,19 +77,19 @@ async function getFileTransformFunction(filePath) {
|
|
|
71
77
|
* @param code - The JavaScript code to convert into a function.
|
|
72
78
|
* @returns A Function created from the provided code.
|
|
73
79
|
*/
|
|
74
|
-
function getInlineTransformFunction(code) {
|
|
75
|
-
return new Function(
|
|
80
|
+
function getInlineTransformFunction(code, inputType) {
|
|
81
|
+
return new Function(inputType, 'context', code.includes('\n') ? code : `return ${code}`);
|
|
76
82
|
}
|
|
77
83
|
/**
|
|
78
84
|
* Determines and retrieves the appropriate transform function based on the input.
|
|
79
85
|
* @param codeOrFilepath - Either inline code or a file path starting with 'file://'.
|
|
80
86
|
* @returns A Promise resolving to the appropriate transform function.
|
|
81
87
|
*/
|
|
82
|
-
async function getTransformFunction(codeOrFilepath) {
|
|
88
|
+
async function getTransformFunction(codeOrFilepath, inputType) {
|
|
83
89
|
if (codeOrFilepath.startsWith('file://')) {
|
|
84
90
|
return getFileTransformFunction(codeOrFilepath);
|
|
85
91
|
}
|
|
86
|
-
return getInlineTransformFunction(codeOrFilepath);
|
|
92
|
+
return getInlineTransformFunction(codeOrFilepath, inputType);
|
|
87
93
|
}
|
|
88
94
|
/**
|
|
89
95
|
* Transforms the output using a specified function or file.
|
|
@@ -100,8 +106,8 @@ async function getTransformFunction(codeOrFilepath) {
|
|
|
100
106
|
* @throws Error if the file format is unsupported or if the transform function
|
|
101
107
|
* doesn't return a value (unless validateReturn is false).
|
|
102
108
|
*/
|
|
103
|
-
async function transform(codeOrFilepath, transformInput, context, validateReturn = true) {
|
|
104
|
-
const postprocessFn = await getTransformFunction(codeOrFilepath);
|
|
109
|
+
async function transform(codeOrFilepath, transformInput, context, validateReturn = true, inputType = TransformInputType.OUTPUT) {
|
|
110
|
+
const postprocessFn = await getTransformFunction(codeOrFilepath, inputType);
|
|
105
111
|
const ret = await Promise.resolve(postprocessFn(transformInput, context));
|
|
106
112
|
if (validateReturn && ret == null) {
|
|
107
113
|
throw new Error(`Transform function did not return a value\n\n${codeOrFilepath}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transform.js","sourceRoot":"","sources":["../../../src/util/transform.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"transform.js","sourceRoot":"","sources":["../../../src/util/transform.ts"],"names":[],"mappings":";;;;;;AA6HA,8BAgBC;AA7ID,gDAAwB;AACxB,2DAAmC;AACnC,gCAAsC;AACtC,uDAAkD;AAElD,yBAAsC;AAOtC,IAAY,kBAGX;AAHD,WAAY,kBAAkB;IAC5B,uCAAiB,CAAA;IACjB,mCAAa,CAAA;AACf,CAAC,EAHW,kBAAkB,kCAAlB,kBAAkB,QAG7B;AAED;;;;GAIG;AACH,SAAS,4BAA4B,CAAC,QAAgB;IACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;AAC3E,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,8BAA8B,CAC3C,QAAgB,EAChB,YAAqB;IAErB,MAAM,cAAc,GAAG,MAAM,IAAA,kBAAY,EAAC,QAAQ,CAAC,CAAC;IAEpD,IAAI,YAAY,IAAI,OAAO,cAAc,CAAC,YAAY,CAAC,KAAK,UAAU,EAAE,CAAC;QACvE,OAAO,cAAc,CAAC,YAAY,CAAC,CAAC;IACtC,CAAC;SAAM,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE,CAAC;QAChD,OAAO,cAAc,CAAC;IACxB,CAAC;SAAM,IAAI,cAAc,CAAC,OAAO,IAAI,OAAO,cAAc,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAClF,OAAO,cAAc,CAAC,OAAO,CAAC;IAChC,CAAC;IACD,MAAM,IAAI,KAAK,CACb,aAAa,QAAQ,mGAAmG,YAAY,GAAG,CACxI,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,0BAA0B,CACjC,QAAgB,EAChB,eAAuB,eAAe;IAEtC,OAAO,KAAK,EAAE,MAAc,EAAE,OAAuB,EAAE,EAAE;QACvD,OAAO,IAAA,uBAAS,EAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,wBAAwB,CAAC,QAAgB;IACtD,MAAM,CAAC,cAAc,EAAE,YAAY,CAAC,GAAG,4BAA4B,CACjE,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CACjC,CAAC;IACF,MAAM,QAAQ,GAAG,cAAI,CAAC,IAAI,CAAC,kBAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;IACpE,IAAI,IAAA,mBAAgB,EAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,OAAO,8BAA8B,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAChE,CAAC;SAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,0BAA0B,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,cAAc,EAAE,CAAC,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAS,0BAA0B,CAAC,IAAY,EAAE,SAA6B;IAC7E,OAAO,IAAI,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,oBAAoB,CACjC,cAAsB,EACtB,SAA6B;IAE7B,IAAI,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,OAAO,wBAAwB,CAAC,cAAc,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,0BAA0B,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACI,KAAK,UAAU,SAAS,CAC7B,cAAsB,EACtB,cAA2C,EAC3C,OAAyB,EACzB,iBAA0B,IAAI,EAC9B,YAAgC,kBAAkB,CAAC,MAAM;IAEzD,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAE5E,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;IAE1E,IAAI,cAAc,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,gDAAgD,cAAc,EAAE,CAAC,CAAC;IACpF,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/validators/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/validators/redteam.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAiBxB,OAAO,KAAK,EAAE,iBAAiB,EAAuB,MAAM,kBAAkB,CAAC;AAmB/E;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;IAU9B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;IAMhC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmCvC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAoK5B,CAAC"}
|
|
@@ -133,39 +133,78 @@ exports.RedteamConfigSchema = zod_1.z
|
|
|
133
133
|
})
|
|
134
134
|
.transform((data) => {
|
|
135
135
|
const pluginMap = new Map();
|
|
136
|
-
|
|
136
|
+
const strategySet = new Set();
|
|
137
|
+
const addPlugin = (id, config, numTests) => {
|
|
138
|
+
const key = `${id}:${JSON.stringify(config)}`;
|
|
139
|
+
const pluginObject = { id };
|
|
140
|
+
if (numTests !== undefined || data.numTests !== undefined) {
|
|
141
|
+
pluginObject.numTests = numTests ?? data.numTests;
|
|
142
|
+
}
|
|
143
|
+
if (config !== undefined) {
|
|
144
|
+
pluginObject.config = config;
|
|
145
|
+
}
|
|
146
|
+
pluginMap.set(key, pluginObject);
|
|
147
|
+
};
|
|
148
|
+
const expandCollection = (collection, config, numTests) => {
|
|
149
|
+
(Array.isArray(collection) ? collection : Array.from(collection)).forEach((item) => {
|
|
150
|
+
// Only add the plugin if it doesn't already exist or if the existing one has undefined numTests
|
|
151
|
+
const existingPlugin = pluginMap.get(`${item}:${JSON.stringify(config)}`);
|
|
152
|
+
if (!existingPlugin || existingPlugin.numTests === undefined) {
|
|
153
|
+
addPlugin(item, config, numTests);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
};
|
|
157
|
+
const handleCollectionExpansion = (id, config, numTests) => {
|
|
158
|
+
if (id === 'harmful') {
|
|
159
|
+
expandCollection(Object.keys(constants_1.HARM_PLUGINS), config, numTests);
|
|
160
|
+
}
|
|
161
|
+
else if (id === 'pii') {
|
|
162
|
+
expandCollection([...constants_1.PII_PLUGINS], config, numTests);
|
|
163
|
+
}
|
|
164
|
+
else if (id === 'default') {
|
|
165
|
+
expandCollection([...constants_1.DEFAULT_PLUGINS], config, numTests);
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
const handlePlugin = (plugin) => {
|
|
137
169
|
const pluginObj = typeof plugin === 'string'
|
|
138
170
|
? { id: plugin, numTests: data.numTests, config: undefined }
|
|
139
171
|
: { ...plugin, numTests: plugin.numTests ?? data.numTests };
|
|
140
|
-
if (pluginObj.id
|
|
141
|
-
Object.
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if (!pluginMap.has(key)) {
|
|
152
|
-
pluginMap.set(key, { id, numTests: pluginObj.numTests, config: pluginObj.config });
|
|
153
|
-
}
|
|
172
|
+
if (constants_1.ALIASED_PLUGIN_MAPPINGS[pluginObj.id]) {
|
|
173
|
+
Object.values(constants_1.ALIASED_PLUGIN_MAPPINGS[pluginObj.id]).forEach(({ plugins, strategies }) => {
|
|
174
|
+
plugins.forEach((id) => {
|
|
175
|
+
if (constants_1.COLLECTIONS.includes(id)) {
|
|
176
|
+
handleCollectionExpansion(id, pluginObj.config, pluginObj.numTests);
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
addPlugin(id, pluginObj.config, pluginObj.numTests);
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
strategies.forEach((strategy) => strategySet.add(strategy));
|
|
154
183
|
});
|
|
155
184
|
}
|
|
156
|
-
else if (pluginObj.id
|
|
157
|
-
|
|
158
|
-
const key = `${id}:${JSON.stringify(pluginObj.config)}`;
|
|
159
|
-
if (!pluginMap.has(key)) {
|
|
160
|
-
pluginMap.set(key, { id, numTests: pluginObj.numTests, config: pluginObj.config });
|
|
161
|
-
}
|
|
162
|
-
});
|
|
185
|
+
else if (constants_1.COLLECTIONS.includes(pluginObj.id)) {
|
|
186
|
+
handleCollectionExpansion(pluginObj.id, pluginObj.config, pluginObj.numTests);
|
|
163
187
|
}
|
|
164
188
|
else {
|
|
165
|
-
const
|
|
166
|
-
|
|
189
|
+
const mapping = Object.entries(constants_1.ALIASED_PLUGIN_MAPPINGS).find(([, value]) => Object.keys(value).includes(pluginObj.id));
|
|
190
|
+
if (mapping) {
|
|
191
|
+
const [, aliasedMapping] = mapping;
|
|
192
|
+
aliasedMapping[pluginObj.id].plugins.forEach((id) => {
|
|
193
|
+
if (constants_1.COLLECTIONS.includes(id)) {
|
|
194
|
+
handleCollectionExpansion(id, pluginObj.config, pluginObj.numTests);
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
addPlugin(id, pluginObj.config, pluginObj.numTests);
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
aliasedMapping[pluginObj.id].strategies.forEach((strategy) => strategySet.add(strategy));
|
|
201
|
+
}
|
|
202
|
+
else {
|
|
203
|
+
addPlugin(pluginObj.id, pluginObj.config, pluginObj.numTests);
|
|
204
|
+
}
|
|
167
205
|
}
|
|
168
|
-
}
|
|
206
|
+
};
|
|
207
|
+
data.plugins.forEach(handlePlugin);
|
|
169
208
|
const uniquePlugins = Array.from(pluginMap.values())
|
|
170
209
|
.filter((plugin) => !constants_1.COLLECTIONS.includes(plugin.id))
|
|
171
210
|
.sort((a, b) => {
|
|
@@ -174,19 +213,17 @@ exports.RedteamConfigSchema = zod_1.z
|
|
|
174
213
|
}
|
|
175
214
|
return JSON.stringify(a.config || {}).localeCompare(JSON.stringify(b.config || {}));
|
|
176
215
|
});
|
|
177
|
-
const strategies = data.strategies
|
|
178
|
-
?.map((strategy) => {
|
|
216
|
+
const strategies = Array.from(new Set([...(data.strategies || []), ...Array.from(strategySet)].flatMap((strategy) => {
|
|
179
217
|
if (typeof strategy === 'string') {
|
|
180
218
|
if (strategy === 'basic') {
|
|
181
219
|
return [];
|
|
182
220
|
}
|
|
183
|
-
return strategy === 'default'
|
|
184
|
-
? constants_1.DEFAULT_STRATEGIES.map((id) => ({ id }))
|
|
185
|
-
: { id: strategy };
|
|
221
|
+
return strategy === 'default' ? constants_1.DEFAULT_STRATEGIES.map((id) => id) : [strategy];
|
|
186
222
|
}
|
|
187
|
-
return strategy;
|
|
188
|
-
})
|
|
189
|
-
.
|
|
223
|
+
return [strategy.id];
|
|
224
|
+
})))
|
|
225
|
+
.map((id) => ({ id }))
|
|
226
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
190
227
|
return {
|
|
191
228
|
numTests: data.numTests,
|
|
192
229
|
plugins: uniquePlugins,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/validators/redteam.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,6BAAwB;AACxB,
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/validators/redteam.ts"],"names":[],"mappings":";;;;;;AAAA,oDAA4B;AAC5B,6BAAwB;AACxB,oDAe8B;AAE9B,uDAAyD;AAEzD;;GAEG;AACH,MAAM,yBAAyB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,EAAE,EAAE,OAAC;SACF,IAAI,CAAC,CAAC,GAAG,uBAAmB,EAAE,GAAG,2BAAe,CAA0B,CAAC;SAC3E,QAAQ,CAAC,oBAAoB,CAAC;IACjC,QAAQ,EAAE,OAAC;SACR,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,OAAO,CAAC,wCAA4B,CAAC;SACrC,QAAQ,CAAC,6CAA6C,CAAC;IAC1D,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CACnF,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,mBAAmB,GAAG,OAAC,CAAC,KAAK,CAAC;IACzC,OAAC;SACE,KAAK,CAAC;QACL,OAAC,CAAC,IAAI,CAAC,CAAC,GAAG,uBAAmB,EAAE,GAAG,2BAAe,CAA0B,CAAC;QAC7E,OAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YACxD,OAAO,EAAE,2EAA2E;SACrF,CAAC;KACH,CAAC;SACD,QAAQ,CAAC,6CAA6C,CAAC;IAC1D,yBAAyB;CAC1B,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,qBAAqB,GAAG,OAAC,CAAC,KAAK,CAAC;IAC3C,OAAC,CAAC,IAAI,CAAC,0BAAkD,CAAC,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IAC3F,OAAC,CAAC,MAAM,CAAC;QACP,EAAE,EAAE,OAAC,CAAC,IAAI,CAAC,0BAAkD,CAAC,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAC/F,MAAM,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;KACrF,CAAC;CACH,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,4BAA4B,GAAG,OAAC,CAAC,MAAM,CAAC;IACnD,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IACrD,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;IACxE,aAAa,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAC7E,iBAAiB,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;IAC3F,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACvE,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IAC/D,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IACzE,cAAc,EAAE,OAAC;SACd,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,wCAAwC,CAAC;IACrD,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACxF,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC1D,OAAO,EAAE,OAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IACjF,UAAU,EAAE,OAAC;SACV,KAAK,CAAC,OAAC,CAAC,IAAI,CAAC,8BAAwE,CAAC,CAAC;SACvF,QAAQ,EAAE;SACV,QAAQ,CAAC,+BAA+B,CAAC;IAC5C,aAAa,EAAE,OAAC;SACb,KAAK,CAAC,OAAC,CAAC,IAAI,CAAC,iCAA2E,CAAC,CAAC;SAC1F,QAAQ,EAAE;SACV,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC3D,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC5E,UAAU,EAAE,OAAC,CAAC,KAAK,CAAC,6BAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IACnF,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IAC1D,KAAK,EAAE,OAAC;SACL,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,gDAAgD,CAAC;CAC9D,CAAC,CAAC;AAEH;;GAEG;AACU,QAAA,mBAAmB,GAAG,OAAC;KACjC,MAAM,CAAC;IACN,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,yIAAyI,CAC1I;IACH,OAAO,EAAE,OAAC;SACP,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,0DAA0D,CAAC;IACvE,QAAQ,EAAE,OAAC;SACR,IAAI,CAAC,GAAG,EAAE,CAAC,0BAAc,CAAC;SAC1B,QAAQ,EAAE;SACV,QAAQ,CAAC,iDAAiD,CAAC;IAC9D,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACxF,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;IACzF,OAAO,EAAE,OAAC;SACP,KAAK,CAAC,2BAAmB,CAAC;SAC1B,QAAQ,CAAC,uCAAuC,CAAC;SACjD,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC;IACvB,UAAU,EAAE,OAAC;SACV,KAAK,CAAC,6BAAqB,CAAC;SAC5B,QAAQ,CACP,IAAA,gBAAM,EAAA;;sBAEQ,8BAAkB,CAAC,IAAI,CAAC,IAAI,CAAC;mBAChC,0BAAc,CAAC,IAAI,CAAC,IAAI,CAAC;SACnC,CACF;SACA,QAAQ,EAAE;SACV,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC;IACvB,cAAc,EAAE,OAAC;SACd,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,wCAAwC,CAAC;IACrD,KAAK,EAAE,OAAC;SACL,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,gDAAgD,CAAC;CAC9D,CAAC;KACD,SAAS,CAAC,CAAC,IAAI,EAAqB,EAAE;IACrC,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAC;IACzD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAY,CAAC;IAExC,MAAM,SAAS,GAAG,CAAC,EAAU,EAAE,MAAW,EAAE,QAA4B,EAAE,EAAE;QAC1E,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,YAAY,GAAwB,EAAE,EAAE,EAAE,CAAC;QACjD,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC1D,YAAY,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;QACpD,CAAC;QACD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC;QAC/B,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACnC,CAAC,CAAC;IAEF,MAAM,gBAAgB,GAAG,CACvB,UAA0C,EAC1C,MAAW,EACX,QAA4B,EAC5B,EAAE;QACF,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACjF,gGAAgG;YAChG,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1E,IAAI,CAAC,cAAc,IAAI,cAAc,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC7D,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,yBAAyB,GAAG,CAAC,EAAU,EAAE,MAAW,EAAE,QAA4B,EAAE,EAAE;QAC1F,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAY,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChE,CAAC;aAAM,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;YACxB,gBAAgB,CAAC,CAAC,GAAG,uBAAW,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YAC5B,gBAAgB,CAAC,CAAC,GAAG,2BAAuB,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACnE,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,CAAC,MAAoC,EAAE,EAAE;QAC5D,MAAM,SAAS,GACb,OAAO,MAAM,KAAK,QAAQ;YACxB,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE;YAC5D,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEhE,IAAI,mCAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,mCAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE;gBACvF,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;oBACrB,IAAI,uBAAW,CAAC,QAAQ,CAAC,EAAS,CAAC,EAAE,CAAC;wBACpC,yBAAyB,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;oBACtE,CAAC;yBAAM,CAAC;wBACN,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;oBACtD,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,UAAU,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,QAAoB,CAAC,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,uBAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAS,CAAC,EAAE,CAAC;YACrD,yBAAyB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QAChF,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,mCAAuB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CACzE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAC1C,CAAC;YACF,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,EAAE,cAAc,CAAC,GAAG,OAAO,CAAC;gBACnC,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;oBAClD,IAAI,uBAAW,CAAC,QAAQ,CAAC,EAAS,CAAC,EAAE,CAAC;wBACpC,yBAAyB,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;oBACtE,CAAC;yBAAM,CAAC;wBACN,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;oBACtD,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC3D,WAAW,CAAC,GAAG,CAAC,QAAoB,CAAC,CACtC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEnC,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;SACjD,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,uBAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAkC,CAAC,CAAC;SACpF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YAClB,OAAO,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEL,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAC3B,IAAI,GAAG,CACL,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC5E,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,8BAAkB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAClF,CAAC;QACD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACvB,CAAC,CAAC,CACH,CACF;SACE,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;SACrB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5C,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,aAAa;QACtB,UAAU;QACV,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7C,CAAC;AACJ,CAAC,CAAC,CAAC;AAEL,sDAAsD;AACtD,6DAA6D;AAC7D,SAAS,MAAM,KAAqB,CAAC;AAGrC,MAAM,EAA6E,CAAC;AACpF,2BAA2B;AAC3B,+FAA+F"}
|
|
@@ -1,42 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
4
|
};
|
|
28
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
6
|
const chalk_1 = __importDefault(require("chalk"));
|
|
30
|
-
const fs = __importStar(require("fs"));
|
|
31
|
-
const path = __importStar(require("path"));
|
|
32
7
|
const checkNodeVersion_1 = require("../src/checkNodeVersion");
|
|
33
8
|
const logger_1 = __importDefault(require("../src/logger"));
|
|
34
|
-
jest.mock('
|
|
9
|
+
jest.mock('../package.json', () => ({
|
|
10
|
+
engines: { node: '>=18.0.1' },
|
|
11
|
+
}));
|
|
35
12
|
jest.mock('../src/logger');
|
|
36
|
-
jest.mock('path');
|
|
37
|
-
const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {
|
|
38
|
-
throw new Error('process.exit called');
|
|
39
|
-
});
|
|
40
13
|
const setNodeVersion = (version) => {
|
|
41
14
|
Object.defineProperty(process, 'version', {
|
|
42
15
|
value: version,
|
|
@@ -45,41 +18,21 @@ const setNodeVersion = (version) => {
|
|
|
45
18
|
};
|
|
46
19
|
describe('checkNodeVersion', () => {
|
|
47
20
|
const originalProcessVersion = process.version;
|
|
48
|
-
beforeEach(() => {
|
|
49
|
-
jest.resetAllMocks();
|
|
50
|
-
jest.mocked(path.resolve).mockImplementation(() => 'mocked/path/to/package.json');
|
|
51
|
-
jest.mocked(fs.readFileSync).mockImplementation(() => JSON.stringify({
|
|
52
|
-
engines: { node: '>=18.0.0' },
|
|
53
|
-
}));
|
|
54
|
-
});
|
|
55
21
|
afterEach(() => {
|
|
56
22
|
setNodeVersion(originalProcessVersion);
|
|
57
23
|
});
|
|
58
|
-
|
|
59
|
-
|
|
24
|
+
it('should handle version strings correctly and throw if required version is not met', () => {
|
|
25
|
+
setNodeVersion('v18.0.0');
|
|
26
|
+
expect(() => (0, checkNodeVersion_1.checkNodeVersion)()).toThrow('You are using Node.js 18.0.0. This version is not supported. Please use Node.js >=18.0.1.');
|
|
60
27
|
});
|
|
61
|
-
it('should not
|
|
62
|
-
setNodeVersion('v18.
|
|
28
|
+
it('should not throw if Node.js version is supported', () => {
|
|
29
|
+
setNodeVersion('v18.0.1');
|
|
63
30
|
expect(() => (0, checkNodeVersion_1.checkNodeVersion)()).not.toThrow();
|
|
64
|
-
expect(logger_1.default.warn).not.toHaveBeenCalled();
|
|
65
|
-
});
|
|
66
|
-
it('should log a warning and exit if Node.js version is too low', () => {
|
|
67
|
-
setNodeVersion('v16.10.0');
|
|
68
|
-
expect((0, checkNodeVersion_1.checkNodeVersion)()).toBeUndefined();
|
|
69
|
-
expect(logger_1.default.warn).toHaveBeenCalledWith(chalk_1.default.yellow('You are using Node.js 16.10.0. This version is not supported. Please use Node.js >=18.0.0.'));
|
|
70
31
|
});
|
|
71
32
|
it('should log a warning if Node.js version format is unexpected', () => {
|
|
72
|
-
setNodeVersion('
|
|
73
|
-
|
|
74
|
-
expect(logger_1.default.warn).toHaveBeenCalledWith(chalk_1.default.yellow('Unexpected Node.js version format:
|
|
75
|
-
});
|
|
76
|
-
it('should handle version strings correctly and exit if required version is not met', () => {
|
|
77
|
-
setNodeVersion('v18.0.0');
|
|
78
|
-
jest.mocked(fs.readFileSync).mockImplementation(() => JSON.stringify({
|
|
79
|
-
engines: { node: '>=18.0.1' },
|
|
80
|
-
}));
|
|
81
|
-
expect((0, checkNodeVersion_1.checkNodeVersion)()).toBeUndefined();
|
|
82
|
-
expect(logger_1.default.warn).toHaveBeenCalledWith(chalk_1.default.yellow('You are using Node.js 18.0.0. This version is not supported. Please use Node.js >=18.0.1.'));
|
|
33
|
+
setNodeVersion('v18');
|
|
34
|
+
(0, checkNodeVersion_1.checkNodeVersion)();
|
|
35
|
+
expect(logger_1.default.warn).toHaveBeenCalledWith(chalk_1.default.yellow('Unexpected Node.js version format: v18. Please use Node.js >=18.0.1.'));
|
|
83
36
|
});
|
|
84
37
|
});
|
|
85
38
|
//# sourceMappingURL=checkNodeVersion.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"checkNodeVersion.test.js","sourceRoot":"","sources":["../../test/checkNodeVersion.test.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"checkNodeVersion.test.js","sourceRoot":"","sources":["../../test/checkNodeVersion.test.ts"],"names":[],"mappings":";;;;;AAAA,kDAA0B;AAC1B,8DAA2D;AAC3D,2DAAmC;AAEnC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAClC,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE;CAC9B,CAAC,CAAC,CAAC;AACJ,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AAE3B,MAAM,cAAc,GAAG,CAAC,OAAe,EAAE,EAAE;IACzC,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE;QACxC,KAAK,EAAE,OAAO;QACd,YAAY,EAAE,IAAI;KACnB,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,MAAM,sBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC;IAE/C,SAAS,CAAC,GAAG,EAAE;QACb,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kFAAkF,EAAE,GAAG,EAAE;QAC1F,cAAc,CAAC,SAAS,CAAC,CAAC;QAE1B,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,mCAAgB,GAAE,CAAC,CAAC,OAAO,CACtC,2FAA2F,CAC5F,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,cAAc,CAAC,SAAS,CAAC,CAAC;QAE1B,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,mCAAgB,GAAE,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,cAAc,CAAC,KAAK,CAAC,CAAC;QAEtB,IAAA,mCAAgB,GAAE,CAAC;QACnB,MAAM,CAAC,gBAAM,CAAC,IAAI,CAAC,CAAC,oBAAoB,CACtC,eAAK,CAAC,MAAM,CAAC,sEAAsE,CAAC,CACrF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -449,6 +449,59 @@ describe('evaluator', () => {
|
|
|
449
449
|
expect(summary.stats.failures).toBe(0);
|
|
450
450
|
expect(summary.results[0].response?.output).toBe('Transformed: Original output');
|
|
451
451
|
});
|
|
452
|
+
it('evaluate with vars transform', async () => {
|
|
453
|
+
const testSuite = {
|
|
454
|
+
providers: [mockApiProvider],
|
|
455
|
+
prompts: [toPrompt('Hello {{ name }}, your age is {{ age }}')],
|
|
456
|
+
tests: [
|
|
457
|
+
{
|
|
458
|
+
vars: { name: 'Alice', age: 30 },
|
|
459
|
+
},
|
|
460
|
+
{
|
|
461
|
+
vars: { name: 'Bob', age: 25 },
|
|
462
|
+
options: {
|
|
463
|
+
transformVars: '{ ...vars, age: vars.age + 5 }',
|
|
464
|
+
},
|
|
465
|
+
},
|
|
466
|
+
],
|
|
467
|
+
defaultTest: {
|
|
468
|
+
options: {
|
|
469
|
+
transformVars: '{ ...vars, name: vars.name.toUpperCase() }',
|
|
470
|
+
},
|
|
471
|
+
},
|
|
472
|
+
};
|
|
473
|
+
await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
|
|
474
|
+
stats: expect.objectContaining({
|
|
475
|
+
successes: 2,
|
|
476
|
+
failures: 0,
|
|
477
|
+
}),
|
|
478
|
+
results: expect.arrayContaining([
|
|
479
|
+
expect.objectContaining({
|
|
480
|
+
prompt: expect.objectContaining({
|
|
481
|
+
raw: 'Hello ALICE, your age is 30',
|
|
482
|
+
label: 'Hello {{ name }}, your age is {{ age }}',
|
|
483
|
+
}),
|
|
484
|
+
response: expect.objectContaining({
|
|
485
|
+
output: 'Test output',
|
|
486
|
+
}),
|
|
487
|
+
}),
|
|
488
|
+
expect.objectContaining({
|
|
489
|
+
// NOTE: test overrides defaultTest transform. Bob not BOB
|
|
490
|
+
prompt: expect.objectContaining({
|
|
491
|
+
raw: 'Hello Bob, your age is 30',
|
|
492
|
+
}),
|
|
493
|
+
response: expect.objectContaining({
|
|
494
|
+
output: 'Test output',
|
|
495
|
+
}),
|
|
496
|
+
vars: {
|
|
497
|
+
name: 'Bob',
|
|
498
|
+
age: 30,
|
|
499
|
+
},
|
|
500
|
+
}),
|
|
501
|
+
]),
|
|
502
|
+
}));
|
|
503
|
+
expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
|
|
504
|
+
});
|
|
452
505
|
it('evaluate with provider transform and test transform', async () => {
|
|
453
506
|
const mockApiProviderWithTransform = {
|
|
454
507
|
id: jest.fn().mockReturnValue('test-provider-transform'),
|