promptfoo 0.96.0 → 0.96.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/package.json +12 -12
- package/dist/src/app/assets/{index-Dwt7E2K_.js → index-CAof5cIE.js} +217 -217
- package/dist/src/app/assets/{index.es-CooNf3HB.js → index.es-CrujH26S.js} +1 -1
- package/dist/src/app/assets/{sync-Bj1WJrHQ.js → sync-gFAH4kmH.js} +1 -1
- package/dist/src/app/index.html +1 -1
- package/dist/src/assertions/answerRelevance.d.ts +3 -0
- package/dist/src/assertions/answerRelevance.d.ts.map +1 -0
- package/dist/src/assertions/answerRelevance.js +19 -0
- package/dist/src/assertions/answerRelevance.js.map +1 -0
- package/dist/src/assertions/bleu.d.ts +4 -14
- package/dist/src/assertions/bleu.d.ts.map +1 -1
- package/dist/src/assertions/bleu.js +19 -3
- package/dist/src/assertions/bleu.js.map +1 -1
- package/dist/src/assertions/classifier.d.ts +3 -0
- package/dist/src/assertions/classifier.d.ts.map +1 -0
- package/dist/src/assertions/classifier.js +22 -0
- package/dist/src/assertions/classifier.js.map +1 -0
- package/dist/src/assertions/contains.d.ts +8 -0
- package/dist/src/assertions/contains.d.ts.map +1 -0
- package/dist/src/assertions/contains.js +110 -0
- package/dist/src/assertions/contains.js.map +1 -0
- package/dist/src/assertions/contextFaithfulness.d.ts +3 -0
- package/dist/src/assertions/contextFaithfulness.d.ts.map +1 -0
- package/dist/src/assertions/contextFaithfulness.js +19 -0
- package/dist/src/assertions/contextFaithfulness.js.map +1 -0
- package/dist/src/assertions/contextRecall.d.ts +3 -0
- package/dist/src/assertions/contextRecall.d.ts.map +1 -0
- package/dist/src/assertions/contextRecall.js +18 -0
- package/dist/src/assertions/contextRecall.js.map +1 -0
- package/dist/src/assertions/contextRelevance.d.ts +3 -0
- package/dist/src/assertions/contextRelevance.d.ts.map +1 -0
- package/dist/src/assertions/contextRelevance.js +19 -0
- package/dist/src/assertions/contextRelevance.js.map +1 -0
- package/dist/src/assertions/cost.d.ts +3 -0
- package/dist/src/assertions/cost.d.ts.map +1 -0
- package/dist/src/assertions/cost.js +22 -0
- package/dist/src/assertions/cost.js.map +1 -0
- package/dist/src/assertions/equals.d.ts +4 -0
- package/dist/src/assertions/equals.d.ts.map +1 -0
- package/dist/src/assertions/equals.js +32 -0
- package/dist/src/assertions/equals.js.map +1 -0
- package/dist/src/assertions/factuality.d.ts +3 -0
- package/dist/src/assertions/factuality.d.ts.map +1 -0
- package/dist/src/assertions/factuality.js +25 -0
- package/dist/src/assertions/factuality.js.map +1 -0
- package/dist/src/assertions/index.d.ts.map +1 -1
- package/dist/src/assertions/index.js +89 -752
- package/dist/src/assertions/index.js.map +1 -1
- package/dist/src/assertions/javascript.d.ts +4 -0
- package/dist/src/assertions/javascript.d.ts.map +1 -0
- package/dist/src/assertions/javascript.js +94 -0
- package/dist/src/assertions/javascript.js.map +1 -0
- package/dist/src/assertions/json.d.ts +3 -4
- package/dist/src/assertions/json.d.ts.map +1 -1
- package/dist/src/assertions/json.js +2 -2
- package/dist/src/assertions/json.js.map +1 -1
- package/dist/src/assertions/latency.d.ts +3 -0
- package/dist/src/assertions/latency.d.ts.map +1 -0
- package/dist/src/assertions/latency.js +22 -0
- package/dist/src/assertions/latency.js.map +1 -0
- package/dist/src/assertions/levenshtein.d.ts +3 -0
- package/dist/src/assertions/levenshtein.d.ts.map +1 -0
- package/dist/src/assertions/levenshtein.js +22 -0
- package/dist/src/assertions/levenshtein.js.map +1 -0
- package/dist/src/assertions/llmRubric.d.ts +3 -0
- package/dist/src/assertions/llmRubric.d.ts.map +1 -0
- package/dist/src/assertions/llmRubric.js +22 -0
- package/dist/src/assertions/llmRubric.js.map +1 -0
- package/dist/src/assertions/modelGradedClosedQa.d.ts +3 -0
- package/dist/src/assertions/modelGradedClosedQa.d.ts.map +1 -0
- package/dist/src/assertions/modelGradedClosedQa.js +25 -0
- package/dist/src/assertions/modelGradedClosedQa.js.map +1 -0
- package/dist/src/assertions/moderation.d.ts +3 -0
- package/dist/src/assertions/moderation.d.ts.map +1 -0
- package/dist/src/assertions/moderation.js +41 -0
- package/dist/src/assertions/moderation.js.map +1 -0
- package/dist/src/assertions/openai.d.ts +5 -0
- package/dist/src/assertions/openai.d.ts.map +1 -0
- package/dist/src/assertions/openai.js +68 -0
- package/dist/src/assertions/openai.js.map +1 -0
- package/dist/src/assertions/perplexity.d.ts +3 -3
- package/dist/src/assertions/perplexity.d.ts.map +1 -1
- package/dist/src/assertions/perplexity.js +2 -2
- package/dist/src/assertions/perplexity.js.map +1 -1
- package/dist/src/assertions/python.d.ts +4 -0
- package/dist/src/assertions/python.d.ts.map +1 -0
- package/dist/src/assertions/python.js +107 -0
- package/dist/src/assertions/python.js.map +1 -0
- package/dist/src/assertions/redteam.d.ts +3 -0
- package/dist/src/assertions/redteam.d.ts.map +1 -0
- package/dist/src/assertions/redteam.js +29 -0
- package/dist/src/assertions/redteam.js.map +1 -0
- package/dist/src/assertions/regex.d.ts +3 -0
- package/dist/src/assertions/regex.d.ts.map +1 -0
- package/dist/src/assertions/regex.js +34 -0
- package/dist/src/assertions/regex.js.map +1 -0
- package/dist/src/assertions/rouge.d.ts +3 -0
- package/dist/src/assertions/rouge.d.ts.map +1 -0
- package/dist/src/assertions/rouge.js +47 -0
- package/dist/src/assertions/rouge.js.map +1 -0
- package/dist/src/assertions/similar.d.ts +3 -0
- package/dist/src/assertions/similar.d.ts.map +1 -0
- package/dist/src/assertions/similar.js +40 -0
- package/dist/src/assertions/similar.js.map +1 -0
- package/dist/src/assertions/sql.d.ts +3 -3
- package/dist/src/assertions/sql.d.ts.map +1 -1
- package/dist/src/assertions/sql.js +14 -3
- package/dist/src/assertions/sql.js.map +1 -1
- package/dist/src/assertions/startsWith.d.ts +3 -0
- package/dist/src/assertions/startsWith.d.ts.map +1 -0
- package/dist/src/assertions/startsWith.js +22 -0
- package/dist/src/assertions/startsWith.js.map +1 -0
- package/dist/src/assertions/utils.d.ts +1 -0
- package/dist/src/assertions/utils.d.ts.map +1 -1
- package/dist/src/assertions/utils.js +7 -0
- package/dist/src/assertions/utils.js.map +1 -1
- package/dist/src/assertions/webhook.d.ts +3 -0
- package/dist/src/assertions/webhook.d.ts.map +1 -0
- package/dist/src/assertions/webhook.js +55 -0
- package/dist/src/assertions/webhook.js.map +1 -0
- package/dist/src/assertions/xml.d.ts +2 -0
- package/dist/src/assertions/xml.d.ts.map +1 -1
- package/dist/src/assertions/xml.js +29 -0
- package/dist/src/assertions/xml.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +3 -0
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/fetch.d.ts.map +1 -1
- package/dist/src/fetch.js +8 -1
- package/dist/src/fetch.js.map +1 -1
- package/dist/src/models/evalResult.d.ts +1 -1
- package/dist/src/models/evalResult.d.ts.map +1 -1
- package/dist/src/models/evalResult.js +1 -1
- package/dist/src/models/evalResult.js.map +1 -1
- package/dist/src/onboarding.d.ts +12 -0
- package/dist/src/onboarding.d.ts.map +1 -1
- package/dist/src/onboarding.js +83 -43
- package/dist/src/onboarding.js.map +1 -1
- package/dist/src/providers/anthropic.d.ts.map +1 -1
- package/dist/src/providers/anthropic.js +12 -2
- package/dist/src/providers/anthropic.js.map +1 -1
- package/dist/src/providers/bedrock.d.ts.map +1 -1
- package/dist/src/providers/bedrock.js +6 -3
- package/dist/src/providers/bedrock.js.map +1 -1
- package/dist/src/redteam/constants.d.ts +4 -3
- package/dist/src/redteam/constants.d.ts.map +1 -1
- package/dist/src/redteam/constants.js +43 -35
- package/dist/src/redteam/constants.js.map +1 -1
- package/dist/src/redteam/index.js +10 -3
- package/dist/src/redteam/index.js.map +1 -1
- package/dist/src/redteam/plugins/base.d.ts.map +1 -1
- package/dist/src/redteam/plugins/base.js +2 -1
- package/dist/src/redteam/plugins/base.js.map +1 -1
- package/dist/src/redteam/plugins/contracts.d.ts.map +1 -1
- package/dist/src/redteam/plugins/contracts.js +20 -9
- package/dist/src/redteam/plugins/contracts.js.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.d.ts.map +1 -1
- package/dist/src/redteam/plugins/crossSessionLeak.js +21 -10
- package/dist/src/redteam/plugins/crossSessionLeak.js.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.d.ts.map +1 -1
- package/dist/src/redteam/plugins/debugAccess.js +20 -9
- package/dist/src/redteam/plugins/debugAccess.js.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.d.ts.map +1 -1
- package/dist/src/redteam/plugins/excessiveAgency.js +18 -7
- package/dist/src/redteam/plugins/excessiveAgency.js.map +1 -1
- package/dist/src/redteam/plugins/hallucination.d.ts.map +1 -1
- package/dist/src/redteam/plugins/hallucination.js +18 -7
- package/dist/src/redteam/plugins/hallucination.js.map +1 -1
- package/dist/src/redteam/plugins/harmful.d.ts +2 -2
- package/dist/src/redteam/plugins/harmful.d.ts.map +1 -1
- package/dist/src/redteam/plugins/harmful.js +44 -26
- package/dist/src/redteam/plugins/harmful.js.map +1 -1
- package/dist/src/redteam/plugins/imitation.d.ts.map +1 -1
- package/dist/src/redteam/plugins/imitation.js +20 -9
- package/dist/src/redteam/plugins/imitation.js.map +1 -1
- package/dist/src/redteam/plugins/index.d.ts +2 -2
- package/dist/src/redteam/plugins/index.d.ts.map +1 -1
- package/dist/src/redteam/plugins/index.js +8 -8
- package/dist/src/redteam/plugins/index.js.map +1 -1
- package/dist/src/redteam/plugins/overreliance.d.ts.map +1 -1
- package/dist/src/redteam/plugins/overreliance.js +18 -7
- package/dist/src/redteam/plugins/overreliance.js.map +1 -1
- package/dist/src/redteam/plugins/pii.d.ts +2 -2
- package/dist/src/redteam/plugins/pii.d.ts.map +1 -1
- package/dist/src/redteam/plugins/pii.js +4 -4
- package/dist/src/redteam/plugins/pii.js.map +1 -1
- package/dist/src/redteam/plugins/politics.d.ts.map +1 -1
- package/dist/src/redteam/plugins/politics.js +20 -9
- package/dist/src/redteam/plugins/politics.js.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.d.ts.map +1 -1
- package/dist/src/redteam/plugins/promptExtraction.js +23 -12
- package/dist/src/redteam/plugins/promptExtraction.js.map +1 -1
- package/dist/src/redteam/plugins/rbac.d.ts.map +1 -1
- package/dist/src/redteam/plugins/rbac.js +20 -9
- package/dist/src/redteam/plugins/rbac.js.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/shellInjection.js +20 -9
- package/dist/src/redteam/plugins/shellInjection.js.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.d.ts.map +1 -1
- package/dist/src/redteam/plugins/sqlInjection.js +18 -7
- package/dist/src/redteam/plugins/sqlInjection.js.map +1 -1
- package/dist/src/redteam/providers/goat.d.ts +2 -0
- package/dist/src/redteam/providers/goat.d.ts.map +1 -1
- package/dist/src/redteam/providers/goat.js +9 -2
- package/dist/src/redteam/providers/goat.js.map +1 -1
- package/dist/src/redteam/strategies/crescendo.d.ts +1 -1
- package/dist/src/redteam/strategies/crescendo.d.ts.map +1 -1
- package/dist/src/redteam/strategies/crescendo.js +2 -1
- package/dist/src/redteam/strategies/crescendo.js.map +1 -1
- package/dist/src/redteam/strategies/goat.d.ts.map +1 -1
- package/dist/src/redteam/strategies/goat.js +1 -0
- package/dist/src/redteam/strategies/goat.js.map +1 -1
- package/dist/src/redteam/strategies/index.js +8 -8
- package/dist/src/redteam/strategies/index.js.map +1 -1
- package/dist/src/redteam/strategies/iterative.d.ts +1 -1
- package/dist/src/redteam/strategies/iterative.d.ts.map +1 -1
- package/dist/src/redteam/strategies/iterative.js +2 -1
- package/dist/src/redteam/strategies/iterative.js.map +1 -1
- package/dist/src/redteam/types.d.ts +11 -1
- package/dist/src/redteam/types.d.ts.map +1 -1
- package/dist/src/types/index.d.ts +109 -2
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js +21 -26
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/util/config/load.d.ts.map +1 -1
- package/dist/src/util/config/load.js +9 -3
- package/dist/src/util/config/load.js.map +1 -1
- package/dist/src/util/convertEvalResultsToTable.d.ts.map +1 -1
- package/dist/src/util/convertEvalResultsToTable.js +4 -8
- package/dist/src/util/convertEvalResultsToTable.js.map +1 -1
- package/dist/src/util/exportToFile/index.d.ts.map +1 -1
- package/dist/src/validators/redteam.d.ts.map +1 -1
- package/dist/src/validators/redteam.js +6 -5
- package/dist/src/validators/redteam.js.map +1 -1
- package/dist/test/assertions/bleu.test.js +38 -13
- package/dist/test/assertions/bleu.test.js.map +1 -1
- package/dist/test/assertions/sql.test.js +268 -167
- package/dist/test/assertions/sql.test.js.map +1 -1
- package/dist/test/commands/init.test.js +1 -1
- package/dist/test/fetch.test.js +17 -0
- package/dist/test/fetch.test.js.map +1 -1
- package/dist/test/onboarding.test.js +8 -7
- package/dist/test/onboarding.test.js.map +1 -1
- package/dist/test/providers/anthropic.test.js +44 -5
- package/dist/test/providers/anthropic.test.js.map +1 -1
- package/dist/test/providers/bedrock.test.js +1 -4
- package/dist/test/providers/bedrock.test.js.map +1 -1
- package/dist/test/util/config/load.test.js +46 -1
- package/dist/test/util/config/load.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +12 -12
- package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240617.json +0 -10
- package/dist/src/redteam/eval/excessive-agency/llm_rubric-20240618.json +0 -10
- package/dist/src/redteam/eval/harmful/llm_rubric-20240723.json +0 -10
- package/dist/src/redteam/eval/harmful/llm_rubric-20240724.json +0 -10
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { AssertionParams } from '../types';
|
|
2
|
+
import { type GradingResult } from '../types';
|
|
3
|
+
export declare const handlePython: ({ assertion, renderedValue, valueFromScript, context, output, }: AssertionParams) => Promise<GradingResult>;
|
|
4
|
+
//# sourceMappingURL=python.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"python.d.ts","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,KAAK,aAAa,EAAmB,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,YAAY,oEAMtB,eAAe,KAAG,OAAO,CAAC,aAAa,CA0GzC,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.handlePython = void 0;
|
|
7
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
|
+
const wrapper_1 = require("../python/wrapper");
|
|
9
|
+
const types_1 = require("../types");
|
|
10
|
+
const handlePython = async ({ assertion, renderedValue, valueFromScript, context, output, }) => {
|
|
11
|
+
(0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'python assertion must have a string value');
|
|
12
|
+
let pass;
|
|
13
|
+
let score;
|
|
14
|
+
try {
|
|
15
|
+
let result;
|
|
16
|
+
if (typeof valueFromScript === 'undefined') {
|
|
17
|
+
const isMultiline = renderedValue.includes('\n');
|
|
18
|
+
let indentStyle = ' ';
|
|
19
|
+
if (isMultiline) {
|
|
20
|
+
// Detect the indentation style of the first indented line
|
|
21
|
+
const match = renderedValue.match(/^(?!\s*$)\s+/m);
|
|
22
|
+
if (match) {
|
|
23
|
+
indentStyle = match[0];
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const pythonScript = `import json
|
|
27
|
+
|
|
28
|
+
def main(output, context):
|
|
29
|
+
${isMultiline
|
|
30
|
+
? renderedValue
|
|
31
|
+
.split('\n')
|
|
32
|
+
.map((line) => `${indentStyle}${line}`)
|
|
33
|
+
.join('\n')
|
|
34
|
+
: ` return ${renderedValue}`}
|
|
35
|
+
`;
|
|
36
|
+
result = await (0, wrapper_1.runPythonCode)(pythonScript, 'main', [output, context]);
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
result = valueFromScript;
|
|
40
|
+
}
|
|
41
|
+
if ((typeof result === 'boolean' && result) ||
|
|
42
|
+
(typeof result === 'string' && result.toLowerCase() === 'true')) {
|
|
43
|
+
pass = true;
|
|
44
|
+
score = 1.0;
|
|
45
|
+
}
|
|
46
|
+
else if ((typeof result === 'boolean' && !result) ||
|
|
47
|
+
(typeof result === 'string' && result.toLowerCase() === 'false')) {
|
|
48
|
+
pass = false;
|
|
49
|
+
score = 0.0;
|
|
50
|
+
}
|
|
51
|
+
else if (typeof result === 'string' && result.startsWith('{')) {
|
|
52
|
+
let parsed;
|
|
53
|
+
try {
|
|
54
|
+
parsed = JSON.parse(result);
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
throw new Error(`Invalid JSON: ${err} when parsing result: ${result}`);
|
|
58
|
+
}
|
|
59
|
+
if (!(0, types_1.isGradingResult)(parsed)) {
|
|
60
|
+
throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead: ${result}`);
|
|
61
|
+
}
|
|
62
|
+
return parsed;
|
|
63
|
+
}
|
|
64
|
+
else if (typeof result === 'object') {
|
|
65
|
+
if (!(0, types_1.isGradingResult)(result)) {
|
|
66
|
+
throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead:\n${JSON.stringify(result, null, 2)}`);
|
|
67
|
+
}
|
|
68
|
+
const pythonGradingResult = result;
|
|
69
|
+
if (assertion.threshold && pythonGradingResult.score < assertion.threshold) {
|
|
70
|
+
pythonGradingResult.pass = false;
|
|
71
|
+
pythonGradingResult.reason = `Python score ${pythonGradingResult.score} is less than threshold ${assertion.threshold}`;
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
...pythonGradingResult,
|
|
75
|
+
assertion,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
score = Number.parseFloat(String(result));
|
|
80
|
+
pass = assertion.threshold ? score >= assertion.threshold : score > 0;
|
|
81
|
+
if (Number.isNaN(score)) {
|
|
82
|
+
throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
|
|
83
|
+
}
|
|
84
|
+
if (typeof assertion.threshold !== 'undefined' && score < assertion.threshold) {
|
|
85
|
+
pass = false;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
return {
|
|
91
|
+
pass: false,
|
|
92
|
+
score: 0,
|
|
93
|
+
reason: `Python code execution failed: ${err.message}`,
|
|
94
|
+
assertion,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
pass,
|
|
99
|
+
score,
|
|
100
|
+
reason: pass
|
|
101
|
+
? 'Assertion passed'
|
|
102
|
+
: `Python code returned ${pass ? 'true' : 'false'}\n${assertion.value}`,
|
|
103
|
+
assertion,
|
|
104
|
+
};
|
|
105
|
+
};
|
|
106
|
+
exports.handlePython = handlePython;
|
|
107
|
+
//# sourceMappingURL=python.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"python.js","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,+CAAkD;AAElD,oCAA+D;AAExD,MAAM,YAAY,GAAG,KAAK,EAAE,EACjC,SAAS,EACT,aAAa,EACb,eAAe,EACf,OAAO,EACP,MAAM,GACU,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,2CAA2C,CAAC,CAAC;IAC1F,IAAI,IAAI,CAAC;IACT,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,IAAI,MAAsE,CAAC;QAC3E,IAAI,OAAO,eAAe,KAAK,WAAW,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,WAAW,GAAG,MAAM,CAAC;YACzB,IAAI,WAAW,EAAE,CAAC;gBAChB,0DAA0D;gBAC1D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACnD,IAAI,KAAK,EAAE,CAAC;oBACV,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,MAAM,YAAY,GAAG;;;EAIzB,WAAW;gBACT,CAAC,CAAC,aAAa;qBACV,KAAK,CAAC,IAAI,CAAC;qBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,WAAW,GAAG,IAAI,EAAE,CAAC;qBACtC,IAAI,CAAC,IAAI,CAAC;gBACf,CAAC,CAAC,cAAc,aAAa,EACjC;CACC,CAAC;YACI,MAAM,GAAG,MAAM,IAAA,uBAAa,EAAC,YAAY,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,eAAe,CAAC;QAC3B,CAAC;QAED,IACE,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC;YACvC,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,EAC/D,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;YACZ,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;aAAM,IACL,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC;YACxC,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,EAChE,CAAC;YACD,IAAI,GAAG,KAAK,CAAC;YACb,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,iGAAiG,MAAM,EAAE,CAC1G,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,kGAAkG,IAAI,CAAC,SAAS,CAC9G,MAAM,EACN,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,mBAAmB,GAAG,MAA0C,CAAC;YACvE,IAAI,SAAS,CAAC,SAAS,IAAI,mBAAmB,CAAC,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC3E,mBAAmB,CAAC,IAAI,GAAG,KAAK,CAAC;gBACjC,mBAAmB,CAAC,MAAM,GAAG,gBAAgB,mBAAmB,CAAC,KAAK,2BAA2B,SAAS,CAAC,SAAS,EAAE,CAAC;YACzH,CAAC;YACD,OAAO;gBACL,GAAG,mBAAmB;gBACtB,SAAS;aACV,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1C,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YACtE,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,kGAAkG,MAAM,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,WAAW,IAAI,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC9E,IAAI,GAAG,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,iCAAkC,GAAa,CAAC,OAAO,EAAE;YACjE,SAAS;SACV,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,KAAK,EAAE;QACzE,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAhHW,QAAA,YAAY,gBAgHvB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,aAAa,kFAQvB,eAAe,KAAG,OAAO,CAAC,aAAa,CAwBzC,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.handleRedteam = void 0;
|
|
7
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
|
+
const graders_1 = require("../redteam/graders");
|
|
9
|
+
const handleRedteam = async ({ assertion, baseType, test, prompt, outputString, provider, renderedValue, }) => {
|
|
10
|
+
const grader = (0, graders_1.getGraderById)(assertion.type);
|
|
11
|
+
(0, tiny_invariant_1.default)(grader, `Unknown promptfoo grader: ${baseType}`);
|
|
12
|
+
(0, tiny_invariant_1.default)(prompt, `Promptfoo grader ${baseType} must have a prompt`);
|
|
13
|
+
const { grade, rubric, suggestions } = await grader.getResult(prompt, outputString, test, provider, renderedValue);
|
|
14
|
+
return {
|
|
15
|
+
assertion: {
|
|
16
|
+
...assertion,
|
|
17
|
+
value: rubric,
|
|
18
|
+
},
|
|
19
|
+
...grade,
|
|
20
|
+
suggestions,
|
|
21
|
+
metadata: {
|
|
22
|
+
// Pass through all test metadata for redteam
|
|
23
|
+
...test.metadata,
|
|
24
|
+
...grade.metadata,
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
exports.handleRedteam = handleRedteam;
|
|
29
|
+
//# sourceMappingURL=redteam.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,gDAAmD;AAG5C,MAAM,aAAa,GAAG,KAAK,EAAE,EAClC,SAAS,EACT,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,YAAY,EACZ,QAAQ,EACR,aAAa,GACG,EAA0B,EAAE;IAC5C,MAAM,MAAM,GAAG,IAAA,uBAAa,EAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAA,wBAAS,EAAC,MAAM,EAAE,6BAA6B,QAAQ,EAAE,CAAC,CAAC;IAC3D,IAAA,wBAAS,EAAC,MAAM,EAAE,oBAAoB,QAAQ,qBAAqB,CAAC,CAAC;IACrE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAC3D,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,QAAQ,EACR,aAAa,CACd,CAAC;IACF,OAAO;QACL,SAAS,EAAE;YACT,GAAG,SAAS;YACZ,KAAK,EAAE,MAAM;SACd;QACD,GAAG,KAAK;QACR,WAAW;QACX,QAAQ,EAAE;YACR,6CAA6C;YAC7C,GAAG,IAAI,CAAC,QAAQ;YAChB,GAAG,KAAK,CAAC,QAAQ;SAClB;KACF,CAAC;AACJ,CAAC,CAAC;AAhCW,QAAA,aAAa,iBAgCxB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"regex.d.ts","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,WAAW,yDAKrB,eAAe,KAAG,aAuBpB,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.handleRegex = void 0;
|
|
7
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
|
+
const handleRegex = ({ assertion, renderedValue, outputString, inverse, }) => {
|
|
9
|
+
(0, tiny_invariant_1.default)(renderedValue, '"regex" assertion type must have a string value');
|
|
10
|
+
(0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"regex" assertion type must have a string value');
|
|
11
|
+
let regex;
|
|
12
|
+
try {
|
|
13
|
+
regex = new RegExp(renderedValue);
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
return {
|
|
17
|
+
pass: false,
|
|
18
|
+
score: 0,
|
|
19
|
+
reason: `Invalid regex pattern: ${error instanceof Error ? error.message : 'unknown error'}`,
|
|
20
|
+
assertion,
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
const pass = regex.test(outputString) !== inverse;
|
|
24
|
+
return {
|
|
25
|
+
pass,
|
|
26
|
+
score: pass ? 1 : 0,
|
|
27
|
+
reason: pass
|
|
28
|
+
? 'Assertion passed'
|
|
29
|
+
: `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
|
|
30
|
+
assertion,
|
|
31
|
+
};
|
|
32
|
+
};
|
|
33
|
+
exports.handleRegex = handleRegex;
|
|
34
|
+
//# sourceMappingURL=regex.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"regex.js","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AAGhC,MAAM,WAAW,GAAG,CAAC,EAC1B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,iDAAiD,CAAC,CAAC;IAC5E,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,iDAAiD,CAAC,CAAC;IAChG,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE;YAC5F,SAAS;SACV,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,OAAO,CAAC;IAClD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,gBAAgB,aAAa,GAAG;QAC/E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA5BW,QAAA,WAAW,eA4BtB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rouge.d.ts","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,wBAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACR,EAAE,eAAe,GAAG,aAAa,CAkBjC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.handleRougeScore = handleRougeScore;
|
|
30
|
+
const rouge = __importStar(require("js-rouge"));
|
|
31
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
32
|
+
function handleRougeScore({ baseType, assertion, renderedValue, outputString, inverse, }) {
|
|
33
|
+
(0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"rouge" assertion type must be a string value');
|
|
34
|
+
const fnName = baseType[baseType.length - 1];
|
|
35
|
+
const rougeMethod = rouge[fnName];
|
|
36
|
+
const score = rougeMethod(outputString, renderedValue, {});
|
|
37
|
+
const pass = score >= (assertion.threshold || 0.75) != inverse;
|
|
38
|
+
return {
|
|
39
|
+
pass,
|
|
40
|
+
score: inverse ? 1 - score : score,
|
|
41
|
+
reason: pass
|
|
42
|
+
? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${assertion.threshold || 0.75}`
|
|
43
|
+
: `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${assertion.threshold || 0.75}`,
|
|
44
|
+
assertion,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=rouge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rouge.js","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,4CAwBC;AA5BD,gDAAkC;AAClC,oEAAuC;AAGvC,SAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS;IAChB,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAC;IAChE,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;IAC/D,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK;QAClC,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,CAAC,OAAO,CAC9C,CAAC,CACF,0CAA0C,SAAS,CAAC,SAAS,IAAI,IAAI,EAAE;YAC1E,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BACjD,SAAS,CAAC,SAAS,IAAI,IACzB,EAAE;QACN,SAAS;KACV,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"similar.d.ts","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,aAAa,+DAMvB,eAAe,KAAG,OAAO,CAAC,aAAa,CA2CzC,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.handleSimilar = void 0;
|
|
7
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
|
+
const matchers_1 = require("../matchers");
|
|
9
|
+
const handleSimilar = async ({ assertion, renderedValue, outputString, inverse, test, }) => {
|
|
10
|
+
(0, tiny_invariant_1.default)(typeof renderedValue === 'string' || Array.isArray(renderedValue), 'Similarity assertion type must have a string or array of strings value');
|
|
11
|
+
if (Array.isArray(renderedValue)) {
|
|
12
|
+
let minScore = Infinity;
|
|
13
|
+
for (const value of renderedValue) {
|
|
14
|
+
const result = await (0, matchers_1.matchesSimilarity)(value, outputString, assertion.threshold || 0.75, inverse, test.options);
|
|
15
|
+
if (result.pass) {
|
|
16
|
+
return {
|
|
17
|
+
assertion,
|
|
18
|
+
...result,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
if (result.score < minScore) {
|
|
22
|
+
minScore = result.score;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
assertion,
|
|
27
|
+
pass: false,
|
|
28
|
+
score: minScore,
|
|
29
|
+
reason: `None of the provided values met the similarity threshold`,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
return {
|
|
34
|
+
assertion,
|
|
35
|
+
...(await (0, matchers_1.matchesSimilarity)(renderedValue, outputString, assertion.threshold || 0.75, inverse, test.options)),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
exports.handleSimilar = handleSimilar;
|
|
40
|
+
//# sourceMappingURL=similar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"similar.js","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,0CAAgD;AAGzC,MAAM,aAAa,GAAG,KAAK,EAAE,EAClC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,EACP,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EACjE,wEAAwE,CACzE,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACjC,IAAI,QAAQ,GAAG,QAAQ,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,IAAA,4BAAiB,EACpC,KAAK,EACL,YAAY,EACZ,SAAS,CAAC,SAAS,IAAI,IAAI,EAC3B,OAAO,EACP,IAAI,CAAC,OAAO,CACb,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAO;oBACL,SAAS;oBACT,GAAG,MAAM;iBACV,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;gBAC5B,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,OAAO;YACL,SAAS;YACT,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,0DAA0D;SACnE,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,SAAS;YACT,GAAG,CAAC,MAAM,IAAA,4BAAiB,EACzB,aAAa,EACb,YAAY,EACZ,SAAS,CAAC,SAAS,IAAI,IAAI,EAC3B,OAAO,EACP,IAAI,CAAC,OAAO,CACb,CAAC;SACH,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAjDW,QAAA,aAAa,iBAiDxB"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
export declare
|
|
1
|
+
import type { AssertionParams, GradingResult } from '../types';
|
|
2
|
+
export declare const handleIsSql: ({ assertion, renderedValue, outputString, inverse, }: AssertionParams) => Promise<GradingResult>;
|
|
3
|
+
export declare const handleContainsSql: (assertionParams: AssertionParams) => Promise<GradingResult>;
|
|
4
4
|
//# sourceMappingURL=sql.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../../src/assertions/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../../src/assertions/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,WAAW,yDAKrB,eAAe,KAAG,OAAO,CAAC,aAAa,CAyEzC,CAAC;AAEF,eAAO,MAAM,iBAAiB,oBACX,eAAe,KAC/B,OAAO,CAAC,aAAa,CAOvB,CAAC"}
|
|
@@ -23,8 +23,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.
|
|
27
|
-
|
|
26
|
+
exports.handleContainsSql = exports.handleIsSql = void 0;
|
|
27
|
+
const utils_1 = require("./utils");
|
|
28
|
+
const handleIsSql = async ({ assertion, renderedValue, outputString, inverse, }) => {
|
|
28
29
|
let pass = false;
|
|
29
30
|
let databaseType = 'MySQL';
|
|
30
31
|
let whiteTableList;
|
|
@@ -83,5 +84,15 @@ async function isSql(outputString, renderedValue, inverse, assertion) {
|
|
|
83
84
|
reason: pass ? 'Assertion passed' : failureReasons.join(' '),
|
|
84
85
|
assertion,
|
|
85
86
|
};
|
|
86
|
-
}
|
|
87
|
+
};
|
|
88
|
+
exports.handleIsSql = handleIsSql;
|
|
89
|
+
const handleContainsSql = async (assertionParams) => {
|
|
90
|
+
const match = (0, utils_1.coerceString)(assertionParams.outputString).match(/```(?:sql)?([^`]+)```/);
|
|
91
|
+
if (match) {
|
|
92
|
+
const sqlCode = match[1].trim();
|
|
93
|
+
return (0, exports.handleIsSql)({ ...assertionParams, outputString: sqlCode });
|
|
94
|
+
}
|
|
95
|
+
return (0, exports.handleIsSql)(assertionParams);
|
|
96
|
+
};
|
|
97
|
+
exports.handleContainsSql = handleContainsSql;
|
|
87
98
|
//# sourceMappingURL=sql.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sql.js","sourceRoot":"","sources":["../../../src/assertions/sql.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sql.js","sourceRoot":"","sources":["../../../src/assertions/sql.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,mCAAuC;AAEhC,MAAM,WAAW,GAAG,KAAK,EAAE,EAChC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAA0B,EAAE;IAC5C,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,YAAY,GAAW,OAAO,CAAC;IACnC,IAAI,cAAoC,CAAC;IACzC,IAAI,eAAqC,CAAC;IAC1C,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,aAIb,CAAC;QAEF,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,OAAO,CAAC;QAC7C,cAAc,GAAG,KAAK,CAAC,aAAa,CAAC;QACrC,eAAe,GAAG,KAAK,CAAC,cAAc,CAAC;IACzC,CAAC;IAED,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,kDAAO,iBAAiB,IAAE,KAAK,CAAC,GAAG,EAAE;QACvE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;IAElC,MAAM,GAAG,GAAoB,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAExD,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,IAAI,CAAC;QACH,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACpC,IAAI,GAAG,CAAC,OAAO,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,OAAO,CAAC;QACf,cAAc,CAAC,IAAI,CACjB,kDAAkD,YAAY,mBAAmB,CAClF,CAAC;IACJ,CAAC;IAED,IAAI,cAAc,EAAE,CAAC;QACnB,GAAG,CAAC,IAAI,GAAG,OAAO,CAAC;QACnB,IAAI,CAAC;YACH,SAAS,CAAC,cAAc,CAAC,YAAY,EAAE,cAAc,EAAE,GAAG,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,OAAO,CAAC;YACf,MAAM,KAAK,GAAG,GAAY,CAAC;YAC3B,cAAc,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QACpB,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;QACpB,IAAI,CAAC;YACH,SAAS,CAAC,cAAc,CAAC,YAAY,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,OAAO,CAAC;YACf,MAAM,KAAK,GAAG,GAAY,CAAC;YAC3B,cAAc,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,IAAI,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,cAAc,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5D,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA9EW,QAAA,WAAW,eA8EtB;AAEK,MAAM,iBAAiB,GAAG,KAAK,EACpC,eAAgC,EACR,EAAE;IAC1B,MAAM,KAAK,GAAG,IAAA,oBAAY,EAAC,eAAe,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACxF,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,IAAA,mBAAW,EAAC,EAAE,GAAG,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAA,mBAAW,EAAC,eAAe,CAAC,CAAC;AACtC,CAAC,CAAC;AATW,QAAA,iBAAiB,qBAS5B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"startsWith.d.ts","sourceRoot":"","sources":["../../../src/assertions/startsWith.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,gBAAgB,yDAK1B,eAAe,KAAG,aAepB,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.handleStartsWith = void 0;
|
|
7
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
|
+
const handleStartsWith = ({ assertion, renderedValue, outputString, inverse, }) => {
|
|
9
|
+
(0, tiny_invariant_1.default)(renderedValue, '"starts-with" assertion type must have a string value');
|
|
10
|
+
(0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"starts-with" assertion type must have a string value');
|
|
11
|
+
const pass = outputString.startsWith(String(renderedValue)) !== inverse;
|
|
12
|
+
return {
|
|
13
|
+
pass,
|
|
14
|
+
score: pass ? 1 : 0,
|
|
15
|
+
reason: pass
|
|
16
|
+
? 'Assertion passed'
|
|
17
|
+
: `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
|
|
18
|
+
assertion,
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
exports.handleStartsWith = handleStartsWith;
|
|
22
|
+
//# sourceMappingURL=startsWith.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"startsWith.js","sourceRoot":"","sources":["../../../src/assertions/startsWith.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AAGhC,MAAM,gBAAgB,GAAG,CAAC,EAC/B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,uDAAuD,CAAC,CAAC;IAClF,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,uDAAuD,CACxD,CAAC;IACF,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,OAAO,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,aAAa,GAAG;QAC9E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AApBW,QAAA,gBAAgB,oBAoB3B"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Assertion, type TestCase } from '../types';
|
|
2
2
|
export declare function getFinalTest(test: TestCase, assertion: Assertion): Readonly<TestCase>;
|
|
3
3
|
export declare function processFileReference(fileRef: string): object | string;
|
|
4
|
+
export declare function coerceString(value: string | object): string;
|
|
4
5
|
//# sourceMappingURL=utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/assertions/utils.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,QAAQ,EAAE,MAAM,UAAU,CAAC;AAIzD,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,sBAUhE;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAYrE"}
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/assertions/utils.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,QAAQ,EAAE,MAAM,UAAU,CAAC;AAIzD,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,sBAUhE;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAYrE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAK3D"}
|
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.getFinalTest = getFinalTest;
|
|
7
7
|
exports.processFileReference = processFileReference;
|
|
8
|
+
exports.coerceString = coerceString;
|
|
8
9
|
const fs_1 = __importDefault(require("fs"));
|
|
9
10
|
const js_yaml_1 = __importDefault(require("js-yaml"));
|
|
10
11
|
const path_1 = __importDefault(require("path"));
|
|
@@ -36,4 +37,10 @@ function processFileReference(fileRef) {
|
|
|
36
37
|
throw new Error(`Unsupported file type: ${filePath}`);
|
|
37
38
|
}
|
|
38
39
|
}
|
|
40
|
+
function coerceString(value) {
|
|
41
|
+
if (typeof value === 'string') {
|
|
42
|
+
return value;
|
|
43
|
+
}
|
|
44
|
+
return JSON.stringify(value);
|
|
45
|
+
}
|
|
39
46
|
//# sourceMappingURL=utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/assertions/utils.ts"],"names":[],"mappings":";;;;;AASA,oCAUC;AAED,oDAYC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/assertions/utils.ts"],"names":[],"mappings":";;;;;AASA,oCAUC;AAED,oDAYC;AAED,oCAKC;AAxCD,4CAAoB;AACpB,sDAA2B;AAC3B,gDAAwB;AACxB,gDAAyB;AACzB,2DAAmC;AAGnC,MAAM,KAAK,GAAG,IAAA,cAAK,GAAE,CAAC;AAEtB,SAAgB,YAAY,CAAC,IAAc,EAAE,SAAoB;IAC/D,YAAY;IACZ,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAExB,6CAA6C;IAC7C,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;IAChC,mEAAmE;IACnE,GAAG,CAAC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC;IACrE,GAAG,CAAC,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,YAAY,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC;IAC9E,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAgB,oBAAoB,CAAC,OAAe;IAClD,MAAM,QAAQ,GAAG,kBAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;IACzC,MAAM,QAAQ,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnD,OAAO,iBAAI,CAAC,IAAI,CAAC,WAAW,CAAW,CAAC;IAC1C,CAAC;SAAM,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QAChC,OAAO,WAAW,CAAC,IAAI,EAAE,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,0BAA0B,QAAQ,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,SAAgB,YAAY,CAAC,KAAsB;IACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"webhook.d.ts","sourceRoot":"","sources":["../../../src/assertions/webhook.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,wBAAsB,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,IAAI,EACJ,MAAM,EACN,MAAM,EACN,OAAO,GACR,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAqD1C"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.handleWebhook = handleWebhook;
|
|
7
|
+
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
|
|
8
|
+
const envars_1 = require("../envars");
|
|
9
|
+
const fetch_1 = require("../fetch");
|
|
10
|
+
async function handleWebhook({ assertion, renderedValue, test, prompt, output, inverse, }) {
|
|
11
|
+
(0, tiny_invariant_1.default)(renderedValue, '"webhook" assertion type must have a URL value');
|
|
12
|
+
(0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
|
|
13
|
+
try {
|
|
14
|
+
const context = {
|
|
15
|
+
prompt,
|
|
16
|
+
vars: test.vars || {},
|
|
17
|
+
};
|
|
18
|
+
const response = await (0, fetch_1.fetchWithRetries)(renderedValue, {
|
|
19
|
+
method: 'POST',
|
|
20
|
+
headers: {
|
|
21
|
+
'Content-Type': 'application/json',
|
|
22
|
+
},
|
|
23
|
+
body: JSON.stringify({ output, context }),
|
|
24
|
+
}, (0, envars_1.getEnvInt)('WEBHOOK_TIMEOUT', 5000));
|
|
25
|
+
if (!response.ok) {
|
|
26
|
+
throw new Error(`Webhook response status: ${response.status}`);
|
|
27
|
+
}
|
|
28
|
+
const jsonResponse = await response.json();
|
|
29
|
+
const pass = jsonResponse.pass !== inverse;
|
|
30
|
+
const score = typeof jsonResponse.score === 'undefined'
|
|
31
|
+
? pass
|
|
32
|
+
? 1
|
|
33
|
+
: 0
|
|
34
|
+
: inverse
|
|
35
|
+
? 1 - jsonResponse.score
|
|
36
|
+
: jsonResponse.score;
|
|
37
|
+
const reason = jsonResponse.reason ||
|
|
38
|
+
(pass ? 'Assertion passed' : `Webhook returned ${inverse ? 'true' : 'false'}`);
|
|
39
|
+
return {
|
|
40
|
+
pass,
|
|
41
|
+
score,
|
|
42
|
+
reason,
|
|
43
|
+
assertion,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
catch (err) {
|
|
47
|
+
return {
|
|
48
|
+
pass: false,
|
|
49
|
+
score: 0,
|
|
50
|
+
reason: `Webhook error: ${err.message}`,
|
|
51
|
+
assertion,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=webhook.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"webhook.js","sourceRoot":"","sources":["../../../src/assertions/webhook.ts"],"names":[],"mappings":";;;;;AAKA,sCA4DC;AAjED,oEAAuC;AACvC,sCAAsC;AACtC,oCAA4C;AAGrC,KAAK,UAAU,aAAa,CAAC,EAClC,SAAS,EACT,aAAa,EACb,IAAI,EACJ,MAAM,EACN,MAAM,EACN,OAAO,GACS;IAChB,IAAA,wBAAS,EAAC,aAAa,EAAE,gDAAgD,CAAC,CAAC;IAC3E,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,gDAAgD,CAAC,CAAC;IAC/F,IAAI,CAAC;QACH,MAAM,OAAO,GAAG;YACd,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;SACtB,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EACrC,aAAa,EACb;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;SAC1C,EACD,IAAA,kBAAS,EAAC,iBAAiB,EAAE,IAAI,CAAC,CACnC,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,KAAK,OAAO,CAAC;QAC3C,MAAM,KAAK,GACT,OAAO,YAAY,CAAC,KAAK,KAAK,WAAW;YACvC,CAAC,CAAC,IAAI;gBACJ,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,OAAO;gBACP,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,KAAK;gBACxB,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC;QAE3B,MAAM,MAAM,GACV,YAAY,CAAC,MAAM;YACnB,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAEjF,OAAO;YACL,IAAI;YACJ,KAAK;YACL,MAAM;YACN,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,kBAAmB,GAAa,CAAC,OAAO,EAAE;YAClD,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { AssertionParams, GradingResult } from '../types';
|
|
1
2
|
export declare function validateXml(xmlString: string, requiredElements?: string[]): {
|
|
2
3
|
isValid: boolean;
|
|
3
4
|
reason: string;
|
|
@@ -6,4 +7,5 @@ export declare function containsXml(outputString: string, requiredElements?: str
|
|
|
6
7
|
isValid: boolean;
|
|
7
8
|
reason: string;
|
|
8
9
|
};
|
|
10
|
+
export declare const handleIsXml: ({ assertion, renderedValue, outputString, inverse, baseType, }: AssertionParams) => GradingResult;
|
|
9
11
|
//# sourceMappingURL=xml.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../../src/assertions/xml.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"xml.d.ts","sourceRoot":"","sources":["../../../src/assertions/xml.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAC1B;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAsCtC;AAED,wBAAgB,WAAW,CACzB,YAAY,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,MAAM,EAAE,GAC1B;IAAE,OAAO,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAgBtC;AAED,eAAO,MAAM,WAAW,mEAMrB,eAAe,KAAG,aA6BpB,CAAC"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleIsXml = void 0;
|
|
3
4
|
exports.validateXml = validateXml;
|
|
4
5
|
exports.containsXml = containsXml;
|
|
5
6
|
const fast_xml_parser_1 = require("fast-xml-parser");
|
|
@@ -54,4 +55,32 @@ function containsXml(outputString, requiredElements) {
|
|
|
54
55
|
}
|
|
55
56
|
return { isValid: false, reason: 'No valid XML content found matching the requirements' };
|
|
56
57
|
}
|
|
58
|
+
const handleIsXml = ({ assertion, renderedValue, outputString, inverse, baseType, }) => {
|
|
59
|
+
let requiredElements;
|
|
60
|
+
if (typeof renderedValue === 'string') {
|
|
61
|
+
requiredElements = renderedValue.split(',').map((el) => el.trim());
|
|
62
|
+
}
|
|
63
|
+
else if (Array.isArray(renderedValue) && renderedValue.length > 0) {
|
|
64
|
+
requiredElements = renderedValue.map((el) => el.toString());
|
|
65
|
+
}
|
|
66
|
+
else if (renderedValue !== null &&
|
|
67
|
+
typeof renderedValue === 'object' &&
|
|
68
|
+
Object.keys(renderedValue).length > 0) {
|
|
69
|
+
if ('requiredElements' in renderedValue && Array.isArray(renderedValue.requiredElements)) {
|
|
70
|
+
requiredElements = renderedValue.requiredElements.map((el) => el.toString());
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
throw new Error('xml assertion must contain a string, array value, or no value');
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const result = (baseType === 'is-xml' ? validateXml : containsXml)(outputString, requiredElements);
|
|
77
|
+
const pass = result.isValid !== inverse;
|
|
78
|
+
return {
|
|
79
|
+
pass,
|
|
80
|
+
score: pass ? 1 : 0,
|
|
81
|
+
reason: pass ? 'Assertion passed' : result.reason,
|
|
82
|
+
assertion,
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
exports.handleIsXml = handleIsXml;
|
|
57
86
|
//# sourceMappingURL=xml.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../../src/assertions/xml.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"xml.js","sourceRoot":"","sources":["../../../src/assertions/xml.ts"],"names":[],"mappings":";;;AAGA,kCAyCC;AAED,kCAmBC;AAjED,qDAA4C;AAG5C,SAAgB,WAAW,CACzB,SAAiB,EACjB,gBAA2B;IAE3B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,2BAAS,CAAC;QAC3B,sBAAsB,EAAE,IAAI;QAC5B,gBAAgB,EAAE,KAAK;QACvB,mBAAmB,EAAE,IAAI;QACzB,aAAa,EAAE,IAAI;KACpB,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,eAAe,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,OAAO,GAAQ,SAAS,CAAC;gBAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;wBAC/B,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,qCAAqC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBAC1E,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,iDAAiD,EAAE,CAAC;IACtF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAwB,GAAa,CAAC,OAAO,EAAE,EAAE,CAAC;IACrF,CAAC;AACH,CAAC;AAED,SAAgB,WAAW,CACzB,YAAoB,EACpB,gBAA2B;IAE3B,MAAM,QAAQ,GAAG,uDAAuD,CAAC;IACzE,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,oCAAoC,EAAE,CAAC;IAC1E,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QACpE,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,sDAAsD,EAAE,CAAC;AAC5F,CAAC;AAEM,MAAM,WAAW,GAAG,CAAC,EAC1B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,EACP,QAAQ,GACQ,EAAiB,EAAE;IACnC,IAAI,gBAAsC,CAAC;IAC3C,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,gBAAgB,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpE,gBAAgB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9D,CAAC;SAAM,IACL,aAAa,KAAK,IAAI;QACtB,OAAO,aAAa,KAAK,QAAQ;QACjC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EACrC,CAAC;QACD,IAAI,kBAAkB,IAAI,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACzF,gBAAgB,GAAG,aAAa,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAChE,YAAY,EACZ,gBAAgB,CACjB,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC;IACxC,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;QACjD,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAnCW,QAAA,WAAW,eAmCtB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAazC,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAO/E;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBzF;AAED,wBAAsB,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAS3F;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CA0BzD;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAYrD;AAgED,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QAyC3C"}
|